# ForFun.info - Complete Robots.txt
# Optimized for Google, Bing, and AI crawlers
# Last updated: 2025-11-30

# ===========================================
# PRIMARY SEARCH ENGINES (Highest Priority)
# ===========================================

# Google Search
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Bing Search  
User-agent: Bingbot
Allow: /
Crawl-delay: 0

# Yandex Search
User-agent: YandexBot
Allow: /

# Baidu Search
User-agent: Baiduspider
Allow: /

# ===========================================
# AI SEARCH & ASSISTANTS (Allow All)
# ===========================================

# OpenAI ChatGPT Search
User-agent: OAI-SearchBot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic Claude
User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

# Google Gemini & AI
User-agent: Gemini-Bot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-Bot
Allow: /

# ===========================================
# SOCIAL MEDIA CRAWLERS
# ===========================================

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# ===========================================
# BLOCK AGGRESSIVE/UNWANTED BOTS
# ===========================================

# Amazon Alexa Bot
User-agent: Amazonbot
Disallow: /

# Apple Extended (Training)
User-agent: Applebot-Extended
Disallow: /

# ByteDance/TikTok Bot
User-agent: Bytespider
Disallow: /

# Common Crawl Bot
User-agent: CCBot
Disallow: /

# Meta AI Training
User-agent: meta-externalagent
Disallow: /

# ===========================================
# DEFAULT RULE (Allow All Others)
# ===========================================

User-agent: *
Allow: /

# ===========================================
# SITEMAP LOCATION
# ===========================================

Sitemap: https://forfun.info/sitemap.xml

# ===========================================
# NOTES FOR WEBMASTERS
# ===========================================
# This robots.txt allows all major search engines and AI assistants
# while blocking training/scraping bots.
# 
# Verify indexing:
# - Google: https://search.google.com/search-console
# - Bing: https://www.bing.com/webmasters
# 
# Test this file:
# - https://www.google.com/webmasters/tools/robots-testing-tool