# robots.txt for PredictAsiaX — AI-visibility optimized
# Goal: appear in ChatGPT / Claude / Gemini / Grok / Perplexity answers about prediction markets.
# All AI training + search + user-fetch bots are explicitly allowed.
# Sensitive routes remain noindex-tagged in-page (see middleware.js).

# ─── Default: allow all, keep API + Next internals off-limits ───
User-agent: *
Allow: /
Allow: /api/og/
Disallow: /api/
Disallow: /_next/

# ─── Content signals: opt-IN for AI use ───
Content-Signal: search=yes,ai-input=yes,ai-train=yes,use=cited

# ─── Major search + social preview crawlers ───
User-agent: Googlebot
Allow: /
Disallow: /api/
Allow: /api/og/

User-agent: Bingbot
Allow: /
Disallow: /api/
Allow: /api/og/

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Discordbot
Allow: /

# ─── AI training + search + user-fetch crawlers (all ALLOWED) ───
# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Google (Gemini training-corpus opt-in)
User-agent: Google-Extended
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Common Crawl — training data for most open-weight LLMs
User-agent: CCBot
Allow: /

# Apple
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta / Llama
User-agent: Meta-ExternalAgent
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

# ByteDance / Doubao
User-agent: Bytespider
Allow: /

# Amazon Alexa / Rufus / Q
User-agent: Amazonbot
Allow: /

# Microsoft Copilot (uses Bingbot, but explicit anyway)
User-agent: DuckAssistBot
Allow: /

# Cloudflare browser-rendering used by many agent frameworks
User-agent: CloudflareBrowserRenderingCrawler
Allow: /

# xAI / Grok
User-agent: xAI-Bot
Allow: /

User-agent: Grokbot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# Diffbot (agent framework)
User-agent: Diffbot
Allow: /

# YouBot / Phind
User-agent: YouBot
Allow: /

User-agent: PhindBot
Allow: /

# ─── SEO tool crawlers — throttled but allowed ───
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

# ─── Sitemap + Feeds ───
Sitemap: https://predictasiax.com/sitemap.xml
# RSS 2.0 feed for /learn + /report content:
# https://predictasiax.com/rss.xml
# JSON Feed 1.1 for the same content:
# https://predictasiax.com/feed.json