# Bayescore — https://bayescore.com
#
# AI retrieval crawlers are explicitly ALLOWED.
# Per RFC 9309 §2.2.2: when Allow and Disallow match the same path,
# the Allow rule takes precedence. This file explicitly grants access
# to all AI retrieval and search crawlers.
#
# Content signals (EU DSM Directive 2019/790, Article 4):
#   search=yes      — indexing and search results: permitted
#   ai-input=yes    — retrieval-augmented generation, live browsing: permitted
#   ai-train=no     — model training and fine-tuning: not permitted

User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /

# ── Standard search engines ──────────────────────────────────────────────────

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# ── AI retrieval crawlers (search / live browsing / RAG) ─────────────────────
# These power live answers in ChatGPT, Claude, Perplexity, Google AI Overviews,
# Apple Intelligence, and similar. Blocking them reduces AI visibility.
# All are retrieval-only — not training crawlers.

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: Cloudflare-AI
Allow: /

User-agent: Diffbot
Allow: /

User-agent: YouBot
Allow: /

User-agent: archive.org_bot
Allow: /

# ── AI training crawlers — blocked ───────────────────────────────────────────
# These crawl for model training / fine-tuning, not retrieval.

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: TikTok Spider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# ── Sitemap ───────────────────────────────────────────────────────────────────

Sitemap: https://bayescore.com/sitemap.xml

# LLM-readable product description:
# https://bayescore.com/llms.txt