# The Wheel Atlas — robots policy
# Strategy: welcome AI *search* crawlers (they drive citations), block AI *training*
# crawlers (no model training on our data).

# Default: all crawlers (Googlebot, Bingbot, etc.) — full access except private areas.
User-agent: *
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /auth
Disallow: /callback
Disallow: /go/

# --- AI SEARCH crawlers: ALLOWED (citations are the whole point) ---
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /auth
Disallow: /callback
Disallow: /go/

User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /auth
Disallow: /callback
Disallow: /go/

User-agent: Perplexity-User
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /auth
Disallow: /callback
Disallow: /go/

# --- AI TRAINING crawlers: BLOCKED (no model training on our data) ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Added 2026-09-01 to match the Cloudflare AI Crawl Control block list (the edge
# enforces; this keeps polite bots from even asking). B2B grounding + AI-training
# data vendors — none has a citing/retrieval role for us. Tokens per Cloudflare Radar.
User-agent: Google-CloudVertexBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: TikTokSpider
Disallow: /

User-agent: Novellum
Disallow: /

User-agent: Terracotta
Disallow: /

User-agent: Timpibot
Disallow: /

# Applebot-Extended never crawls: it is Apple's robots-only opt-out that stops
# Applebot's already-fetched pages being used for Apple Intelligence training.
# Applebot itself (Siri/Spotlight/Safari) stays allowed — no Cloudflare equivalent.
User-agent: Applebot-Extended
Disallow: /

# Added 2026-10-02 (Chris; manual 03 §3.16): Parallel Web Systems resells its index as
# full-page extraction and price monitoring (Extract / Monitor APIs) — a data-resale crawler,
# not a citing search engine for us. ShapBot ignored the `*` group's `Disallow: /go/` (3,080
# /go hits on 2026-10-01) and Shap-User is documented as not obeying robots.txt, so the
# Cloudflare edge enforces; this keeps a polite client from asking.
User-agent: ShapBot
Disallow: /

User-agent: Shap-User
Disallow: /

Sitemap: https://thewheelatlas.com/sitemap.xml