User-agent: *
Allow: /
Disallow: /cdn-cgi/
Disallow: /cities.json

# ── AI policy: answer engines YES, model training NO ──────────────────────────
# These are different crawlers. The bot that indexes us so we show up in AI
# answers is not the bot that harvests training data, so blocking the training
# one costs no visibility. The previous version of this file got that split
# wrong: it blocked the retired "anthropic-ai" token while ClaudeBot (the real
# training crawler) walked straight through.
#
# ALLOWED ON PURPOSE — these drive AI answers and send real referral traffic
# (GA4 shows ~359 sessions/28d from chatgpt.com alone). They inherit the
# "User-agent: *" allow above. Do NOT add Disallow rules for them:
#   OAI-SearchBot     ChatGPT's search index
#   ChatGPT-User      live fetch when someone shares or asks about a link
#   Claude-User       live fetch for a Claude user
#   Claude-SearchBot  Claude's search index
#   PerplexityBot     Perplexity index
#   Perplexity-User   live fetch for a Perplexity user
#   Googlebot         also what feeds Google's AI Overviews
#   Bingbot           also what feeds Copilot
#   Applebot          Siri and Spotlight
#
# BLOCKED — model training only.

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Bytespider
Disallow: /

# Retired OpenAI/Anthropic tokens. Harmless to keep, some older crawlers still
# honor them.
User-agent: anthropic-ai
Disallow: /

# ⛔ Google-Extended is deliberately NOT blocked — do not "fix" this for
# consistency with the training bots above. It's a single switch covering BOTH
# Gemini training and Gemini app grounding, with no way to split them, and being
# cited in Gemini answers is worth more to us than the training opt-out. This is
# the one accepted exception to the no-training rule.

# ── SEO-tool / no-referral crawlers: blocked ──────────────────────────────────
# These crawl the whole site (100k+ URLs) for their own link databases and send
# zero visitors. On 2026-09-11 they were ~28% of all Worker invocations and, since
# they walk the long tail, an outsized share of Worker CPU (every uncached page
# render). A WAF custom rule ("Block SEO-tool / no-referral crawlers by UA") also
# blocks them at the edge; this section is the polite signal so the well-behaved
# ones stop retrying. Search + AI answer engines above are NOT in this list.

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SleepBot
Disallow: /

User-agent: AionBot
Disallow: /

User-agent: ExaSearchBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: KeenableBot
Disallow: /

User-agent: Reflectionbot
Disallow: /

# Meta AI search indexer: ~11k requests/day, zero visitors from meta.ai in GA4 (30d,
# 2026-10-16). Facebook link previews come from facebookexternalhit, not this.
User-agent: meta-webindexer
Disallow: /

# Amazonbot is deliberately NOT blocked: Amazon's Associates site review reads this file
# and rejected us for it (2026-10-02). Keep it allowed while we run Amazon affiliate links.

# Sitemap
Sitemap: https://kidseventsthisweek.com/sitemap.xml