# Crawlers: use the sitemaps; /search has an unbounded parameter space and
# every hit costs serverless quota, so it is disallowed (results pages are
# reachable through person/place/surname pages instead).
#
# AI crawlers are blocked entirely: in one week they generated ~2M worker
# requests (mostly Anthropic's crawler range 216.73.216.0/22) link-walking
# the ~13M person/household pages. Traditional search engines (Googlebot,
# Bingbot) are unaffected. Google-Extended/Applebot-Extended only opt out
# of AI training — they do not affect Google Search or Siri/Spotlight.
#
# July 2026: meta-webindexer (Meta AI search, a NEWER token than the
# meta-external* pair below) generated ~1.7M requests/week the same way, and
# Amzn-SearchBot (which ignores the "*" rule and is a different token from
# Amazonbot) added ~60k/week. New crawler tokens appear regularly — when
# request volume spikes, check the top user agents before assuming the list
# below still covers everything. A Cloudflare WAF custom rule also blocks
# these UAs as a backstop, since robots.txt compliance is voluntary.

User-agent: *
Disallow: /search

# --- AI training / scraping bots ---
User-agent: ClaudeBot
Disallow: /

User-agent: Claude-User
Disallow: /

User-agent: Claude-SearchBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: meta-externalfetcher
Disallow: /

User-agent: meta-webindexer
Disallow: /

User-agent: Amzn-SearchBot
Disallow: /

User-agent: Amzn-User
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: BacklinksExtendedBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Timpibot
Disallow: /

Sitemap: https://facts.ie/sitemap.xml
Sitemap: https://facts.ie/sitemap-households-1926.xml