# ── AI Training Crawlers (Block) ─────────────────────────────────
# These crawlers take content to train models rather than to answer a question
# someone is asking right now. Blocking them does NOT affect search indexing or
# AI citation — the crawlers that decide whether an assistant can cite us are
# named separately below, and they stay allowed on purpose.
#
# This file is a request, not a fence: a well-behaved crawler honours it and a
# determined copier ignores it. It is the cheap half of the answer. The half
# that has teeth is the usage terms plus edge-level rate limiting.

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: Scrapy
Disallow: /

# ── AI Search/Citation Crawlers (Allow) ──────────────────────────
# These crawlers power real-time AI search results and citations.
# Allowing them = our site can appear in ChatGPT, Perplexity answers.

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

# The bots above are the training crawlers; these are the SEARCH crawlers, and
# they are the ones that decide whether an assistant can cite us at all. They
# were allowed only by falling through to `User-agent: *` — undeclared, and one
# future catch-all rule away from silently ending our AI visibility. Naming
# them makes that intent explicit and un-clobberable.
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

# ── Standard Search Crawlers (Allow) ─────────────────────────────

User-agent: *
Allow: /
Disallow: /assets/

Sitemap: https://theasmrworld.com/sitemap.xml