# gtm-api.com robots.txt: AEO crawler policy.
#
# Served by the theme (inc/robots-txt.php replaces the body WordPress
# generates), so this file is the source of truth and ships with a theme
# deploy. Never hand-edit robots.txt on the droplet: a physical file in the
# docroot shadows this one and takes the policy back out of git.
#
# Position: we want to be BOTH retrieved and trained on. Answer engines cite
# what they retrieve, and sitting in the training data is distribution, not
# leakage. Everything public is open. Fenced: wp-admin, the REST API, the
# WordPress feeds other than the site feed, the search-result URLs and
# xmlrpc, none of which are content.
#
# Those Disallows are crawl hygiene, not a position. Every one of those URLs
# duplicates a page that is already in the sitemap and open to every crawler
# below. Search feeds in particular mint a new URL per query, so the space is
# unbounded. The site feed /feed/ stays open: Search Console names a feed as
# the referring page that first led Googlebot to /safe-linkedin-automation/.
# The PHP half of the same decision is inc/feeds.php.
#
# Content-Signal (contentsignals.org, Cloudflare): search / ai-input / ai-train
# each take yes|no. ai-input covers RAG and grounding, the channel that
# actually produces citations, and the one Cloudflare's managed default left
# unset entirely. use=full is the permissive end of Cloudflare's extension.
#
# One group per audience rather than one per bot: RFC 9309 has a crawler obey
# only its single best-matching group, so a per-bot group would hide the
# Content-Signal line from every bot that has one. Consecutive User-agent lines
# share a group, which keeps the roster explicit without splitting the signal.
#
# User-agent strings evolve: re-verify quarterly against a maintained list.

# ---- Named crawlers we care about ----
# AI answer engines (they cite sources):
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
# AI training crawlers (we want to be in the models):
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: cohere-ai
User-agent: CCBot
# Classic search indexes (Googlebot also feeds AI Overviews):
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes, use=full
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-json/
Allow: /wp-json/gtm/v1/status
Disallow: /xmlrpc.php
Disallow: /search/
Disallow: /*?s=
Disallow: /*/feed/
Disallow: /*?feed=

# ---- Everyone else ----
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes, use=full
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-json/
Allow: /wp-json/gtm/v1/status
Disallow: /xmlrpc.php
Disallow: /search/
Disallow: /*?s=
Disallow: /*/feed/
Disallow: /*?feed=

Sitemap: https://gtm-api.com/sitemap_index.xml