{
  "id": "access.robots.ai-crawlers",
  "protocol": "access",
  "layer": "discovery",
  "role": "merchant",
  "status": "recommended",
  "title": "Know which AI crawlers and agents your robots.txt allows",
  "description": "robots.txt is read per user-agent token (RFC 9309). AI vendors publish separate tokens for training, for search or retrieval, and for user-triggered fetches, so one Disallow rarely does what a shop expects.",
  "goal": "Make a deliberate choice per kind of bot instead of an accidental one: blocking training crawlers does not have to remove your products from AI answers.",
  "note": "Each vendor states its own tokens and semantics; we only list tokens from official vendor pages. Google-Extended and Applebot-Extended are control tokens that do not crawl. Applebot follows Googlebot rules when it is not named. Bingbot feeds the Bing index, which Microsoft says also feeds Copilot and grounding results; there is no separate Copilot token, and NOARCHIVE or NOCACHE limit Copilot use (Bing Webmaster Guidelines). Full token list with kind and source per vendor: /commerce/api/bots (MCP tool list_ai_bots).",
  "how_to_implement": {
    "steps": [
      "List every AI token in /commerce/api/bots and decide allow or disallow per kind (training, search, user).",
      "Write explicit groups per token in robots.txt; a group for a token replaces the * group for that bot.",
      "Remember the most specific (longest) rule wins and allow wins a tie (RFC 9309).",
      "Check product and category paths, not only /.",
      "Re-check after platform or CDN changes, which can rewrite robots.txt."
    ]
  },
  "resources": [
    {
      "label": "RFC 9309: Robots Exclusion Protocol",
      "url": "https://www.rfc-editor.org/rfc/rfc9309.html",
      "type": "spec",
      "publisher": "IETF"
    },
    {
      "label": "OpenAI: overview of OpenAI crawlers",
      "url": "https://platform.openai.com/docs/bots",
      "type": "docs",
      "publisher": "OpenAI"
    },
    {
      "label": "Anthropic: crawlers and how to block them",
      "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "type": "docs",
      "publisher": "Anthropic"
    },
    {
      "label": "Google: common crawlers (incl. Google-Extended)",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Google: user-triggered fetchers (incl. Google-Agent)",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Apple: About Applebot",
      "url": "https://support.apple.com/en-us/119829",
      "type": "docs",
      "publisher": "Apple"
    },
    {
      "label": "Perplexity: crawlers",
      "url": "https://docs.perplexity.ai/guides/bots",
      "type": "docs",
      "publisher": "Perplexity"
    },
    {
      "label": "Meta: web crawlers",
      "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "type": "docs",
      "publisher": "Meta"
    },
    {
      "label": "Common Crawl: CCBot",
      "url": "https://commoncrawl.org/ccbot",
      "type": "docs",
      "publisher": "Common Crawl"
    },
    {
      "label": "Bing Webmaster: which crawlers does Bing use",
      "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "type": "docs",
      "publisher": "Microsoft"
    },
    {
      "label": "Bing Webmaster Guidelines",
      "url": "https://www.bing.com/webmasters/help/webmaster-guidelines-30fba23a",
      "type": "docs",
      "publisher": "Microsoft"
    }
  ],
  "audit": {
    "method": "robots_txt",
    "automatable": true,
    "url_pattern": "/robots.txt",
    "expect": {
      "bot_kinds": [
        "training",
        "search",
        "user",
        "classic_search"
      ],
      "allow_paths": [
        "/",
        "{product_url}"
      ]
    },
    "human": "Fetch /robots.txt, resolve the group for each token in /commerce/api/bots (RFC 9309 matching), and report whether / and a sample product URL are allowed, grouped by kind. A missing robots.txt (4xx) means all allowed."
  },
  "how_to_verify": "Fetch /robots.txt, resolve the group for each token in /commerce/api/bots (RFC 9309 matching), and report whether / and a sample product URL are allowed, grouped by kind. A missing robots.txt (4xx) means all allowed.",
  "artifacts": [
    "/robots.txt"
  ],
  "depends_on": [],
  "status_note": "Each vendor states its own tokens and semantics; we only list tokens from official vendor pages. Google-Extended and Applebot-Extended are control tokens that do not crawl. Applebot follows Googlebot rules when it is not named. Bingbot feeds the Bing index, which Microsoft says also feeds Copilot and grounding results; there is no separate Copilot token, and NOARCHIVE or NOCACHE limit Copilot use (Bing Webmaster Guidelines). Full token list with kind and source per vendor: /commerce/api/bots (MCP tool list_ai_bots).",
  "verified": true,
  "slug": "access-robots-ai-crawlers",
  "category": "discovery",
  "url": "https://vreeman.ai/commerce/checklist#access.robots.ai-crawlers",
  "updated": "2026-10-12",
  "sources": [
    {
      "label": "RFC 9309: Robots Exclusion Protocol",
      "url": "https://www.rfc-editor.org/rfc/rfc9309.html",
      "type": "spec",
      "publisher": "IETF"
    },
    {
      "label": "OpenAI: overview of OpenAI crawlers",
      "url": "https://platform.openai.com/docs/bots",
      "type": "docs",
      "publisher": "OpenAI"
    },
    {
      "label": "Anthropic: crawlers and how to block them",
      "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "type": "docs",
      "publisher": "Anthropic"
    },
    {
      "label": "Google: common crawlers (incl. Google-Extended)",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Google: user-triggered fetchers (incl. Google-Agent)",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Apple: About Applebot",
      "url": "https://support.apple.com/en-us/119829",
      "type": "docs",
      "publisher": "Apple"
    },
    {
      "label": "Perplexity: crawlers",
      "url": "https://docs.perplexity.ai/guides/bots",
      "type": "docs",
      "publisher": "Perplexity"
    },
    {
      "label": "Meta: web crawlers",
      "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "type": "docs",
      "publisher": "Meta"
    },
    {
      "label": "Common Crawl: CCBot",
      "url": "https://commoncrawl.org/ccbot",
      "type": "docs",
      "publisher": "Common Crawl"
    },
    {
      "label": "Bing Webmaster: which crawlers does Bing use",
      "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "type": "docs",
      "publisher": "Microsoft"
    },
    {
      "label": "Bing Webmaster Guidelines",
      "url": "https://www.bing.com/webmasters/help/webmaster-guidelines-30fba23a",
      "type": "docs",
      "publisher": "Microsoft"
    }
  ],
  "related_slugs": []
}