{
  "id": "access.robots.training-choice",
  "protocol": "access",
  "layer": "discovery",
  "role": "merchant",
  "status": "optional",
  "title": "Decide on AI training crawlers as a business choice",
  "description": "Training crawlers and tokens (GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, Meta-ExternalAgent, CCBot) govern whether content may be used to train models. Vendors state these are separate from search: for example Google-Extended does not affect Google Search, and OpenAI treats GPTBot and OAI-SearchBot independently.",
  "goal": "Choose deliberately whether your copy and images may train models, without hurting AI search visibility.",
  "note": "Optional: no official source ties training access to shopping visibility. Note that Google-Extended also covers grounding in Gemini Apps and Vertex AI, per Google. CCBot builds an open repository of web crawl data (Common Crawl) that anyone can access and analyze.",
  "how_to_implement": {
    "steps": [
      "Decide allow or disallow per training token; write explicit groups.",
      "Do not use a blanket User-agent: * Disallow to block training, because it also blocks search bots that fall back to *.",
      "Document the choice in your robots.txt comments."
    ]
  },
  "resources": [
    {
      "label": "OpenAI: overview of OpenAI crawlers",
      "url": "https://platform.openai.com/docs/bots",
      "type": "docs",
      "publisher": "OpenAI"
    },
    {
      "label": "Anthropic: crawlers and how to block them",
      "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "type": "docs",
      "publisher": "Anthropic"
    },
    {
      "label": "Google: common crawlers (incl. Google-Extended)",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Apple: About Applebot",
      "url": "https://support.apple.com/en-us/119829",
      "type": "docs",
      "publisher": "Apple"
    },
    {
      "label": "Meta: web crawlers",
      "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "type": "docs",
      "publisher": "Meta"
    },
    {
      "label": "Common Crawl: CCBot",
      "url": "https://commoncrawl.org/ccbot",
      "type": "docs",
      "publisher": "Common Crawl"
    }
  ],
  "audit": {
    "method": "robots_txt",
    "automatable": true,
    "url_pattern": "/robots.txt",
    "expect": {
      "bot_kinds": [
        "training"
      ],
      "allow_paths": [
        "/",
        "{product_url}"
      ]
    },
    "human": "Reports per training token whether / is allowed. Informational: both outcomes are valid choices."
  },
  "how_to_verify": "Reports per training token whether / is allowed. Informational: both outcomes are valid choices.",
  "artifacts": [
    "/robots.txt"
  ],
  "depends_on": [
    "access.robots.ai-crawlers"
  ],
  "status_note": "Optional: no official source ties training access to shopping visibility. Note that Google-Extended also covers grounding in Gemini Apps and Vertex AI, per Google. CCBot builds an open repository of web crawl data (Common Crawl) that anyone can access and analyze.",
  "verified": true,
  "status_rationale": "Training is a business choice, so Optional.",
  "slug": "access-robots-training-choice",
  "category": "discovery",
  "url": "https://vreeman.ai/commerce/checklist#access.robots.training-choice",
  "updated": "2026-10-19",
  "sources": [
    {
      "label": "OpenAI: overview of OpenAI crawlers",
      "url": "https://platform.openai.com/docs/bots",
      "type": "docs",
      "publisher": "OpenAI"
    },
    {
      "label": "Anthropic: crawlers and how to block them",
      "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "type": "docs",
      "publisher": "Anthropic"
    },
    {
      "label": "Google: common crawlers (incl. Google-Extended)",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Apple: About Applebot",
      "url": "https://support.apple.com/en-us/119829",
      "type": "docs",
      "publisher": "Apple"
    },
    {
      "label": "Meta: web crawlers",
      "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "type": "docs",
      "publisher": "Meta"
    },
    {
      "label": "Common Crawl: CCBot",
      "url": "https://commoncrawl.org/ccbot",
      "type": "docs",
      "publisher": "Common Crawl"
    }
  ],
  "related_slugs": [
    "access-robots-ai-crawlers"
  ]
}