{
  "schema_version": "1.0.0",
  "last_checked": "2026-10-19",
  "note": "AI crawler and agent robots.txt tokens, grouped by kind. Sources are only each vendor's own bot documentation. Kinds: training (content may train models), search (indexes content to show or cite it in AI answers), user (fetches a page when a person asks; vendors say these may not follow robots.txt). Blocking training does not block search or user agents. verified:false means no official vendor page was found for that exact claim.",
  "kinds": {
    "training": "Crawls or controls use of content for training AI models. Allowing or blocking is a business choice.",
    "search": "Indexes content so an AI search or assistant can show, cite and link it. Blocking removes you from those answers.",
    "user": "Fetches a page in real time when a person asks an assistant or agent. Vendors state robots.txt may not apply.",
    "classic_search": "Traditional search engine crawlers that AI features also build on.",
    "other": "Generic crawlers not tied to a specific product.",
    "merchant_center": "Audit group: Googlebot, Googlebot-Image and Storebot-Google, the crawlers Merchant Center product, landing and image checks depend on (our grouping)."
  },
  "bots": [
    {
      "token": "GPTBot",
      "vendor": "OpenAI",
      "kind": "training",
      "respects_robots": true,
      "purpose": "Content that may be used to train OpenAI foundation models.",
      "source": "https://platform.openai.com/docs/bots",
      "verified": true
    },
    {
      "token": "OAI-SearchBot",
      "vendor": "OpenAI",
      "kind": "search",
      "respects_robots": true,
      "purpose": "Surfaces sites in ChatGPT search results.",
      "source": "https://platform.openai.com/docs/bots",
      "verified": true
    },
    {
      "token": "ChatGPT-User",
      "vendor": "OpenAI",
      "kind": "user",
      "respects_robots": false,
      "purpose": "User actions in ChatGPT and Custom GPTs; OpenAI says robots.txt rules may not apply.",
      "source": "https://platform.openai.com/docs/bots",
      "verified": true
    },
    {
      "token": "OAI-AdsBot",
      "vendor": "OpenAI",
      "kind": "other",
      "respects_robots": null,
      "purpose": "Validates the safety of landing pages submitted as ads on ChatGPT and may use their content to decide when an ad is relevant. Only visits pages submitted as ads; data is not used to train generative AI foundation models. OpenAI's page does not say whether it follows robots.txt.",
      "source": "https://developers.openai.com/api/docs/bots",
      "verified": true
    },
    {
      "token": "ClaudeBot",
      "vendor": "Anthropic",
      "kind": "training",
      "respects_robots": true,
      "purpose": "Collects content that could contribute to model training.",
      "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "verified": true
    },
    {
      "token": "Claude-SearchBot",
      "vendor": "Anthropic",
      "kind": "search",
      "respects_robots": true,
      "purpose": "Improves search result quality for Claude users.",
      "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "verified": true
    },
    {
      "token": "Claude-User",
      "vendor": "Anthropic",
      "kind": "user",
      "respects_robots": true,
      "purpose": "Fetches pages when a user asks Claude; Anthropic lets sites control it via robots.txt.",
      "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "verified": true
    },
    {
      "token": "Google-Extended",
      "vendor": "Google",
      "kind": "training",
      "respects_robots": true,
      "purpose": "Control token (no own crawler): use of crawled content for Gemini training and grounding in Gemini Apps and Vertex AI. Does not affect Google Search.",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "verified": true
    },
    {
      "token": "Googlebot",
      "vendor": "Google",
      "kind": "classic_search",
      "respects_robots": true,
      "purpose": "Google Search, including AI features in Search.",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "verified": true,
      "merchant_center": true
    },
    {
      "token": "Storebot-Google",
      "vendor": "Google",
      "kind": "classic_search",
      "respects_robots": true,
      "purpose": "All Google Shopping surfaces. Merchant Center: crawls product, cart and checkout pages to verify price, shipping and availability; allowing it is 'highly encouraged'.",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "verified": true,
      "merchant_center": true,
      "source2": [
        "https://support.google.com/merchants/answer/13294660",
        "https://support.google.com/merchants/answer/13484511"
      ]
    },
    {
      "token": "Googlebot-Image",
      "vendor": "Google",
      "kind": "classic_search",
      "respects_robots": true,
      "merchant_center": true,
      "purpose": "Google Images and image features in Search; also matches Googlebot rules.",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "verified": true
    },
    {
      "token": "GoogleOther",
      "vendor": "Google",
      "kind": "other",
      "respects_robots": true,
      "purpose": "Generic Google crawler for one-off crawls such as internal research; affects no specific product.",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "verified": true
    },
    {
      "token": "Google-Agent",
      "vendor": "Google",
      "kind": "user",
      "respects_robots": false,
      "purpose": "Agents on Google infrastructure acting on user request; user-triggered fetchers generally ignore robots.txt. Google is experimenting with Web Bot Auth for it.",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers",
      "verified": true
    },
    {
      "token": "Applebot",
      "vendor": "Apple",
      "kind": "search",
      "respects_robots": true,
      "purpose": "Search in Spotlight, Siri and Safari; data may also be used for Apple models unless Applebot-Extended is disallowed. Falls back to Googlebot rules when not named.",
      "source": "https://support.apple.com/en-us/119829",
      "verified": true
    },
    {
      "token": "Applebot-Extended",
      "vendor": "Apple",
      "kind": "training",
      "respects_robots": true,
      "purpose": "Control token (does not crawl): opt out of training Apple foundation models.",
      "source": "https://support.apple.com/en-us/119829",
      "verified": true
    },
    {
      "token": "PerplexityBot",
      "vendor": "Perplexity",
      "kind": "search",
      "respects_robots": true,
      "purpose": "Surfaces and links sites in Perplexity search; Perplexity says it is not used for foundation model training.",
      "source": "https://docs.perplexity.ai/guides/bots",
      "verified": true
    },
    {
      "token": "Perplexity-User",
      "vendor": "Perplexity",
      "kind": "user",
      "respects_robots": false,
      "purpose": "User actions in Perplexity; generally ignores robots.txt.",
      "source": "https://docs.perplexity.ai/guides/bots",
      "verified": true
    },
    {
      "token": "Meta-ExternalAgent",
      "vendor": "Meta",
      "kind": "training",
      "respects_robots": true,
      "purpose": "Training foundation AI models or indexing content directly.",
      "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "verified": true
    },
    {
      "token": "Meta-WebIndexer",
      "vendor": "Meta",
      "kind": "search",
      "respects_robots": true,
      "purpose": "Meta AI search quality, citing and linking content.",
      "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "verified": true
    },
    {
      "token": "Meta-ExternalFetcher",
      "vendor": "Meta",
      "kind": "user",
      "respects_robots": false,
      "purpose": "Fetches links at a user's request, including agentic tasks; may bypass robots.txt.",
      "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "verified": true
    },
    {
      "token": "Meta-ExternalAds",
      "vendor": "Meta",
      "kind": "other",
      "respects_robots": null,
      "purpose": "Crawls the web for use cases such as improving advertising and other business-related products and services. Meta's page does not say whether it follows robots.txt.",
      "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "verified": true
    },
    {
      "token": "CCBot",
      "vendor": "Common Crawl",
      "kind": "training",
      "respects_robots": true,
      "purpose": "Builds an open repository of web crawl data (Common Crawl) that anyone can access and analyze.",
      "source": "https://commoncrawl.org/ccbot",
      "verified": true
    },
    {
      "token": "Bingbot",
      "vendor": "Microsoft",
      "kind": "classic_search",
      "respects_robots": true,
      "purpose": "Bing index, which Microsoft says also feeds Copilot and grounding results. No separate Copilot token; use NOARCHIVE or NOCACHE to limit Copilot use.",
      "source": "https://www.bing.com/webmasters/help/webmaster-guidelines-30fba23a",
      "verified": true,
      "sources": [
        "https://www.bing.com/webmasters/help/webmaster-guidelines-30fba23a",
        "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0"
      ]
    }
  ],
  "cloudflare": {
    "note": "Cloudflare can block AI bots at the edge regardless of robots.txt. Per Cloudflare docs, from 15 September 2026 new domains default to blocking bots classified as Training or Agent on pages with ads, with Search allowed; presets are Search, Agent and Training. AI Crawl Control lets you allow or block individual crawlers and track robots.txt compliance. Managed robots.txt can prepend AI disallow rules.",
    "sources": [
      "https://developers.cloudflare.com/bots/additional-configurations/block-ai-bots/",
      "https://developers.cloudflare.com/ai-crawl-control/",
      "https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/",
      "https://developers.cloudflare.com/bots/reference/bot-verification/web-bot-auth/"
    ]
  },
  "spec": {
    "robots_txt": "https://www.rfc-editor.org/rfc/rfc9309.html",
    "summary": "RFC 9309: a crawler obeys the group whose user-agent matches its product token (case-insensitive), else the * group; most specific (longest) matching rule wins, allow wins a tie; a missing robots.txt (4xx) means everything is allowed; 5xx means assume complete disallow."
  }
}