{
  "id": "access.robots.search-and-user-agents",
  "protocol": "access",
  "layer": "discovery",
  "role": "merchant",
  "status": "recommended",
  "title": "Allow AI search and user-triggered agents on product pages",
  "description": "Search or retrieval bots (OAI-SearchBot, Claude-SearchBot, PerplexityBot, Meta-WebIndexer, Applebot) index pages so assistants can cite and link them. User-triggered agents (ChatGPT-User, Claude-User, Perplexity-User, Google-Agent, Meta-ExternalFetcher) fetch a page when a shopper asks.",
  "goal": "Shoppers who ask an assistant about your products can only be sent to pages it was allowed to read.",
  "note": "Recommended for shops, our reading: OpenAI says sites opted out of OAI-SearchBot will not be shown in ChatGPT search answers; Anthropic says blocking Claude-SearchBot or Claude-User may reduce visibility. OpenAI, Perplexity, Google and Meta say their user-triggered fetchers may not follow robots.txt, while Anthropic lets sites control Claude-User through it. Not a vendor requirement.",
  "how_to_implement": {
    "steps": [
      "Do not disallow search or user tokens on product, category and policy pages.",
      "If you block bots at the CDN, make sure search and agent bots still reach those pages (see access.cloudflare.ai-bots).",
      "Allow published IP ranges in your WAF where the vendor publishes them."
    ]
  },
  "resources": [
    {
      "label": "OpenAI: overview of OpenAI crawlers",
      "url": "https://platform.openai.com/docs/bots",
      "type": "docs",
      "publisher": "OpenAI"
    },
    {
      "label": "Anthropic: crawlers and how to block them",
      "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "type": "docs",
      "publisher": "Anthropic"
    },
    {
      "label": "Perplexity: crawlers",
      "url": "https://docs.perplexity.ai/guides/bots",
      "type": "docs",
      "publisher": "Perplexity"
    },
    {
      "label": "Google: user-triggered fetchers (incl. Google-Agent)",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Meta: web crawlers",
      "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "type": "docs",
      "publisher": "Meta"
    },
    {
      "label": "Apple: About Applebot",
      "url": "https://support.apple.com/en-us/119829",
      "type": "docs",
      "publisher": "Apple"
    },
    {
      "label": "RFC 9309: Robots Exclusion Protocol",
      "url": "https://www.rfc-editor.org/rfc/rfc9309.html",
      "type": "spec",
      "publisher": "IETF"
    }
  ],
  "audit": {
    "method": "robots_txt",
    "automatable": true,
    "url_pattern": "/robots.txt",
    "expect": {
      "bot_kinds": [
        "search",
        "user"
      ],
      "allow_paths": [
        "/",
        "{product_url}"
      ]
    },
    "human": "For every search and user token, / and a sample product URL resolve to allowed under RFC 9309 matching."
  },
  "how_to_verify": "For every search and user token, / and a sample product URL resolve to allowed under RFC 9309 matching.",
  "artifacts": [
    "/robots.txt"
  ],
  "depends_on": [
    "access.robots.ai-crawlers"
  ],
  "status_note": "Recommended for shops, our reading: OpenAI says sites opted out of OAI-SearchBot will not be shown in ChatGPT search answers; Anthropic says blocking Claude-SearchBot or Claude-User may reduce visibility. OpenAI, Perplexity, Google and Meta say their user-triggered fetchers may not follow robots.txt, while Anthropic lets sites control Claude-User through it. Not a vendor requirement.",
  "verified": true,
  "status_rationale": "Recommended for shops because blocking removes products from AI answers per the vendors' own docs; a business can still decide otherwise.",
  "slug": "access-robots-search-and-user-agents",
  "category": "discovery",
  "url": "https://vreeman.ai/commerce/checklist#access.robots.search-and-user-agents",
  "updated": "2026-10-12",
  "sources": [
    {
      "label": "OpenAI: overview of OpenAI crawlers",
      "url": "https://platform.openai.com/docs/bots",
      "type": "docs",
      "publisher": "OpenAI"
    },
    {
      "label": "Anthropic: crawlers and how to block them",
      "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "type": "docs",
      "publisher": "Anthropic"
    },
    {
      "label": "Perplexity: crawlers",
      "url": "https://docs.perplexity.ai/guides/bots",
      "type": "docs",
      "publisher": "Perplexity"
    },
    {
      "label": "Google: user-triggered fetchers (incl. Google-Agent)",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Meta: web crawlers",
      "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "type": "docs",
      "publisher": "Meta"
    },
    {
      "label": "Apple: About Applebot",
      "url": "https://support.apple.com/en-us/119829",
      "type": "docs",
      "publisher": "Apple"
    },
    {
      "label": "RFC 9309: Robots Exclusion Protocol",
      "url": "https://www.rfc-editor.org/rfc/rfc9309.html",
      "type": "spec",
      "publisher": "IETF"
    }
  ],
  "related_slugs": [
    "access-robots-ai-crawlers"
  ]
}