{
  "id": "access.cloudflare.ai-bots",
  "protocol": "access",
  "layer": "discovery",
  "role": "merchant",
  "status": "recommended",
  "title": "Check CDN AI bot blocking (Cloudflare AI bot policies, AI Crawl Control)",
  "description": "Cloudflare can block AI bots at the edge whatever robots.txt says. Its AI bot policies have Search, Agent and Training presets. Per Cloudflare, from 15 September 2026 new domains default to blocking Training and Agent bots on pages with ads, with Search allowed. AI Crawl Control allows or blocks individual crawlers and tracks robots.txt compliance; managed robots.txt can add AI disallow rules.",
  "goal": "Avoid an edge rule silently blocking the shopping agents your robots.txt allows.",
  "note": "Cloudflare's Agent preset covers chat fetch bots and browser-use agents, so blocking it can stop user-triggered shopping agents. A remote audit cannot see dashboard settings and does not impersonate bots, so this is a manual check. Signed agents can be recognised with Web Bot Auth (web-bot-auth.verify).",
  "how_to_implement": {
    "steps": [
      "In Cloudflare Security Settings, review Configure AI bot policies: Search, Agent and Training.",
      "For shops, keep Search and Agent allowed at least on product pages; choose Training separately.",
      "Review AI Crawl Control for blocked crawlers and robots.txt violations.",
      "If managed robots.txt is on, check the rules it prepends."
    ]
  },
  "resources": [
    {
      "label": "Cloudflare: Block AI bots and AI bot policies",
      "url": "https://developers.cloudflare.com/bots/additional-configurations/block-ai-bots/",
      "type": "docs",
      "publisher": "Cloudflare"
    },
    {
      "label": "Cloudflare: AI Crawl Control",
      "url": "https://developers.cloudflare.com/ai-crawl-control/",
      "type": "docs",
      "publisher": "Cloudflare"
    },
    {
      "label": "Cloudflare: managed robots.txt",
      "url": "https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/",
      "type": "docs",
      "publisher": "Cloudflare"
    },
    {
      "label": "Cloudflare: Web Bot Auth",
      "url": "https://developers.cloudflare.com/bots/reference/bot-verification/web-bot-auth/",
      "type": "docs",
      "publisher": "Cloudflare"
    }
  ],
  "audit": {
    "method": "platform_console",
    "automatable": false,
    "human": "Cloudflare dashboard: AI bot policies and AI Crawl Control match your robots.txt choices; Agent and Search not blocked on product pages."
  },
  "how_to_verify": "Cloudflare dashboard: AI bot policies and AI Crawl Control match your robots.txt choices; Agent and Search not blocked on product pages.",
  "artifacts": [],
  "depends_on": [
    "access.robots.search-and-user-agents"
  ],
  "status_note": "Cloudflare's Agent preset covers chat fetch bots and browser-use agents, so blocking it can stop user-triggered shopping agents. A remote audit cannot see dashboard settings and does not impersonate bots, so this is a manual check. Signed agents can be recognised with Web Bot Auth (web-bot-auth.verify).",
  "verified": true,
  "slug": "access-cloudflare-ai-bots",
  "category": "discovery",
  "url": "https://vreeman.ai/commerce/checklist#access.cloudflare.ai-bots",
  "updated": "2026-10-12",
  "sources": [
    {
      "label": "Cloudflare: Block AI bots and AI bot policies",
      "url": "https://developers.cloudflare.com/bots/additional-configurations/block-ai-bots/",
      "type": "docs",
      "publisher": "Cloudflare"
    },
    {
      "label": "Cloudflare: AI Crawl Control",
      "url": "https://developers.cloudflare.com/ai-crawl-control/",
      "type": "docs",
      "publisher": "Cloudflare"
    },
    {
      "label": "Cloudflare: managed robots.txt",
      "url": "https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/",
      "type": "docs",
      "publisher": "Cloudflare"
    },
    {
      "label": "Cloudflare: Web Bot Auth",
      "url": "https://developers.cloudflare.com/bots/reference/bot-verification/web-bot-auth/",
      "type": "docs",
      "publisher": "Cloudflare"
    }
  ],
  "related_slugs": [
    "access-robots-search-and-user-agents"
  ]
}