{
  "id": "access.robots.merchant-center",
  "protocol": "access",
  "layer": "discovery",
  "role": "merchant",
  "status": "required",
  "title": "Let Googlebot, Googlebot-Image and Storebot-Google crawl product pages, checkout and images",
  "description": "Google's StoreBot crawls product detail, cart and checkout pages to collect price, shipping, availability and payment data and to verify what you submit in Merchant Center; Google calls allowing it highly encouraged. When StoreBot cannot reach an in-store product page, Merchant Center issues a warning or suspension, and Google's fix is to let StoreBot, Googlebot and Googlebot-Image through robots.txt, bot detection, firewalls and CDNs.",
  "goal": "Avoid Merchant Center warnings or suspensions and keep Merchant Center data verifiable on Google Shopping surfaces.",
  "note": "Required for shops using Merchant Center, based on Google's own warning-or-suspension text for in-store pages (answer 13484511). For online-only product pages, Google's StoreBot page (answer 13294660) says highly encouraged and states no penalty. Google advises reverse DNS verification over static IP allowlists, because StoreBot can use different IPs than search crawlers. Product images often sit on a CDN host with its own robots.txt.",
  "how_to_implement": {
    "steps": [
      "robots.txt: do not disallow Storebot-Google, Googlebot or Googlebot-Image on product, cart, checkout and image paths; Google's example is a group per token with an empty Disallow.",
      "Bot detection, WAF or CDN (for example Cloudflare): make sure these crawlers get no 403 or 404; verify Google by reverse DNS rather than static IP lists.",
      "Keep landing pages fast, since slow pages can cause crawler timeouts.",
      "Add product structured data: it takes precedence over what StoreBot extracts.",
      "Then reprocess your feed in Merchant Center."
    ]
  },
  "resources": [
    {
      "label": "Google Merchant Center: fix StoreBot crawler can't access your in-store product page",
      "url": "https://support.google.com/merchants/answer/13484511",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Google Merchant Center: About the Google StoreBot crawler",
      "url": "https://support.google.com/merchants/answer/13294660",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Google: common crawlers (incl. Google-Extended)",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "RFC 9309: Robots Exclusion Protocol",
      "url": "https://www.rfc-editor.org/rfc/rfc9309.html",
      "type": "spec",
      "publisher": "IETF"
    }
  ],
  "audit": {
    "method": "robots_txt",
    "automatable": true,
    "url_pattern": "/robots.txt",
    "expect": {
      "bot_kinds": [
        "merchant_center"
      ],
      "allow_paths": [
        "/",
        "{product_url}",
        "{image_url}"
      ]
    },
    "human": "For Googlebot, Googlebot-Image and Storebot-Google: / , a sample product URL and a sample product image URL resolve to allowed. WAF, bot detection and CDN blocks need a manual check in Merchant Center diagnostics."
  },
  "how_to_verify": "For Googlebot, Googlebot-Image and Storebot-Google: / , a sample product URL and a sample product image URL resolve to allowed. WAF, bot detection and CDN blocks need a manual check in Merchant Center diagnostics.",
  "artifacts": [
    "/robots.txt"
  ],
  "depends_on": [
    "access.robots.ai-crawlers"
  ],
  "status_note": "Required for shops using Merchant Center, based on Google's own warning-or-suspension text for in-store pages (answer 13484511). For online-only product pages, Google's StoreBot page (answer 13294660) says highly encouraged and states no penalty. Google advises reverse DNS verification over static IP allowlists, because StoreBot can use different IPs than search crawlers. Product images often sit on a CDN host with its own robots.txt.",
  "verified": true,
  "status_rationale": "Required: Google issues a warning or suspension when StoreBot cannot access in-store product pages (answer 13484511).",
  "slug": "access-robots-merchant-center",
  "category": "discovery",
  "url": "https://vreeman.ai/commerce/checklist#access.robots.merchant-center",
  "updated": "2026-10-12",
  "sources": [
    {
      "label": "Google Merchant Center: fix StoreBot crawler can't access your in-store product page",
      "url": "https://support.google.com/merchants/answer/13484511",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Google Merchant Center: About the Google StoreBot crawler",
      "url": "https://support.google.com/merchants/answer/13294660",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "Google: common crawlers (incl. Google-Extended)",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "type": "docs",
      "publisher": "Google"
    },
    {
      "label": "RFC 9309: Robots Exclusion Protocol",
      "url": "https://www.rfc-editor.org/rfc/rfc9309.html",
      "type": "spec",
      "publisher": "IETF"
    }
  ],
  "related_slugs": [
    "access-robots-ai-crawlers"
  ]
}