{
 "schema": "elementera-ai-crawler-registry/1",
 "name": "Elementera AI crawler registry",
 "url": "https://www.elementera.com/tools/ai-crawler-registry",
 "description": "AI crawler and control tokens with their vendor, purpose, robots.txt behaviour, documentation and published IP ranges. date_verified is the day the vendor documentation page was fetched and the token was found in it; null means no vendor page exists or the token was not found.",
 "last_verified": "2026-10-14",
 "generated_from": {
  "crawler_check_list": {
   "last_verified": "2026-10-05"
  },
  "crawler_access_study": {
   "verified_on": "2026-10-06"
  },
  "ai_robots_txt": {
   "repo": "https://github.com/ai-robots-txt/ai.robots.txt",
   "commit": "9ad8a47e"
  },
  "vendor_pages_checked_on": "2026-10-14"
 },
 "purposes": {
  "search": "Crawls pages so the vendor's AI search or answer product can find and cite them.",
  "user-fetch": "Fetches a page when a person asks the assistant to look at it. Not an automatic crawler.",
  "training": "Crawler or control token that governs use of your content for model training.",
  "other": "A crawler with a narrow purpose that does not decide AI search or training eligibility."
 },
 "groups": {
  "ai_search": "Crawlers that let an AI search or answer product find and cite your pages.",
  "ai_training": "Crawlers and control tokens that govern use of your content for model training.",
  "ai_user_fetch": "Fetchers that read one page when a person asks an assistant to. Not automatic crawlers.",
  "other": "Narrow or mixed-purpose crawlers. They do not decide AI search or training on their own.",
  "web_search": "Search engine crawlers. Blocking one removes your pages from that search engine, including AI answers built on its index."
 },
 "honours_robots_values": {
  "yes": "Follows robots.txt.",
  "may-not": "The vendor says robots.txt may not apply, usually because a person triggered the request.",
  "no": "Reported not to follow robots.txt.",
  "unclear": "No clear statement found."
 },
 "crawlers": [
  {
   "token": "OAI-SearchBot",
   "vendor": "OpenAI",
   "product": "ChatGPT search",
   "purpose": "search",
   "group": "ai_search",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://developers.openai.com/api/docs/bots",
   "ip_ranges_url": "https://openai.com/searchbot.json",
   "date_verified": "2026-10-14",
   "note": null,
   "in_checker": true
  },
  {
   "token": "ChatGPT-User",
   "vendor": "OpenAI",
   "product": "ChatGPT search",
   "purpose": "user-fetch",
   "group": "ai_user_fetch",
   "kind": "user_agent",
   "honours_robots": "may-not",
   "honours_robots_source": "vendor documentation, as recorded in the AI crawler check",
   "docs_url": "https://developers.openai.com/api/docs/bots",
   "ip_ranges_url": "https://openai.com/chatgpt-user.json",
   "date_verified": "2026-10-14",
   "note": "OpenAI says ChatGPT-User is triggered by a user, and robots.txt rules may not apply to it.",
   "in_checker": true
  },
  {
   "token": "GPTBot",
   "vendor": "OpenAI",
   "product": "ChatGPT search",
   "purpose": "training",
   "group": "ai_training",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://developers.openai.com/api/docs/bots",
   "ip_ranges_url": "https://openai.com/gptbot.json",
   "date_verified": "2026-10-14",
   "note": null,
   "in_checker": true
  },
  {
   "token": "OAI-AdsBot",
   "vendor": "OpenAI",
   "product": "ChatGPT search",
   "purpose": "other",
   "group": "other",
   "kind": "user_agent",
   "honours_robots": "unclear",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://developers.openai.com/api/docs/bots",
   "ip_ranges_url": "https://openai.com/adsbot.json",
   "date_verified": "2026-10-14",
   "note": "Checks ad landing pages submitted to OpenAI. OpenAI says its data is not used for training.",
   "in_checker": true
  },
  {
   "token": "Claude-SearchBot",
   "vendor": "Anthropic",
   "product": "Claude's web search",
   "purpose": "search",
   "group": "ai_search",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
   "ip_ranges_url": "https://claude.com/crawling/bots.json",
   "date_verified": "2026-10-14",
   "note": null,
   "in_checker": true
  },
  {
   "token": "Claude-User",
   "vendor": "Anthropic",
   "product": "Claude's web search",
   "purpose": "user-fetch",
   "group": "ai_user_fetch",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
   "ip_ranges_url": "https://claude.com/crawling/bots.json",
   "date_verified": "2026-10-14",
   "note": null,
   "in_checker": true
  },
  {
   "token": "ClaudeBot",
   "vendor": "Anthropic",
   "product": "Claude's web search",
   "purpose": "training",
   "group": "ai_training",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
   "ip_ranges_url": "https://claude.com/crawling/bots.json",
   "date_verified": "2026-10-14",
   "note": null,
   "in_checker": true
  },
  {
   "token": "PerplexityBot",
   "vendor": "Perplexity",
   "product": "Perplexity search",
   "purpose": "search",
   "group": "ai_search",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
   "ip_ranges_url": "https://www.perplexity.com/perplexitybot.json",
   "date_verified": "2026-10-14",
   "note": null,
   "in_checker": true
  },
  {
   "token": "Perplexity-User",
   "vendor": "Perplexity",
   "product": "Perplexity search",
   "purpose": "user-fetch",
   "group": "ai_user_fetch",
   "kind": "user_agent",
   "honours_robots": "may-not",
   "honours_robots_source": "vendor documentation, as recorded in the AI crawler check",
   "docs_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
   "ip_ranges_url": "https://www.perplexity.com/perplexity-user.json",
   "date_verified": "2026-10-14",
   "note": "Perplexity says Perplexity-User is triggered by a user and generally ignores robots.txt.",
   "in_checker": true
  },
  {
   "token": "Googlebot",
   "vendor": "Google",
   "product": "Google Search, including its AI features",
   "purpose": "search",
   "group": "web_search",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "vendor documentation, as recorded in the AI crawler check",
   "docs_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
   "ip_ranges_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
   "date_verified": "2026-10-14",
   "note": "Also the crawler for Google AI Overviews and AI Mode, which draw on the Google Search index.",
   "in_checker": true
  },
  {
   "token": "Google-Extended",
   "vendor": "Google",
   "product": "Google Search, including its AI features",
   "purpose": "training",
   "group": "ai_training",
   "kind": "control_token",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
   "ip_ranges_url": null,
   "date_verified": "2026-10-14",
   "note": "A robots.txt control token, not a separate crawler. Google says it does not affect Google Search.",
   "in_checker": true
  },
  {
   "token": "Google-CloudVertexBot",
   "vendor": "Google",
   "product": "Google Search, including its AI features",
   "purpose": "other",
   "group": "other",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
   "ip_ranges_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
   "date_verified": "2026-10-14",
   "note": "Crawls only for Vertex AI Agents that a site owner set up. Google says it does not affect Google Search.",
   "in_checker": true
  },
  {
   "token": "Bingbot",
   "vendor": "Microsoft",
   "product": "Bing search",
   "purpose": "search",
   "group": "web_search",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "vendor documentation, as recorded in the AI crawler check",
   "docs_url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
   "ip_ranges_url": "https://www.bing.com/toolbox/bingbot.json",
   "date_verified": "2026-10-14",
   "note": "Microsoft Copilot retrieves from the Bing search service.",
   "in_checker": true
  },
  {
   "token": "Meta-ExternalFetcher",
   "vendor": "Meta",
   "product": null,
   "purpose": "user-fetch",
   "group": "ai_user_fetch",
   "kind": "user_agent",
   "honours_robots": "may-not",
   "honours_robots_source": "vendor documentation page, read 2026-10-14",
   "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "ip_ranges_url": null,
   "date_verified": "2026-10-14",
   "note": "Meta says Meta-ExternalFetcher fetches links at a user's request, so it may bypass robots.txt.",
   "in_checker": false
  },
  {
   "token": "CCBot",
   "vendor": "Common Crawl",
   "product": null,
   "purpose": "training",
   "group": "ai_training",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://commoncrawl.org/ccbot",
   "ip_ranges_url": "https://index.commoncrawl.org/ccbot.json",
   "date_verified": "2026-10-14",
   "note": "Builds the open Common Crawl dataset, which is widely used for model training; classed as training for that reason.",
   "in_checker": false
  },
  {
   "token": "Applebot-Extended",
   "vendor": "Apple",
   "product": null,
   "purpose": "training",
   "group": "ai_training",
   "kind": "control_token",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://support.apple.com/en-us/119829",
   "ip_ranges_url": null,
   "date_verified": "2026-10-14",
   "note": "A control token: Apple documents it as governing use of Applebot-crawled data for training, not as a separate crawler.",
   "in_checker": false
  },
  {
   "token": "Meta-ExternalAgent",
   "vendor": "Meta",
   "product": null,
   "purpose": "training",
   "group": "ai_training",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "ip_ranges_url": null,
   "date_verified": "2026-10-14",
   "note": null,
   "in_checker": false
  },
  {
   "token": "Bytespider",
   "vendor": "ByteDance",
   "product": null,
   "purpose": "training",
   "group": "ai_training",
   "kind": "user_agent",
   "honours_robots": "no",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": null,
   "ip_ranges_url": null,
   "date_verified": null,
   "note": "ai.robots.txt lists it as LLM training and records that it does not respect robots.txt.",
   "in_checker": false
  },
  {
   "token": "Applebot",
   "vendor": "Apple",
   "product": null,
   "purpose": "search",
   "group": "web_search",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://support.apple.com/en-us/119829",
   "ip_ranges_url": "https://search.developer.apple.com/applebot.json",
   "date_verified": "2026-10-14",
   "note": null,
   "in_checker": false
  },
  {
   "token": "GoogleOther",
   "vendor": "Google",
   "product": null,
   "purpose": "other",
   "group": "other",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
   "ip_ranges_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
   "date_verified": "2026-10-14",
   "note": null,
   "in_checker": false
  },
  {
   "token": "Amazonbot",
   "vendor": "Amazon",
   "product": null,
   "purpose": "other",
   "group": "other",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": "https://developer.amazon.com/amazonbot",
   "ip_ranges_url": "https://developer.amazon.com/amazonbot/ip-addresses/",
   "date_verified": "2026-10-14",
   "note": "Mixed purpose: per Amazon, it improves Amazon products and services and may be used to train Amazon AI models. Amazon names separate agents for Alexa: Amzn-SearchBot for search experiences such as Alexa, and Amzn-User for Alexa queries that need up-to-date information.",
   "in_checker": false
  },
  {
   "token": "anthropic-ai",
   "vendor": "Anthropic",
   "product": null,
   "purpose": "other",
   "group": "other",
   "kind": "legacy_token",
   "honours_robots": "unclear",
   "honours_robots_source": "ai.robots.txt robots.json @9ad8a47e",
   "docs_url": null,
   "ip_ranges_url": null,
   "date_verified": null,
   "note": "Legacy token. Not present on Anthropic's current crawler page (fetched 2026-10-06); still common in robots.txt files as a training opt-out.",
   "in_checker": false
  },
  {
   "token": "MistralAI-User",
   "vendor": "Mistral",
   "product": "Le Chat",
   "purpose": "user-fetch",
   "group": "ai_user_fetch",
   "kind": "user_agent",
   "honours_robots": "unclear",
   "honours_robots_source": "vendor documentation page, read 2026-10-18",
   "docs_url": "https://docs.mistral.ai/robots/",
   "ip_ranges_url": "https://mistral.ai/mistralai-user-ips.json",
   "date_verified": "2026-10-18",
   "note": "Mistral says MistralAI-User is not used to crawl the web automatically or to collect training data; the page does not say whether it follows robots.txt.",
   "in_checker": false
  },
  {
   "token": "MistralAI-Index",
   "vendor": "Mistral",
   "product": "Le Chat",
   "purpose": "search",
   "group": "ai_search",
   "kind": "user_agent",
   "honours_robots": "unclear",
   "honours_robots_source": "vendor documentation page, read 2026-10-18",
   "docs_url": "https://docs.mistral.ai/robots/",
   "ip_ranges_url": "https://mistral.ai/mistralai-index-ips.json",
   "date_verified": "2026-10-18",
   "note": "Mistral says content crawled by MistralAI-Index is not used for training; the page does not say whether it follows robots.txt.",
   "in_checker": false
  },
  {
   "token": "MistralAI-Training",
   "vendor": "Mistral",
   "product": null,
   "purpose": "training",
   "group": "ai_training",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "vendor documentation page, read 2026-10-18",
   "docs_url": "https://docs.mistral.ai/robots/",
   "ip_ranges_url": null,
   "date_verified": "2026-10-18",
   "note": "Mistral says MistralAI-Training crawls web content to build datasets for training its models.",
   "in_checker": false
  },
  {
   "token": "DuckAssistBot",
   "vendor": "DuckDuckGo",
   "product": "Duck.ai / DuckAssist",
   "purpose": "user-fetch",
   "group": "ai_user_fetch",
   "kind": "user_agent",
   "honours_robots": "yes",
   "honours_robots_source": "vendor documentation page, read 2026-10-18",
   "docs_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
   "ip_ranges_url": "https://duckduckgo.com/duckassistbot.json",
   "date_verified": "2026-10-18",
   "note": "DuckDuckGo says the data is not used to train AI models and that disallowing DuckAssistBot in robots.txt opts a site out of its AI-assisted answers (after 72 hours).",
   "in_checker": false
  }
 ],
 "vendors_without_documented_agent": [
  {
   "vendor": "Brave",
   "product": "Brave Search Ask",
   "ua": "UNDOCUMENTED",
   "docs_url": "https://search.brave.com/help/brave-search-crawler",
   "quote": "does not advertise a differentiated user agent",
   "note": "Brave's crawler uses no distinct user agent by design and follows Googlebot's robots.txt rules: a page Googlebot cannot crawl, Brave does not crawl either.",
   "checked_on": "2026-10-18"
  },
  {
   "vendor": "xAI",
   "product": "Grok",
   "ua": "UNDOCUMENTED",
   "docs_url": null,
   "quote": null,
   "note": "Third-party directories list GrokBot / xAI-Grok / xAI-SearchBot strings from observed traffic; none is named on an xAI page.",
   "checked_on": "2026-10-18"
  },
  {
   "vendor": "DeepSeek",
   "product": "DeepSeek",
   "ua": "UNDOCUMENTED",
   "docs_url": null,
   "quote": null,
   "note": "Third-party directories list a DeepSeekBot string pointing to www.deepseek.com/bot, which returns 404.",
   "checked_on": "2026-10-18"
  },
  {
   "vendor": "Moonshot AI",
   "product": "Kimi",
   "ua": "UNDOCUMENTED",
   "docs_url": null,
   "quote": null,
   "note": "Third-party directories list KimiBot / Kimi-SearchBot / Kimi-User; kimi.moonshot.cn/bot redirects to a product page that names no crawler.",
   "checked_on": "2026-10-18"
  },
  {
   "vendor": "Alibaba",
   "product": "Qwen",
   "ua": "UNDOCUMENTED",
   "docs_url": null,
   "quote": null,
   "note": "Third-party directories list QwenBot / Qwen-User / TongyiBot as unverified; no Alibaba page found.",
   "checked_on": "2026-10-18"
  },
  {
   "vendor": "Zhipu AI",
   "product": "Z.ai (GLM)",
   "ua": "UNDOCUMENTED",
   "docs_url": null,
   "quote": null,
   "note": "Third-party directories list ChatGLM-Spider with disputed robots.txt behaviour; no Zhipu or Z.ai page found.",
   "checked_on": "2026-10-18"
  }
 ]
}