{"dataset":"AI crawler directory","url":"https://kuldeep.cc/data/ai-search/crawlers","creator":"Kuldeep Paul (kuldeep.cc)","license":"CC BY 4.0 — https://creativecommons.org/licenses/by/4.0/","updated":"2026-10-29","rows":[{"token":"GPTBot","operator":"OpenAI","purpose":"training","product":"OpenAI foundation models","respectsRobots":"Yes","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot","ipList":"https://openai.com/gptbot.json","docs":"https://developers.openai.com/api/docs/bots","source":"kuldeep.cc (official documentation)"},{"token":"OAI-SearchBot","operator":"OpenAI","purpose":"search","product":"ChatGPT search","respectsRobots":"Yes","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot","ipList":"https://openai.com/searchbot.json","docs":"https://developers.openai.com/api/docs/bots","source":"kuldeep.cc (official documentation)"},{"token":"ChatGPT-User","operator":"OpenAI","purpose":"user","product":"ChatGPT, Custom GPTs, GPT Actions","respectsRobots":"Partial","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot","ipList":"https://openai.com/chatgpt-user.json","docs":"https://developers.openai.com/api/docs/bots","source":"kuldeep.cc (official documentation)"},{"token":"OAI-AdsBot","operator":"OpenAI","purpose":"ads","product":"ChatGPT Ads","respectsRobots":"Not applicable","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot","ipList":"https://openai.com/adsbot.json","docs":"https://developers.openai.com/api/docs/bots","source":"kuldeep.cc (official documentation)"},{"token":"ChatGPT Agent","operator":"OpenAI","purpose":"agent","product":"ChatGPT agent mode","respectsRobots":"Unclear","userAgent":"","ipList":"","docs":"https://developers.openai.com/api/docs/bots","source":"kuldeep.cc (official documentation)"},{"token":"ClaudeBot","operator":"Anthropic","purpose":"training","product":"Anthropic model development","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","source":"kuldeep.cc (official documentation)"},{"token":"Claude-SearchBot","operator":"Anthropic","purpose":"search","product":"Claude web search","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","source":"kuldeep.cc (official documentation)"},{"token":"Claude-User","operator":"Anthropic","purpose":"user","product":"Claude","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","source":"kuldeep.cc (official documentation)"},{"token":"anthropic-ai","operator":"Anthropic","purpose":"training","product":"Legacy token","respectsRobots":"Unclear","userAgent":"","ipList":"","docs":"","source":"kuldeep.cc (official documentation)"},{"token":"Claude-Web","operator":"Anthropic","purpose":"user","product":"Legacy token","respectsRobots":"Unclear","userAgent":"","ipList":"","docs":"","source":"kuldeep.cc (official documentation)"},{"token":"PerplexityBot","operator":"Perplexity AI","purpose":"search","product":"Perplexity search index","respectsRobots":"Yes","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)","ipList":"https://www.perplexity.com/perplexitybot.json","docs":"https://docs.perplexity.ai/guides/bots","source":"kuldeep.cc (official documentation)"},{"token":"Perplexity-User","operator":"Perplexity AI","purpose":"user","product":"Perplexity, Comet","respectsRobots":"No","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)","ipList":"https://www.perplexity.com/perplexity-user.json","docs":"https://docs.perplexity.ai/guides/bots","source":"kuldeep.cc (official documentation)"},{"token":"Google-Extended","operator":"Google","purpose":"training","product":"Gemini training and grounding","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","source":"kuldeep.cc (official documentation)"},{"token":"GoogleOther","operator":"Google","purpose":"mixed","product":"Google internal research and development","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","source":"kuldeep.cc (official documentation)"},{"token":"Google-CloudVertexBot","operator":"Google","purpose":"agent","product":"Vertex AI Agents","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","source":"kuldeep.cc (official documentation)"},{"token":"Google-Agent","operator":"Google","purpose":"agent","product":"Google AI agents (e.g. Project Mariner)","respectsRobots":"Partial","userAgent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent)","ipList":"","docs":"https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers","source":"kuldeep.cc (official documentation)"},{"token":"Googlebot","operator":"Google","purpose":"search","product":"Google Search, AI Overviews, AI Mode","respectsRobots":"Yes","userAgent":"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","ipList":"https://developers.google.com/static/search/apis/ipranges/googlebot.json","docs":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","source":"kuldeep.cc (official documentation)"},{"token":"Applebot","operator":"Apple","purpose":"search","product":"Siri, Spotlight, Safari suggestions","respectsRobots":"Yes","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)","ipList":"https://search.developer.apple.com/applebot.json","docs":"https://support.apple.com/en-us/119829","source":"kuldeep.cc (official documentation)"},{"token":"Applebot-Extended","operator":"Apple","purpose":"training","product":"Apple Intelligence and foundation models","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://support.apple.com/en-us/119829","source":"kuldeep.cc (official documentation)"},{"token":"Meta-ExternalAgent","operator":"Meta","purpose":"training","product":"Meta AI models and products","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","source":"kuldeep.cc (official documentation)"},{"token":"Meta-ExternalFetcher","operator":"Meta","purpose":"user","product":"Meta AI","respectsRobots":"No","userAgent":"","ipList":"","docs":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","source":"kuldeep.cc (official documentation)"},{"token":"meta-webindexer","operator":"Meta","purpose":"search","product":"Meta AI search","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","source":"kuldeep.cc (official documentation)"},{"token":"FacebookBot","operator":"Meta","purpose":"training","product":"Meta speech and language models","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","source":"kuldeep.cc (official documentation)"},{"token":"Amazonbot","operator":"Amazon","purpose":"mixed","product":"Alexa answers and Amazon AI services","respectsRobots":"Yes","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1) Chrome/W.X.Y.Z Safari/537.36","ipList":"https://developer.amazon.com/amazonbot/ip-addresses/","docs":"https://developer.amazon.com/amazonbot","source":"kuldeep.cc (official documentation)"},{"token":"Amzn-SearchBot","operator":"Amazon","purpose":"search","product":"Alexa and Amazon search experiences","respectsRobots":"Yes","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/0.1) Chrome/W.X.Y.Z Safari/537.36","ipList":"https://developer.amazon.com/amazonbot/searchbot-ip-addresses/","docs":"https://developer.amazon.com/amazonbot","source":"kuldeep.cc (official documentation)"},{"token":"Amzn-User","operator":"Amazon","purpose":"user","product":"Alexa and Amazon AI assistants","respectsRobots":"Partial","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/W.X.Y.Z Safari/537.36","ipList":"https://developer.amazon.com/amazonbot/live-ip-addresses/","docs":"https://developer.amazon.com/amazonbot","source":"kuldeep.cc (official documentation)"},{"token":"CCBot","operator":"Common Crawl","purpose":"training","product":"Common Crawl open dataset","respectsRobots":"Yes","userAgent":"CCBot/2.0 (https://commoncrawl.org/faq/)","ipList":"https://index.commoncrawl.org/ccbot.json","docs":"https://commoncrawl.org/ccbot","source":"kuldeep.cc (official documentation)"},{"token":"Bytespider","operator":"ByteDance","purpose":"training","product":"ByteDance LLMs (Doubao)","respectsRobots":"No","userAgent":"","ipList":"","docs":"","source":"kuldeep.cc (official documentation)"},{"token":"Bingbot","operator":"Microsoft","purpose":"search","product":"Bing, Microsoft Copilot","respectsRobots":"Yes","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/W.X.Y.Z Safari/537.36","ipList":"https://www.bing.com/toolbox/bingbot.json","docs":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","source":"kuldeep.cc (official documentation)"},{"token":"MistralAI-User","operator":"Mistral AI","purpose":"user","product":"Le Chat","respectsRobots":"Yes","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)","ipList":"","docs":"https://docs.mistral.ai/robots","source":"kuldeep.cc (official documentation)"},{"token":"MistralAI-Index","operator":"Mistral AI","purpose":"search","product":"Le Chat search","respectsRobots":"Yes","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)","ipList":"","docs":"https://docs.mistral.ai/robots","source":"kuldeep.cc (official documentation)"},{"token":"MistralAI-Training","operator":"Mistral AI","purpose":"training","product":"Mistral models","respectsRobots":"Yes","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Training/1.0; +https://docs.mistral.ai/robots)","ipList":"","docs":"https://docs.mistral.ai/robots","source":"kuldeep.cc (official documentation)"},{"token":"DuckAssistBot","operator":"DuckDuckGo","purpose":"search","product":"DuckDuckGo AI-assisted answers","respectsRobots":"Yes","userAgent":"DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)","ipList":"https://duckduckgo.com/duckassistbot.json","docs":"https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot","source":"kuldeep.cc (official documentation)"},{"token":"YouBot","operator":"You.com","purpose":"search","product":"You.com search and APIs","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://about.you.com/youbot/","source":"kuldeep.cc (official documentation)"},{"token":"cohere-ai","operator":"Cohere","purpose":"user","product":"Cohere products","respectsRobots":"Unclear","userAgent":"","ipList":"","docs":"","source":"kuldeep.cc (official documentation)"},{"token":"cohere-training-data-crawler","operator":"Cohere","purpose":"training","product":"Cohere LLMs","respectsRobots":"Unclear","userAgent":"","ipList":"","docs":"","source":"kuldeep.cc (official documentation)"},{"token":"DeepSeekBot","operator":"DeepSeek","purpose":"mixed","product":"DeepSeek models and search","respectsRobots":"No","userAgent":"Mozilla/5.0 (compatible; DeepSeekBot/1.0; +https://www.deepseek.com/)","ipList":"","docs":"","source":"kuldeep.cc (official documentation)"},{"token":"GrokBot","operator":"xAI","purpose":"search","product":"Grok","respectsRobots":"Unclear","userAgent":"","ipList":"","docs":"","source":"kuldeep.cc (official documentation)"},{"token":"KimiBot","operator":"Moonshot AI","purpose":"training","product":"Kimi models","respectsRobots":"Yes","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; KimiBot/1.0; +https://www.kimi.com/policies/kimi-crawlers","ipList":"","docs":"https://www.kimi.ai/policies/kimi-crawlers","source":"kuldeep.cc (official documentation)"},{"token":"Kimi-SearchBot","operator":"Moonshot AI","purpose":"search","product":"Kimi search","respectsRobots":"Yes","userAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Kimi-SearchBot/1.0; +https://www.kimi.com/policies/kimi-crawlers","ipList":"","docs":"https://www.kimi.ai/policies/kimi-crawlers","source":"kuldeep.cc (official documentation)"},{"token":"AI2Bot","operator":"Ai2","purpose":"training","product":"Open language models (OLMo)","respectsRobots":"Yes","userAgent":"Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)","ipList":"","docs":"https://allenai.org/crawler","source":"kuldeep.cc (official documentation)"},{"token":"Bravebot","operator":"Brave","purpose":"search","product":"Brave Search index","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"https://search.brave.com/help/brave-search-crawler","source":"kuldeep.cc (official documentation)"},{"token":"PetalBot","operator":"Huawei","purpose":"search","product":"Petal Search and Huawei assistant","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"kuldeep.cc (official documentation)"},{"token":"Diffbot","operator":"Diffbot","purpose":"mixed","product":"Knowledge graph and AI data","respectsRobots":"Partial","userAgent":"","ipList":"","docs":"https://www.diffbot.com/","source":"kuldeep.cc (official documentation)"},{"token":"Timpibot","operator":"Timpi","purpose":"training","product":"Timpi search and LLM data","respectsRobots":"Unclear","userAgent":"","ipList":"","docs":"","source":"kuldeep.cc (official documentation)"},{"token":"AddSearchBot","operator":"","purpose":"AI Search Crawlers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"AgentDataBot","operator":"","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"AgentTimes","operator":"The Agent Times","purpose":"Data Scraper from RSS Feeds.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"AGI Agent","operator":"AGI, Inc","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"AI2Bot-DeepResearchEval","operator":"Ai2, a non-profit AI research institute","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Ai2Bot-Dolma","operator":"Ai2","purpose":"Content is used to train open language models.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"aiHitBot","operator":"aiHit","purpose":"A massive, artificial intelligence/machine learning, automated system.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"AIWebIndex","operator":"Lyrenth","purpose":"AI Search Crawlers","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"amazon-kendra","operator":"Amazon","purpose":"Collects data for AI natural language search","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"amazon-QBusiness","operator":"","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"AmazonBuyForMe","operator":"Amazon","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Andibot","operator":"Andi","purpose":"Search engine using generative AI, AI Search Assistant","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Anomura","operator":"Direqt","purpose":"Collects data for AI search","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ApifyBot","operator":"","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ApifyWebsiteContentCrawler","operator":"","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Aranet-SearchBot","operator":"","purpose":"Undocumented AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"atlassian-bot","operator":"Atlassian","purpose":"AI search, assistants and agents","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Awario","operator":"Awario","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"AzureAI-SearchBot","operator":"","purpose":"AI Search Crawlers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"bedrockbot","operator":"Amazon","purpose":"Data scraping for custom AI applications.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"bigsur.ai","operator":"Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for busi…","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"BixelBot","operator":"","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Brightbot","operator":"","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Brightbot 1.0","operator":"https://brightdata.com/brightbot","purpose":"LLM/AI training.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"BuddyBot","operator":"BuddyBotLearning","purpose":"AI Learning Companion","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"CartAI Agentic Commerce","operator":"CartAI that accesses online stores to support user-directed shopping and checkout workflows","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Channel3Bot","operator":"","purpose":"AI Search Crawlers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ChatGLM-Spider","operator":"","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Claude-Code","operator":"","purpose":"AI Coding Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"CledaraBot","operator":"Cledara that visits SaaS providers to complete customer-approved administrative tasks, including invoice collection and user managem…","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Cloudflare-AutoRAG","operator":"Cloudflare","purpose":"Collects data for AI search","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"CloudflareBrowserRenderingCrawler","operator":"Cloudflare that returns rendered website content for research, monitoring, and AI data workflows","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"CloudVertexBot","operator":"","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Code","operator":"","purpose":"AI Coding Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Codex","operator":"","purpose":"AI Coding Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Cotoyogi","operator":"ROIS","purpose":"AI LLM Scraper.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"CragCrawler","operator":"CragSoftware, a Brazil-based software company specializing in data engineering and AI web scraping services","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Crawl4AI","operator":"","purpose":"Undocumented AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Crawlspace","operator":"Crawlspace","purpose":"AI Data Providers","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Cursor","operator":"","purpose":"AI Coding Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Datenbank Crawler","operator":"Datenbank","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Devin","operator":"Devin AI","purpose":"AI Coding Agents","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Diffbot-User","operator":"Diffbot","purpose":"AI Assistants","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Dot","operator":"","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"DoubaoBot","operator":"ByteDance","purpose":"AI crawler for ByteDance's Doubao AI assistant.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Echobot Bot","operator":"Echobox","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"EchoboxBot","operator":"Echobox","purpose":"Data collection to support AI-powered products.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ElevenlabsBot","operator":"","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ERNIEBot","operator":"Baidu","purpose":"Collects public web content for Baidu's ERNIE large language models.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ExaBot","operator":"","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ExaSearchBot","operator":"Exa","purpose":"AI Search Crawlers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ExaUser","operator":"","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"facebookexternalhit","operator":"Meta/Facebook","purpose":"Ostensibly only for sharing, but likely used as an AI crawler as well","product":"","respectsRobots":"No","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Factset_spyderbot","operator":"Factset","purpose":"AI model training.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"FirecrawlAgent","operator":"Firecrawl that extracts web content and converts it into structured data for use in LLM and AI applications","purpose":"AI Data Providers","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"FriendlyCrawler","operator":"Unknown","purpose":"We are using the data from the crawler to build datasets for machine learning experiments.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"GeistHaus-PageFetcher","operator":"GeistHaus, a company developing AI systems for therapy and psychological assessment","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Gemini-Deep-Research","operator":"","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Google-Firebase","operator":"Google","purpose":"Used as part of AI apps developed by users of Google's Firebase AI products.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Google-Gemini-CLI","operator":"","purpose":"AI Coding Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Google-NotebookLM","operator":"","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"GoogleAgent-Mariner","operator":"Google","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"GoogleAgent-URLContext","operator":"Google that retrieves web content on behalf of Gemini API users","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"GoogleOther-Image","operator":"Google","purpose":"Scrapes data.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"GoogleOther-Video","operator":"Google","purpose":"Scrapes data.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Grok Bot","operator":"","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"GrokAppAndroid","operator":"xAI","purpose":"AI Search Crawlers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"HarkBot","operator":"","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"HenkBot","operator":"","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Hunyuan","operator":"Tencent","purpose":"Hunyuan is a training crawler operated by Tencent.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"iAskBot","operator":"","purpose":"Undocumented AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"iaskspider","operator":"","purpose":"Undocumented AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ICC-Crawler","operator":"NICT","purpose":"Scrapes data to train and support AI technologies.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ImagesiftBot","operator":"ImageSift","purpose":"ImageSiftBot is a web crawler that scrapes the internet for publicly available images to support their suite of web intelligence products","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"imageSpider","operator":"","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"img2dataset","operator":"img2dataset","purpose":"Scrapes images for use in LLMs.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Instinct","operator":"","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ISSCyberRiskCrawler","operator":"ISS-Corporate","purpose":"Scrapes data to train machine learning models.","product":"","respectsRobots":"No","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"kagi-fetcher","operator":"Kagi that fetches web content to answer user queries through Kagi AI, their suite of AI-powered tools including Assistant, Res…","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Kangaroo Bot","operator":"","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"KeenableBot","operator":"Keenable","purpose":"AI Search Crawlers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Kimi-Agent","operator":"Moonshot AI that browses websites while carrying out user-directed research and other tasks","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Kimi-User","operator":"Moonshot AI that fetches web content on behalf of users interacting with Kimi","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"KlaviyoAIBot","operator":"Klaviyo","purpose":"AI Assistants","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"KunatoCrawler","operator":"","purpose":"Undocumented AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"laion-huggingface-processor","operator":"","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"LAIONDownloader","operator":"Large-scale Artificial Intelligence Open Network","purpose":"AI tools and models for machine learning research.","product":"","respectsRobots":"No","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"LCC","operator":"","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Lightpanda","operator":"Anyone who downloads the Lightpanda client. Possibly being used by a Grok-adjacent organization's botnet.","purpose":"AI Data Scrapers","product":"","respectsRobots":"At the discretion of Lightpanda users.","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"LinerBot","operator":"","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Linguee Bot","operator":"Linguee","purpose":"AI powered translation service","product":"","respectsRobots":"No","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"LinkupBot","operator":"","purpose":"AI Search Crawlers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Manus-User","operator":"Butterfly Effect, a company based in China","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"MoonshotBot","operator":"Kimi","purpose":"MoonshotBot is a training crawler.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Mozilla-Tabstack","operator":"Mozilla that performs programmatic, AI-driven interactions with web content through Tabstack","purpose":"AI Data Providers","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Muse","operator":"","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"MyCentralAIScraperBot","operator":"","purpose":"AI data scraper","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"NagetBot","operator":"Naget Inc (founded by Chris Samarinas, headquarter in Amherst, Massachusetts)","purpose":"AI data scraper","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"nbot","operator":"NBot","purpose":"AI Search Crawlers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"netEstate Imprint Crawler","operator":"netEstate","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"newsai","operator":"","purpose":"AI data scraper","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"NotebookLM","operator":"","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"NovaAct","operator":"","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"omgili","operator":"Webz.io","purpose":"Data is sold.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"omgilibot","operator":"Webz.io","purpose":"Data is sold.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"OpenAI","operator":"OpenAI","purpose":"","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"opencode","operator":"","purpose":"AI Coding Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Operator","operator":"","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"PanguBot","operator":"the Chinese company Huawei","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Panscient","operator":"Panscient","purpose":"Data collection and analysis using machine learning and AI.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"panscient.com","operator":"Panscient","purpose":"Data collection and analysis using machine learning and AI.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Payhawk Invoice Fetching Agent","operator":"Payhawk that retrieves invoices from supplier websites for users and attaches them to expense records","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"PhindBot","operator":"phind","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Poggio-Citations","operator":"Poggio, a company that provides AI sales enablement tools for creating tailored narratives, business cases, and account plan…","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Poseidon Research Crawler","operator":"Poseidon Research","purpose":"AI research crawler","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"qodercli","operator":"Qoder that helps developers build, debug, and modify software from the terminal","purpose":"AI Coding Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"QualifiedBot","operator":"Qualified","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Querit-SearchBot","operator":"Querit that indexes web content for their search API service, which is designed to provide real-time search results for larg…","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"QueritBot","operator":"Querit, a company providing a search API for large language model integration","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"QuillBot","operator":"Quillbot","purpose":"Company offers AI detection, writing tools and other services.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"quillbot.com","operator":"Quillbot","purpose":"Company offers AI detection, writing tools and other services.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"QwenBot","operator":"Alibaba","purpose":"Collects public web content for Alibaba's Qwen (Tongyi Qianwen) large language models.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Reflectionbot","operator":"Reflection","purpose":"Undocumented AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"SBIntuitionsBot","operator":"SB Intuitions","purpose":"Uses data gathered in AI development and information analysis.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Scrapy","operator":"Zyte","purpose":"Scrapes data for a variety of uses including training AI.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"SemrushBot-OCOB","operator":"Semrush","purpose":"Crawls your site for ContentShake AI tool.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"SemrushBot-SWA","operator":"Semrush","purpose":"Checks URLs on your site for SEO Writing Assistant.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Shap-User","operator":"","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ShapBot","operator":"Parallel","purpose":"AI Data Providers","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Sidetrade indexer bot","operator":"Sidetrade","purpose":"Extracts data for a variety of uses including training AI.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Spider","operator":"","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"TavilyBot","operator":"","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Terra Cotta","operator":"","purpose":"AI Data Providers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"TerraCotta","operator":"Ceramic AI","purpose":"AI Data Providers","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Thinkbot","operator":"Thinkbot","purpose":"Insights on AI integration and automation.","product":"","respectsRobots":"No","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"TikTokSpider","operator":"ByteDance","purpose":"LLM training.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"TongyiBot","operator":"Alibaba that fetches web content for the Tongyi Qianwen assistant and related Qwen-generated answers","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Trae","operator":"","purpose":"AI Coding Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"TwinAgent","operator":"Twin, a platform that creates automated workers to perform tasks by integrating with APIs and controlling web applications through browser automa…","purpose":"AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"UseAI","operator":"","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"VelenPublicWebCrawler","operator":"Velen Crawler","purpose":"Scrapes data for business data sets and machine learning models.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"WARDBot","operator":"WEBSPARK","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"Webzio-Extended","operator":"","purpose":"AI Data Scrapers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"wpbot","operator":"QuantumCloud","purpose":"Live chat support and lead generation.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"WRTNBot","operator":"","purpose":"Undocumented AI Agents","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"xAI-Grok","operator":"xAI","purpose":"xAI-Grok is an xAI crawler token associated with Grok content collection.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"YaK","operator":"Meltwater","purpose":"According to the Meltwater Consumer Intelligence page 'By applying AI, data science, and market research expertise to a live feed of global data sources, we transform unstructured data into actionable insights allowing better decision-making'.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"YandexAdditional","operator":"Yandex","purpose":"Scrapes/analyzes data for the YandexGPT LLM.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"YandexAdditionalBot","operator":"Yandex","purpose":"Scrapes/analyzes data for the YandexGPT LLM.","product":"","respectsRobots":"Yes","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"YiBot","operator":"01.ai, an AI company founded by Kai-Fu Lee.","purpose":"This bot crawls the web to gather data and content for indexation and AI model training.","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"YiyanBot","operator":"Baidu that fetches web content for the yiyan","purpose":"AI Assistants","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"},{"token":"ZanistaBot","operator":"","purpose":"AI Search Crawlers","product":"","respectsRobots":"","userAgent":"","ipList":"","docs":"","source":"ai.robots.txt (MIT)"}]}