{"dataset":"Who blocks AI crawlers? robots.txt study","url":"https://kuldeep.cc/data/ai-search/robots-txt-study","creator":"Kuldeep Paul (kuldeep.cc)","license":"CC BY 4.0 — https://creativecommons.org/licenses/by/4.0/","updated":"2026-10-29","rows":[{"rank":1,"domain":"google.com","blocks":"","allows":"","llmsTxt":false},{"rank":2,"domain":"cloudflare.com","blocks":"","allows":"GPTBot ChatGPT-User anthropic-ai Claude-Web PerplexityBot Google-Extended CCBot cohere-ai","llmsTxt":true},{"rank":3,"domain":"amazonaws.com","blocks":"","allows":"","llmsTxt":false},{"rank":4,"domain":"facebook.com","blocks":"GPTBot ClaudeBot PerplexityBot Google-Extended Applebot-Extended Amazonbot Brightbot Scrapy YaK","allows":"","llmsTxt":false},{"rank":6,"domain":"gstatic.com","blocks":"","allows":"","llmsTxt":false},{"rank":7,"domain":"fbcdn.net","blocks":"GPTBot ClaudeBot PerplexityBot Google-Extended Applebot-Extended Amazonbot Brightbot Scrapy YaK","allows":"","llmsTxt":false},{"rank":10,"domain":"microsoft.com","blocks":"","allows":"","llmsTxt":false},{"rank":11,"domain":"youtube.com","blocks":"","allows":"","llmsTxt":false},{"rank":12,"domain":"apple.com","blocks":"","allows":"","llmsTxt":false},{"rank":14,"domain":"instagram.com","blocks":"GPTBot ClaudeBot PerplexityBot Google-Extended Applebot-Extended Amazonbot Brightbot Scrapy YaK","allows":"","llmsTxt":false},{"rank":15,"domain":"twitter.com","blocks":"","allows":"Google-Extended Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer FacebookBot","llmsTxt":false},{"rank":17,"domain":"mail.ru","blocks":"","allows":"","llmsTxt":false},{"rank":22,"domain":"googlevideo.com","blocks":"","allows":"","llmsTxt":false},{"rank":25,"domain":"dzen.ru","blocks":"","allows":"","llmsTxt":false},{"rank":30,"domain":"amazon.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":33,"domain":"github.com","blocks":"Bytespider","allows":"GPTBot OAI-SearchBot ClaudeBot anthropic-ai PerplexityBot","llmsTxt":true},{"rank":36,"domain":"wikipedia.org","blocks":"","allows":"","llmsTxt":false},{"rank":39,"domain":"fastly.net","blocks":"","allows":"","llmsTxt":true},{"rank":42,"domain":"whatsapp.net","blocks":"GPTBot ClaudeBot PerplexityBot Google-Extended Applebot-Extended Amazonbot Brightbot Scrapy YaK","allows":"","llmsTxt":false},{"rank":43,"domain":"netflix.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User PerplexityBot Google-Extended","llmsTxt":false},{"rank":45,"domain":"bing.com","blocks":"","allows":"","llmsTxt":false},{"rank":47,"domain":"x.com","blocks":"","allows":"Google-Extended Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer FacebookBot","llmsTxt":false},{"rank":48,"domain":"youtu.be","blocks":"","allows":"","llmsTxt":false},{"rank":49,"domain":"gandi.net","blocks":"","allows":"","llmsTxt":false},{"rank":50,"domain":"doubleclick.net","blocks":"","allows":"","llmsTxt":false},{"rank":52,"domain":"pinterest.com","blocks":"YandexAdditional YandexAdditionalBot","allows":"FacebookBot DuckAssistBot Bravebot","llmsTxt":false},{"rank":54,"domain":"icloud.com","blocks":"","allows":"","llmsTxt":false},{"rank":58,"domain":"cloudflare.net","blocks":"","allows":"","llmsTxt":false},{"rank":59,"domain":"wordpress.org","blocks":"","allows":"GPTBot ClaudeBot anthropic-ai PerplexityBot Google-Extended Applebot-Extended CCBot Bytespider","llmsTxt":true},{"rank":61,"domain":"adobe.com","blocks":"","allows":"","llmsTxt":true},{"rank":66,"domain":"whatsapp.com","blocks":"GPTBot ClaudeBot PerplexityBot Google-Extended Applebot-Extended Amazonbot Brightbot Scrapy YaK","allows":"","llmsTxt":false},{"rank":67,"domain":"zoom.us","blocks":"","allows":"","llmsTxt":false},{"rank":68,"domain":"myfritz.net","blocks":"","allows":"","llmsTxt":false},{"rank":70,"domain":"digicert.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User PerplexityBot Perplexity-User Google-Extended","llmsTxt":true},{"rank":72,"domain":"yahoo.com","blocks":"GPTBot ChatGPT-User ClaudeBot anthropic-ai Claude-Web PerplexityBot Google-Extended FacebookBot CCBot Bytespider YouBot cohere-ai Diffbot FriendlyCrawler ImagesiftBot img2dataset omgili omgilibot panscient.com Scrapy","allows":"OAI-SearchBot Claude-SearchBot","llmsTxt":false},{"rank":73,"domain":"wa.me","blocks":"","allows":"","llmsTxt":false},{"rank":76,"domain":"opera.com","blocks":"","allows":"","llmsTxt":true},{"rank":77,"domain":"roblox.com","blocks":"","allows":"","llmsTxt":false},{"rank":78,"domain":"spotify.com","blocks":"","allows":"","llmsTxt":false},{"rank":83,"domain":"ntp.org","blocks":"","allows":"","llmsTxt":false},{"rank":85,"domain":"amazonvideo.com","blocks":"ChatGPT-User Claude-User PerplexityBot Perplexity-User GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":87,"domain":"vimeo.com","blocks":"FacebookBot CCBot Bytespider Diffbot ImagesiftBot omgili omgilibot","allows":"GPTBot ClaudeBot anthropic-ai Claude-Web Google-Extended","llmsTxt":false},{"rank":88,"domain":"nginx.org","blocks":"","allows":"","llmsTxt":false},{"rank":89,"domain":"qq.com","blocks":"","allows":"","llmsTxt":false},{"rank":92,"domain":"msn.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot AI2Bot Diffbot Timpibot Ai2Bot-Dolma bedrockbot Cotoyogi FriendlyCrawler GoogleOther-Image GoogleOther-Video iaskspider ICC-Crawler ImagesiftBot img2dataset ISSCyberRiskCrawler Kangaroo Bot KunatoCrawler omgili omgilibot PanguBot QualifiedBot Scrapy Sidetrade indexer bot TikTokSpider VelenPublicWebCrawler WARDBot Webzio-Extended WRTNBot","allows":"","llmsTxt":false},{"rank":94,"domain":"blogspot.com","blocks":"","allows":"","llmsTxt":false},{"rank":95,"domain":"mozilla.org","blocks":"","allows":"","llmsTxt":false},{"rank":96,"domain":"f5.com","blocks":"","allows":"","llmsTxt":false},{"rank":102,"domain":"reddit.com","blocks":"","allows":"","llmsTxt":false},{"rank":103,"domain":"sentry.io","blocks":"","allows":"","llmsTxt":true},{"rank":107,"domain":"yandex.ru","blocks":"","allows":"OAI-SearchBot","llmsTxt":false},{"rank":108,"domain":"ui.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-Web PerplexityBot Google-Extended Applebot-Extended CCBot","llmsTxt":false},{"rank":109,"domain":"europa.eu","blocks":"","allows":"","llmsTxt":false},{"rank":112,"domain":"chatgpt.com","blocks":"anthropic-ai Claude-Web PerplexityBot Google-Extended FacebookBot CCBot Bytespider img2dataset omgili omgilibot","allows":"","llmsTxt":false},{"rank":116,"domain":"ipify.org","blocks":"","allows":"","llmsTxt":false},{"rank":117,"domain":"bit.ly","blocks":"","allows":"","llmsTxt":false},{"rank":118,"domain":"intuit.com","blocks":"","allows":"","llmsTxt":true},{"rank":120,"domain":"samsung.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User PerplexityBot Google-Extended","llmsTxt":true},{"rank":122,"domain":"skybridge.click","blocks":"","allows":"","llmsTxt":false},{"rank":127,"domain":"registrar-servers.com","blocks":"","allows":"","llmsTxt":false},{"rank":129,"domain":"t-online.de","blocks":"GPTBot ClaudeBot PerplexityBot Applebot-Extended Meta-ExternalAgent CCBot Bytespider","allows":"Google-Extended","llmsTxt":false},{"rank":131,"domain":"openai.com","blocks":"","allows":"","llmsTxt":false},{"rank":135,"domain":"android.com","blocks":"","allows":"GPTBot OAI-SearchBot ClaudeBot Claude-SearchBot anthropic-ai PerplexityBot Google-Extended Applebot-Extended CCBot MistralAI-User YouBot cohere-ai Diffbot","llmsTxt":false},{"rank":136,"domain":"dropbox.com","blocks":"","allows":"","llmsTxt":true},{"rank":138,"domain":"adobe.io","blocks":"","allows":"","llmsTxt":false},{"rank":140,"domain":"discord.gg","blocks":"","allows":"","llmsTxt":false},{"rank":143,"domain":"apache.org","blocks":"","allows":"","llmsTxt":false},{"rank":146,"domain":"vk.ru","blocks":"","allows":"","llmsTxt":false},{"rank":149,"domain":"webex.com","blocks":"","allows":"","llmsTxt":true},{"rank":150,"domain":"archive.org","blocks":"","allows":"","llmsTxt":false},{"rank":157,"domain":"snapchat.com","blocks":"GPTBot ClaudeBot anthropic-ai Claude-Web Google-Extended GoogleOther Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider YouBot cohere-ai cohere-training-data-crawler AI2Bot Diffbot Timpibot Ai2Bot-Dolma Brightbot 1.0 Crawlspace FriendlyCrawler GoogleOther-Image GoogleOther-Video iaskspider ICC-Crawler ImagesiftBot img2dataset ISSCyberRiskCrawler Kangaroo Bot omgili omgilibot PanguBot Scrapy SemrushBot-OCOB SemrushBot-SWA Sidetrade indexer bot VelenPublicWebCrawler Webzio-Extended","allows":"OAI-SearchBot ChatGPT-User Claude-SearchBot Claude-User PerplexityBot Perplexity-User meta-webindexer DuckAssistBot","llmsTxt":false},{"rank":160,"domain":"tumblr.com","blocks":"ClaudeBot anthropic-ai Google-Extended Applebot-Extended Meta-ExternalAgent FacebookBot Amazonbot CCBot YouBot ImagesiftBot omgili omgilibot","allows":"","llmsTxt":false},{"rank":163,"domain":"nytimes.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer FacebookBot CCBot Bytespider DuckAssistBot YouBot cohere-ai Diffbot Timpibot Brightbot Diffbot-User EchoboxBot ExaBot ExaSearchBot FirecrawlAgent FriendlyCrawler ImagesiftBot MyCentralAIScraperBot omgili omgilibot Poseidon Research Crawler quillbot.com Scrapy ShapBot","allows":"Amazonbot","llmsTxt":false},{"rank":165,"domain":"gravatar.com","blocks":"GPTBot Google-Extended FacebookBot Amazonbot CCBot Bytespider omgili omgilibot","allows":"","llmsTxt":true},{"rank":166,"domain":"kaspersky.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended Applebot-Extended CCBot MistralAI-User DuckAssistBot YouBot DeepSeekBot Diffbot","llmsTxt":true},{"rank":168,"domain":"ubuntu.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Applebot-Extended Meta-ExternalAgent Bytespider cohere-ai","llmsTxt":true},{"rank":170,"domain":"shopify.com","blocks":"","allows":"","llmsTxt":true},{"rank":173,"domain":"google-analytics.com","blocks":"","allows":"","llmsTxt":false},{"rank":174,"domain":"paypal.com","blocks":"","allows":"","llmsTxt":true},{"rank":176,"domain":"afternic.com","blocks":"","allows":"","llmsTxt":false},{"rank":178,"domain":"samsungcloud.com","blocks":"","allows":"","llmsTxt":false},{"rank":180,"domain":"flickr.com","blocks":"","allows":"OAI-SearchBot ChatGPT-User Claude-SearchBot Claude-User PerplexityBot","llmsTxt":false},{"rank":182,"domain":"w3.org","blocks":"","allows":"","llmsTxt":false},{"rank":184,"domain":"pki.goog","blocks":"","allows":"","llmsTxt":false},{"rank":185,"domain":"mts.ru","blocks":"","allows":"","llmsTxt":false},{"rank":186,"domain":"unity3d.com","blocks":"Bytespider","allows":"","llmsTxt":true},{"rank":187,"domain":"creativecommons.org","blocks":"","allows":"","llmsTxt":false},{"rank":189,"domain":"discord.com","blocks":"","allows":"","llmsTxt":false},{"rank":191,"domain":"ebay.com","blocks":"GPTBot OAI-AdsBot ClaudeBot anthropic-ai PerplexityBot Applebot-Extended Meta-ExternalAgent Amazonbot CCBot Bytespider ChatGLM-Spider","allows":"OAI-SearchBot ChatGPT-User Claude-SearchBot Claude-User","llmsTxt":false},{"rank":192,"domain":"soundcloud.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Google-Extended Applebot-Extended Meta-ExternalAgent Amazonbot CCBot Bytespider cohere-ai","llmsTxt":false},{"rank":193,"domain":"macromedia.com","blocks":"","allows":"","llmsTxt":true},{"rank":195,"domain":"cnn.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User GoogleOther Applebot-Extended FacebookBot Amazonbot Amzn-SearchBot Amzn-User CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler AI2Bot Diffbot Timpibot Ai2Bot-Dolma Brightbot 1.0 Crawlspace Diffbot-User EchoboxBot ExaBot ExaSearchBot ExaUser FirecrawlAgent FriendlyCrawler GoogleOther-Image GoogleOther-Video iaskspider ICC-Crawler ImagesiftBot img2dataset ISSCyberRiskCrawler Kangaroo Bot MyCentralAIScraperBot omgili omgilibot PanguBot Panscient Poseidon Research Crawler QuillBot quillbot.com SBIntuitionsBot Scrapy SemrushBot-OCOB SemrushBot-SWA Shap-User ShapBot Sidetrade indexer bot TavilyBot VelenPublicWebCrawler Webzio-Extended YandexAdditional YandexAdditionalBot","allows":"","llmsTxt":false},{"rank":196,"domain":"theguardian.com","blocks":"ClaudeBot Claude-SearchBot Claude-User anthropic-ai PerplexityBot Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent FacebookBot Amazonbot Amzn-SearchBot CCBot Bytespider DuckAssistBot YouBot amazon-QBusiness YandexAdditional YandexAdditionalBot","allows":"","llmsTxt":false},{"rank":197,"domain":"youtube-nocookie.com","blocks":"","allows":"","llmsTxt":false},{"rank":198,"domain":"medium.com","blocks":"GPTBot ClaudeBot GoogleOther Applebot-Extended Meta-ExternalAgent FacebookBot Amazonbot Bytespider","allows":"","llmsTxt":false},{"rank":203,"domain":"stripe.com","blocks":"","allows":"","llmsTxt":true},{"rank":216,"domain":"bbc.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Amazonbot Amzn-SearchBot Amzn-User CCBot Bytespider YouBot cohere-ai Diffbot ApifyBot ApifyWebsiteContentCrawler Brightbot CloudflareBrowserRenderingCrawler Diffbot-User ExaBot FirecrawlAgent omgili omgilibot Scrapy ShapBot TavilyBot YandexAdditional YandexAdditionalBot","allows":"","llmsTxt":false},{"rank":217,"domain":"criteo.com","blocks":"","allows":"","llmsTxt":false},{"rank":220,"domain":"forter.com","blocks":"Meta-ExternalAgent FacebookBot CCBot Bytespider Diffbot ImagesiftBot omgili omgilibot","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai PerplexityBot Perplexity-User Google-Extended Applebot-Extended Amazonbot MistralAI-User cohere-ai DeepSeekBot","llmsTxt":true},{"rank":224,"domain":"sciencedirect.com","blocks":"GPTBot ChatGPT-User ClaudeBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Google-CloudVertexBot Google-Agent Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider MistralAI-User YouBot cohere-ai cohere-training-data-crawler DeepSeekBot AI2Bot Diffbot Timpibot ChatGLM-Spider Claude-Code FriendlyCrawler Google-NotebookLM iAskBot ImagesiftBot img2dataset omgili omgilibot QuillBot ShapBot Webzio-Extended","allows":"OAI-SearchBot Claude-SearchBot DuckAssistBot","llmsTxt":false},{"rank":225,"domain":"twitch.tv","blocks":"","allows":"Amazonbot Amzn-SearchBot Amzn-User","llmsTxt":false},{"rank":227,"domain":"researchgate.net","blocks":"","allows":"","llmsTxt":false},{"rank":228,"domain":"publicnode.com","blocks":"","allows":"","llmsTxt":false},{"rank":229,"domain":"applovin.com","blocks":"","allows":"","llmsTxt":false},{"rank":230,"domain":"mit.edu","blocks":"","allows":"","llmsTxt":false},{"rank":234,"domain":"nist.gov","blocks":"","allows":"","llmsTxt":false},{"rank":236,"domain":"bbc.co.uk","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Amazonbot Amzn-SearchBot Amzn-User CCBot Bytespider YouBot cohere-ai Diffbot ApifyBot ApifyWebsiteContentCrawler Brightbot CloudflareBrowserRenderingCrawler Diffbot-User ExaBot FirecrawlAgent omgili omgilibot Scrapy ShapBot TavilyBot YandexAdditional YandexAdditionalBot","allows":"","llmsTxt":false},{"rank":238,"domain":"cpanel.net","blocks":"","allows":"","llmsTxt":false},{"rank":239,"domain":"namecheap.com","blocks":"","allows":"","llmsTxt":false},{"rank":240,"domain":"hcaptcha.com","blocks":"","allows":"","llmsTxt":true},{"rank":242,"domain":"cisco.com","blocks":"","allows":"","llmsTxt":true},{"rank":245,"domain":"t.co","blocks":"","allows":"","llmsTxt":false},{"rank":246,"domain":"roku.com","blocks":"","allows":"","llmsTxt":true},{"rank":247,"domain":"amplitude.com","blocks":"Bytespider","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot anthropic-ai PerplexityBot Perplexity-User Google-Extended Applebot-Extended Meta-ExternalAgent Amazonbot CCBot cohere-ai","llmsTxt":true},{"rank":250,"domain":"weather.com","blocks":"GoogleOther Google-CloudVertexBot FacebookBot Amazonbot CCBot Bytespider YouBot Diffbot FriendlyCrawler ImagesiftBot omgili omgilibot Scrapy","allows":"ChatGPT-User","llmsTxt":true},{"rank":251,"domain":"sourceforge.net","blocks":"","allows":"","llmsTxt":true},{"rank":254,"domain":"astrumu.io","blocks":"","allows":"","llmsTxt":false},{"rank":261,"domain":"hubspot.com","blocks":"","allows":"","llmsTxt":true},{"rank":263,"domain":"linktr.ee","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ChatGPT Agent ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalFetcher Amazonbot CCBot MistralAI-User DuckAssistBot Bravebot","llmsTxt":true},{"rank":264,"domain":"imdb.com","blocks":"","allows":"Amazonbot Amzn-SearchBot Amzn-User DuckAssistBot","llmsTxt":false},{"rank":269,"domain":"userapi.com","blocks":"","allows":"","llmsTxt":false},{"rank":271,"domain":"who.int","blocks":"CCBot aiHitBot ExaBot Panscient Scrapy","allows":"","llmsTxt":false},{"rank":275,"domain":"wixsite.com","blocks":"","allows":"","llmsTxt":false},{"rank":276,"domain":"canva.com","blocks":"GPTBot ClaudeBot anthropic-ai Claude-Web Applebot-Extended Meta-ExternalAgent FacebookBot CCBot Bytespider AI2Bot Diffbot Timpibot Ai2Bot-Dolma FriendlyCrawler ICC-Crawler ImagesiftBot img2dataset omgili Scrapy VelenPublicWebCrawler Webzio-Extended","allows":"PerplexityBot Perplexity-User","llmsTxt":false},{"rank":277,"domain":"ibm.com","blocks":"","allows":"","llmsTxt":false},{"rank":281,"domain":"launchpad.net","blocks":"GPTBot OAI-SearchBot ClaudeBot Claude-SearchBot Claude-User PerplexityBot Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider DuckAssistBot YouBot cohere-ai cohere-training-data-crawler AI2Bot Diffbot Timpibot Ai2Bot-Dolma aiHitBot Cotoyogi Factset_spyderbot FirecrawlAgent ICC-Crawler ImagesiftBot img2dataset Kangaroo Bot omgili omgilibot PanguBot Scrapy SemrushBot-OCOB TikTokSpider VelenPublicWebCrawler Webzio-Extended","allows":"","llmsTxt":false},{"rank":282,"domain":"ip-api.com","blocks":"","allows":"","llmsTxt":false},{"rank":283,"domain":"reuters.com","blocks":"Applebot-Extended","allows":"OAI-SearchBot ChatGPT-User GoogleOther meta-webindexer Amzn-User","llmsTxt":false},{"rank":284,"domain":"brave.com","blocks":"","allows":"","llmsTxt":false},{"rank":286,"domain":"datadoghq.com","blocks":"","allows":"","llmsTxt":true},{"rank":288,"domain":"hp.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User OAI-AdsBot ClaudeBot Claude-SearchBot Claude-User PerplexityBot Perplexity-User Google-Extended","llmsTxt":false},{"rank":290,"domain":"tinyurl.com","blocks":"","allows":"","llmsTxt":false},{"rank":293,"domain":"booking.com","blocks":"","allows":"","llmsTxt":false},{"rank":294,"domain":"www.gov.uk","blocks":"","allows":"Meta-ExternalAgent","llmsTxt":false},{"rank":297,"domain":"nature.com","blocks":"GPTBot ChatGPT-User ClaudeBot anthropic-ai Claude-Web PerplexityBot Google-Extended GoogleOther Applebot-Extended Meta-ExternalAgent FacebookBot Amazonbot CCBot Bytespider YouBot cohere-ai Diffbot FriendlyCrawler ImagesiftBot img2dataset omgili omgilibot","allows":"","llmsTxt":false},{"rank":298,"domain":"steampowered.com","blocks":"","allows":"","llmsTxt":false},{"rank":302,"domain":"demdex.net","blocks":"","allows":"","llmsTxt":false},{"rank":304,"domain":"salesforce.com","blocks":"","allows":"","llmsTxt":false},{"rank":305,"domain":"springer.com","blocks":"","allows":"","llmsTxt":false},{"rank":310,"domain":"alibaba.com","blocks":"GPTBot Google-Extended","allows":"","llmsTxt":false},{"rank":313,"domain":"vungle.com","blocks":"","allows":"","llmsTxt":false},{"rank":318,"domain":"weibo.com","blocks":"OAI-SearchBot ChatGPT-User Perplexity-User Meta-ExternalAgent Bytespider DeepSeekBot","allows":"","llmsTxt":false},{"rank":319,"domain":"pubmatic.com","blocks":"","allows":"","llmsTxt":true},{"rank":321,"domain":"harvard.edu","blocks":"","allows":"","llmsTxt":false},{"rank":322,"domain":"ok.ru","blocks":"","allows":"","llmsTxt":false},{"rank":323,"domain":"slack.com","blocks":"","allows":"","llmsTxt":true},{"rank":325,"domain":"rubiconproject.com","blocks":"","allows":"","llmsTxt":true},{"rank":326,"domain":"amazon.co.uk","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":328,"domain":"etsy.com","blocks":"","allows":"","llmsTxt":true},{"rank":329,"domain":"launchdarkly.com","blocks":"","allows":"","llmsTxt":false},{"rank":334,"domain":"gmail.com","blocks":"","allows":"","llmsTxt":false},{"rank":336,"domain":"php.net","blocks":"","allows":"","llmsTxt":false},{"rank":337,"domain":"naver.com","blocks":"","allows":"","llmsTxt":false},{"rank":340,"domain":"bilibili.com","blocks":"","allows":"","llmsTxt":true},{"rank":341,"domain":"weebly.com","blocks":"","allows":"","llmsTxt":false},{"rank":342,"domain":"tp-link.com","blocks":"","allows":"","llmsTxt":false},{"rank":344,"domain":"trustpilot.com","blocks":"anthropic-ai Applebot-Extended Bytespider YouBot cohere-ai Diffbot Timpibot ImagesiftBot Kangaroo Bot omgili omgilibot Webzio-Extended","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User PerplexityBot Google-Extended Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer FacebookBot CCBot","llmsTxt":false},{"rank":345,"domain":"ea.com","blocks":"Meta-ExternalAgent Bytespider cohere-ai cohere-training-data-crawler AI2Bot FriendlyCrawler ICC-Crawler","allows":"","llmsTxt":true},{"rank":346,"domain":"zendesk.com","blocks":"","allows":"","llmsTxt":true},{"rank":347,"domain":"gnu.org","blocks":"panscient.com SemrushBot-OCOB SemrushBot-SWA","allows":"","llmsTxt":false},{"rank":349,"domain":"grammarly.com","blocks":"","allows":"","llmsTxt":false},{"rank":350,"domain":"aliyun.com","blocks":"","allows":"GPTBot ClaudeBot CCBot","llmsTxt":false},{"rank":351,"domain":"branch.io","blocks":"","allows":"GPTBot ClaudeBot anthropic-ai PerplexityBot Google-Extended Applebot-Extended CCBot","llmsTxt":true},{"rank":353,"domain":"dell.com","blocks":"","allows":"Bytespider","llmsTxt":true},{"rank":356,"domain":"cdc.gov","blocks":"","allows":"","llmsTxt":false},{"rank":357,"domain":"checkpoint.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-Web PerplexityBot Perplexity-User Google-Extended CCBot","llmsTxt":true},{"rank":358,"domain":"netflix.net","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User PerplexityBot Google-Extended","llmsTxt":false},{"rank":361,"domain":"tradingview.com","blocks":"","allows":"OAI-SearchBot ChatGPT-User ClaudeBot PerplexityBot Google-Extended GoogleOther Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Amazonbot Bytespider cohere-training-data-crawler DeepSeekBot AI2Bot","llmsTxt":false},{"rank":362,"domain":"nvidia.com","blocks":"","allows":"","llmsTxt":true},{"rank":363,"domain":"espn.com","blocks":"GPTBot ChatGPT-User anthropic-ai Google-Extended FacebookBot CCBot Bytespider omgili omgilibot","allows":"","llmsTxt":false},{"rank":364,"domain":"anthropic.com","blocks":"","allows":"","llmsTxt":false},{"rank":366,"domain":"cm.com","blocks":"","allows":"","llmsTxt":true},{"rank":369,"domain":"nasa.gov","blocks":"","allows":"","llmsTxt":false},{"rank":370,"domain":"opendns.com","blocks":"","allows":"","llmsTxt":false},{"rank":375,"domain":"gitlab.com","blocks":"","allows":"","llmsTxt":false},{"rank":377,"domain":"issuu.com","blocks":"AddSearchBot SemrushBot-OCOB YaK","allows":"","llmsTxt":false},{"rank":379,"domain":"stanford.edu","blocks":"","allows":"","llmsTxt":false},{"rank":382,"domain":"wsj.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User GoogleOther Google-CloudVertexBot","llmsTxt":false},{"rank":383,"domain":"washingtonpost.com","blocks":"ClaudeBot anthropic-ai Claude-Web PerplexityBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider cohere-training-data-crawler Diffbot ImagesiftBot omgili omgilibot","allows":"","llmsTxt":false},{"rank":385,"domain":"name.com","blocks":"Meta-ExternalAgent","allows":"","llmsTxt":true},{"rank":386,"domain":"bloomberg.com","blocks":"GPTBot ClaudeBot anthropic-ai Claude-Web PerplexityBot Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher Amazonbot CCBot Bytespider DuckAssistBot YouBot cohere-ai Diffbot Brightbot","allows":"OAI-SearchBot ChatGPT-User Claude-SearchBot Claude-User","llmsTxt":false},{"rank":387,"domain":"amazon.de","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":389,"domain":"plesk.com","blocks":"","allows":"","llmsTxt":true},{"rank":394,"domain":"paloaltonetworks.com","blocks":"","allows":"OAI-SearchBot ChatGPT-User Claude-SearchBot Claude-User PerplexityBot","llmsTxt":true},{"rank":396,"domain":"bsky.app","blocks":"","allows":"","llmsTxt":false},{"rank":398,"domain":"drom.ru","blocks":"","allows":"","llmsTxt":false},{"rank":399,"domain":"inmobi.com","blocks":"","allows":"","llmsTxt":false},{"rank":401,"domain":"yandex.com","blocks":"","allows":"","llmsTxt":false},{"rank":403,"domain":"go.com","blocks":"GPTBot Google-Extended","allows":"","llmsTxt":false},{"rank":406,"domain":"calendly.com","blocks":"CCBot","allows":"","llmsTxt":true},{"rank":407,"domain":"netangels.ru","blocks":"","allows":"","llmsTxt":false},{"rank":409,"domain":"archive.ph","blocks":"","allows":"","llmsTxt":false},{"rank":411,"domain":"claude.ai","blocks":"GPTBot OAI-SearchBot ChatGPT-User Google-Extended","allows":"","llmsTxt":false},{"rank":412,"domain":"okta.com","blocks":"","allows":"","llmsTxt":true},{"rank":414,"domain":"line.me","blocks":"","allows":"","llmsTxt":false},{"rank":416,"domain":"duckduckgo.com","blocks":"","allows":"","llmsTxt":false},{"rank":418,"domain":"unsplash.com","blocks":"ClaudeBot Claude-SearchBot Bytespider ImagesiftBot SemrushBot-OCOB SemrushBot-SWA","allows":"","llmsTxt":false},{"rank":419,"domain":"hostgator.com","blocks":"","allows":"","llmsTxt":false},{"rank":420,"domain":"indeed.com","blocks":"GoogleOther MyCentralAIScraperBot omgili omgilibot Scrapy Webzio-Extended","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai PerplexityBot Perplexity-User Google-Extended Google-CloudVertexBot Google-Agent Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer FacebookBot Amazonbot CCBot Bytespider DuckAssistBot YouBot cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot","llmsTxt":true},{"rank":421,"domain":"aol.com","blocks":"GPTBot ClaudeBot anthropic-ai Claude-Web Google-Extended FacebookBot CCBot Bytespider YouBot cohere-ai Diffbot FriendlyCrawler ImagesiftBot img2dataset omgili omgilibot panscient.com Scrapy","allows":"ChatGPT-User PerplexityBot","llmsTxt":false},{"rank":427,"domain":"steamcommunity.com","blocks":"","allows":"","llmsTxt":false},{"rank":428,"domain":"taboola.com","blocks":"","allows":"","llmsTxt":true},{"rank":429,"domain":"substack.com","blocks":"","allows":"","llmsTxt":false},{"rank":431,"domain":"temu.com","blocks":"Bytespider","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User PerplexityBot Google-Extended FacebookBot","llmsTxt":false},{"rank":435,"domain":"dailymotion.com","blocks":"","allows":"","llmsTxt":true},{"rank":437,"domain":"telekom.de","blocks":"","allows":"","llmsTxt":false},{"rank":442,"domain":"dynatrace.com","blocks":"","allows":"","llmsTxt":true},{"rank":443,"domain":"wps.com","blocks":"","allows":"","llmsTxt":true},{"rank":446,"domain":"intel.com","blocks":"ICC-Crawler","allows":"GPTBot ChatGPT-User ClaudeBot anthropic-ai PerplexityBot Google-Extended","llmsTxt":true},{"rank":453,"domain":"atlassian.net","blocks":"","allows":"","llmsTxt":false},{"rank":454,"domain":"archive.is","blocks":"","allows":"","llmsTxt":false},{"rank":455,"domain":"bluehost.com","blocks":"","allows":"","llmsTxt":true},{"rank":456,"domain":"taobao.com","blocks":"","allows":"","llmsTxt":false},{"rank":458,"domain":"arxiv.org","blocks":"","allows":"","llmsTxt":false},{"rank":461,"domain":"yahoo.co.jp","blocks":"GPTBot Google-Extended","allows":"","llmsTxt":false},{"rank":464,"domain":"pixabay.com","blocks":"GPTBot ClaudeBot Google-Extended Applebot-Extended CCBot Bytespider","allows":"","llmsTxt":false},{"rank":465,"domain":"huawei.com","blocks":"","allows":"","llmsTxt":false},{"rank":466,"domain":"duolingo.com","blocks":"","allows":"","llmsTxt":false},{"rank":468,"domain":"selectel.ru","blocks":"","allows":"","llmsTxt":true},{"rank":469,"domain":"speedtest.net","blocks":"","allows":"","llmsTxt":false},{"rank":471,"domain":"businessinsider.com","blocks":"ClaudeBot anthropic-ai CCBot Bytespider Diffbot Timpibot Scrapy","allows":"","llmsTxt":true},{"rank":473,"domain":"fandom.com","blocks":"GPTBot OAI-SearchBot ClaudeBot Meta-ExternalAgent Meta-ExternalFetcher CCBot GrokBot xAI-Grok","allows":"","llmsTxt":false},{"rank":474,"domain":"cnbc.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Google-Agent Applebot-Extended Meta-ExternalAgent FacebookBot Amazonbot CCBot Bytespider cohere-ai Diffbot Google-NotebookLM omgili omgilibot Scrapy","allows":"","llmsTxt":false},{"rank":477,"domain":"walmart.com","blocks":"","allows":"","llmsTxt":false},{"rank":481,"domain":"kwai.net","blocks":"","allows":"","llmsTxt":false},{"rank":482,"domain":"goodreads.com","blocks":"GPTBot CCBot","allows":"","llmsTxt":false},{"rank":483,"domain":"shein.com","blocks":"","allows":"","llmsTxt":true},{"rank":485,"domain":"appsflyer.com","blocks":"","allows":"","llmsTxt":false},{"rank":486,"domain":"ietf.org","blocks":"","allows":"","llmsTxt":false},{"rank":490,"domain":"amazon.co.jp","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":491,"domain":"quora.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User PerplexityBot Applebot-Extended Bytespider Bravebot Diffbot ImagesiftBot Scrapy","allows":"Google-Extended Amazonbot","llmsTxt":false},{"rank":493,"domain":"statista.com","blocks":"Brightbot 1.0","allows":"GPTBot ClaudeBot anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider DuckAssistBot YouBot cohere-ai cohere-training-data-crawler AI2Bot Diffbot Timpibot","llmsTxt":false},{"rank":497,"domain":"aliexpress.com","blocks":"GPTBot ClaudeBot Google-Extended Amazonbot CCBot Bytespider","allows":"","llmsTxt":false},{"rank":498,"domain":"scribd.com","blocks":"ClaudeBot Claude-Web CCBot Bytespider AI2Bot Diffbot ImagesiftBot omgili","allows":"","llmsTxt":false},{"rank":503,"domain":"teamviewer.com","blocks":"","allows":"","llmsTxt":false},{"rank":504,"domain":"addtoany.com","blocks":"","allows":"","llmsTxt":false},{"rank":505,"domain":"eventbrite.com","blocks":"","allows":"CCBot","llmsTxt":false},{"rank":506,"domain":"sina.com.cn","blocks":"","allows":"","llmsTxt":false},{"rank":509,"domain":"foxnews.com","blocks":"","allows":"","llmsTxt":true},{"rank":510,"domain":"noaa.gov","blocks":"","allows":"","llmsTxt":false},{"rank":511,"domain":"supercell.com","blocks":"","allows":"","llmsTxt":false},{"rank":512,"domain":"trendmicro.com","blocks":"","allows":"OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended Meta-ExternalAgent FacebookBot Amazonbot CCBot Bytespider DuckAssistBot cohere-ai","llmsTxt":true},{"rank":515,"domain":"indiatimes.com","blocks":"Meta-ExternalAgent","allows":"","llmsTxt":false},{"rank":516,"domain":"blogger.com","blocks":"","allows":"","llmsTxt":false},{"rank":518,"domain":"google.de","blocks":"","allows":"","llmsTxt":false},{"rank":519,"domain":"amzn.to","blocks":"","allows":"","llmsTxt":false},{"rank":521,"domain":"google.com.hk","blocks":"","allows":"","llmsTxt":false},{"rank":524,"domain":"mobile.de","blocks":"","allows":"","llmsTxt":false},{"rank":525,"domain":"behance.net","blocks":"GPTBot ClaudeBot anthropic-ai Claude-Web Google-Extended Applebot-Extended Meta-ExternalAgent FacebookBot CCBot Bytespider cohere-training-data-crawler AI2Bot Diffbot Timpibot ImagesiftBot Kangaroo Bot omgili omgilibot PanguBot Webzio-Extended","allows":"","llmsTxt":false},{"rank":530,"domain":"openstreetmap.org","blocks":"","allows":"","llmsTxt":false},{"rank":531,"domain":"docker.com","blocks":"","allows":"","llmsTxt":false},{"rank":532,"domain":"tplinkcloud.com","blocks":"","allows":"","llmsTxt":false},{"rank":534,"domain":"braze.com","blocks":"","allows":"","llmsTxt":true},{"rank":535,"domain":"uol.com.br","blocks":"ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler AI2Bot Diffbot Timpibot Cotoyogi Datenbank Crawler Devin Factset_spyderbot Gemini-Deep-Research GoogleAgent-Mariner ICC-Crawler ImagesiftBot Kangaroo Bot netEstate Imprint Crawler NovaAct omgili omgilibot Operator PanguBot SemrushBot-OCOB Webzio-Extended","allows":"","llmsTxt":false},{"rank":537,"domain":"cloudns.net","blocks":"","allows":"","llmsTxt":true},{"rank":539,"domain":"ivi.ru","blocks":"","allows":"","llmsTxt":false},{"rank":543,"domain":"mysql.com","blocks":"","allows":"","llmsTxt":false},{"rank":544,"domain":"newrelic.com","blocks":"","allows":"","llmsTxt":true},{"rank":545,"domain":"netcraze.io","blocks":"","allows":"","llmsTxt":false},{"rank":546,"domain":"capcut.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User OAI-AdsBot ClaudeBot Claude-SearchBot Claude-User PerplexityBot Perplexity-User Google-Extended Google-Agent Applebot-Extended Meta-ExternalFetcher meta-webindexer Amzn-SearchBot Amzn-User CCBot DuckAssistBot","llmsTxt":true},{"rank":547,"domain":"mozilla.com","blocks":"","allows":"","llmsTxt":false},{"rank":550,"domain":"slideshare.net","blocks":"ClaudeBot Claude-Web CCBot Bytespider AI2Bot Diffbot ImagesiftBot omgili","allows":"","llmsTxt":false},{"rank":552,"domain":"ft.com","blocks":"ClaudeBot anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot CCBot Bytespider cohere-ai Diffbot omgili omgilibot","allows":"","llmsTxt":false},{"rank":554,"domain":"doubleverify.com","blocks":"","allows":"","llmsTxt":false},{"rank":557,"domain":"sohu.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Google-CloudVertexBot Google-Agent Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai DeepSeekBot Diffbot","llmsTxt":false},{"rank":558,"domain":"firefox.com","blocks":"","allows":"","llmsTxt":false},{"rank":562,"domain":"visualstudio.com","blocks":"","allows":"","llmsTxt":false},{"rank":564,"domain":"tandfonline.com","blocks":"GPTBot","allows":"","llmsTxt":false},{"rank":568,"domain":"fast.com","blocks":"","allows":"","llmsTxt":false},{"rank":570,"domain":"wired.com","blocks":"ClaudeBot Claude-SearchBot Claude-User PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer Amazonbot CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler Bravebot Diffbot Timpibot Brightbot 1.0 Factset_spyderbot FirecrawlAgent PanguBot Scrapy Shap-User ShapBot Webzio-Extended","allows":"","llmsTxt":false},{"rank":571,"domain":"cornell.edu","blocks":"","allows":"","llmsTxt":false},{"rank":572,"domain":"tripadvisor.com","blocks":"GPTBot ClaudeBot Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Amazonbot CCBot Bytespider YouBot cohere-ai img2dataset PanguBot","allows":"OAI-SearchBot ChatGPT-User PerplexityBot","llmsTxt":false},{"rank":573,"domain":"playstation.com","blocks":"","allows":"","llmsTxt":true},{"rank":576,"domain":"herokuapp.com","blocks":"","allows":"","llmsTxt":false},{"rank":577,"domain":"shop.app","blocks":"","allows":"","llmsTxt":true},{"rank":578,"domain":"klaviyo.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User PerplexityBot Perplexity-User Google-Agent Meta-ExternalAgent DuckAssistBot YouBot","llmsTxt":true},{"rank":579,"domain":"comcast.com","blocks":"","allows":"","llmsTxt":true},{"rank":582,"domain":"dreamhost.com","blocks":"","allows":"","llmsTxt":true},{"rank":583,"domain":"time.com","blocks":"","allows":"","llmsTxt":true},{"rank":584,"domain":"ezvizlife.com","blocks":"","allows":"","llmsTxt":false},{"rank":585,"domain":"jquery.com","blocks":"","allows":"","llmsTxt":false},{"rank":587,"domain":"squarespace.com","blocks":"","allows":"GPTBot ClaudeBot anthropic-ai Google-Extended GoogleOther Applebot-Extended Meta-ExternalAgent FacebookBot Amazonbot CCBot Bytespider DuckAssistBot YouBot cohere-ai cohere-training-data-crawler AI2Bot","llmsTxt":true},{"rank":588,"domain":"free.fr","blocks":"","allows":"","llmsTxt":false},{"rank":591,"domain":"threads.com","blocks":"GPTBot ClaudeBot PerplexityBot Google-Extended Applebot-Extended Amazonbot Brightbot Scrapy YaK","allows":"","llmsTxt":false},{"rank":592,"domain":"deviantart.com","blocks":"","allows":"","llmsTxt":false},{"rank":593,"domain":"rakuten.co.jp","blocks":"","allows":"","llmsTxt":false},{"rank":597,"domain":"eset.com","blocks":"","allows":"","llmsTxt":false},{"rank":598,"domain":"360yield.com","blocks":"","allows":"","llmsTxt":false},{"rank":599,"domain":"ca.gov","blocks":"","allows":"","llmsTxt":false},{"rank":604,"domain":"media.net","blocks":"","allows":"","llmsTxt":false},{"rank":605,"domain":"atlassian.com","blocks":"","allows":"","llmsTxt":true},{"rank":606,"domain":"target.com","blocks":"","allows":"","llmsTxt":true},{"rank":613,"domain":"digitalocean.com","blocks":"","allows":"","llmsTxt":false},{"rank":616,"domain":"wix.com","blocks":"","allows":"","llmsTxt":true},{"rank":617,"domain":"usatoday.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ChatGPT Agent ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Google-Agent Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer FacebookBot Amazonbot Amzn-SearchBot Amzn-User CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot AI2Bot Bravebot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot AmazonBuyForMe Andibot Anomura ApifyBot atlassian-bot bigsur.ai Brightbot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Crawlspace Datenbank Crawler Devin Diffbot-User Echobot Bot EchoboxBot ExaBot Factset_spyderbot FirecrawlAgent FriendlyCrawler Gemini-Deep-Research Google-NotebookLM GoogleAgent-Mariner iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot laion-huggingface-processor LCC LinerBot LinkupBot Manus-User netEstate Imprint Crawler NovaAct omgili omgilibot PanguBot PhindBot Poggio-Citations Poseidon Research Crawler QualifiedBot QuillBot SBIntuitionsBot Scrapy SemrushBot-OCOB ShapBot Sidetrade indexer bot Spider TavilyBot TikTokSpider TwinAgent VelenPublicWebCrawler WARDBot Webzio-Extended YandexAdditional YandexAdditionalBot ZanistaBot","allows":"","llmsTxt":false},{"rank":619,"domain":"dailymail.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai DeepSeekBot AI2Bot Diffbot Timpibot Ai2Bot-Dolma bedrockbot ChatGLM-Spider CloudflareBrowserRenderingCrawler Cotoyogi EchoboxBot FriendlyCrawler Gemini-Deep-Research GoogleOther-Image GoogleOther-Video iaskspider ICC-Crawler ImagesiftBot img2dataset ISSCyberRiskCrawler Kangaroo Bot KunatoCrawler LinerBot omgili omgilibot PanguBot QualifiedBot Scrapy Sidetrade indexer bot TikTokSpider VelenPublicWebCrawler WARDBot Webzio-Extended wpbot WRTNBot","allows":"GoogleOther","llmsTxt":false},{"rank":624,"domain":"sagepub.com","blocks":"Meta-ExternalAgent Amazonbot CCBot Bytespider cohere-ai Diffbot ImagesiftBot omgili omgilibot","allows":"GPTBot ChatGPT-User ClaudeBot anthropic-ai Google-Extended Applebot-Extended","llmsTxt":false},{"rank":626,"domain":"endpoints.news","blocks":"GPTBot ClaudeBot anthropic-ai Claude-Web PerplexityBot Perplexity-User CCBot Bytespider cohere-ai Diffbot omgili omgilibot","allows":"Google-Extended GoogleOther","llmsTxt":false},{"rank":628,"domain":"berkeley.edu","blocks":"","allows":"","llmsTxt":false},{"rank":629,"domain":"telegraph.co.uk","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User GoogleOther Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider MistralAI-User DuckAssistBot cohere-ai DeepSeekBot AI2Bot Diffbot Timpibot Ai2Bot-Dolma bedrockbot ChatGLM-Spider Cotoyogi FriendlyCrawler GoogleOther-Image GoogleOther-Video iaskspider ICC-Crawler ImagesiftBot img2dataset ISSCyberRiskCrawler Kangaroo Bot KunatoCrawler omgili omgilibot PanguBot QualifiedBot Scrapy Sidetrade indexer bot TikTokSpider VelenPublicWebCrawler WARDBot Webzio-Extended WRTNBot","allows":"Google-Extended","llmsTxt":false},{"rank":630,"domain":"avito.ru","blocks":"","allows":"","llmsTxt":false},{"rank":631,"domain":"hotjar.com","blocks":"","allows":"","llmsTxt":false},{"rank":632,"domain":"amazon.fr","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":633,"domain":"amazon.ca","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":634,"domain":"techcrunch.com","blocks":"GPTBot ChatGPT-User ClaudeBot anthropic-ai Claude-Web Google-Extended Applebot-Extended FacebookBot CCBot Bytespider cohere-ai Diffbot omgili","allows":"","llmsTxt":false},{"rank":635,"domain":"nintendo.com","blocks":"GPTBot ClaudeBot Claude-Web Google-Extended Applebot-Extended Meta-ExternalAgent FacebookBot CCBot Bytespider cohere-ai cohere-training-data-crawler AI2Bot Diffbot Timpibot Ai2Bot-Dolma FriendlyCrawler ICC-Crawler Kangaroo Bot omgili omgilibot PanguBot Webzio-Extended","allows":"","llmsTxt":false},{"rank":636,"domain":"ikea.com","blocks":"Amazonbot","allows":"","llmsTxt":false},{"rank":637,"domain":"bitrix24.ru","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User PerplexityBot Perplexity-User Google-Extended","llmsTxt":true},{"rank":641,"domain":"merriam-webster.com","blocks":"","allows":"","llmsTxt":false},{"rank":642,"domain":"verisign.com","blocks":"","allows":"","llmsTxt":false},{"rank":643,"domain":"sophos.com","blocks":"Amazonbot DeepSeekBot","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot anthropic-ai PerplexityBot Google-Extended Amzn-SearchBot Amzn-User YouBot","llmsTxt":true},{"rank":644,"domain":"onetrust.com","blocks":"","allows":"","llmsTxt":true},{"rank":645,"domain":"jotform.com","blocks":"","allows":"GPTBot OAI-SearchBot OAI-AdsBot ClaudeBot anthropic-ai Claude-Web PerplexityBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot CCBot Bytespider cohere-ai Diffbot","llmsTxt":false},{"rank":651,"domain":"beget.ru","blocks":"","allows":"","llmsTxt":false},{"rank":654,"domain":"tencent.com","blocks":"","allows":"","llmsTxt":false},{"rank":655,"domain":"intercom.io","blocks":"","allows":"","llmsTxt":true},{"rank":657,"domain":"meta.com","blocks":"Scrapy YaK","allows":"","llmsTxt":false},{"rank":659,"domain":"charter.net","blocks":"","allows":"","llmsTxt":false},{"rank":661,"domain":"2gis.com","blocks":"GoogleOther","allows":"","llmsTxt":false},{"rank":662,"domain":"loc.gov","blocks":"","allows":"","llmsTxt":false},{"rank":664,"domain":"stackadapt.com","blocks":"","allows":"","llmsTxt":true},{"rank":668,"domain":"iso.org","blocks":"GPTBot ChatGPT-User Google-Extended FacebookBot CCBot omgili omgilibot","allows":"","llmsTxt":false},{"rank":670,"domain":"optimizely.com","blocks":"","allows":"","llmsTxt":true},{"rank":673,"domain":"binance.com","blocks":"","allows":"","llmsTxt":false},{"rank":674,"domain":"elpais.com","blocks":"ClaudeBot Claude-SearchBot Claude-User PerplexityBot Perplexity-User Amazonbot Amzn-SearchBot Amzn-User CCBot Bytespider MistralAI-User DuckAssistBot DeepSeekBot ExaBot LinerBot omgilibot","allows":"","llmsTxt":false},{"rank":676,"domain":"amazon.in","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":677,"domain":"nike.com","blocks":"","allows":"","llmsTxt":false},{"rank":678,"domain":"primevideo.com","blocks":"ChatGPT-User Claude-User PerplexityBot Perplexity-User GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":680,"domain":"redhat.com","blocks":"","allows":"","llmsTxt":true},{"rank":681,"domain":"sharethrough.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-User PerplexityBot GoogleOther Meta-ExternalAgent CCBot Bytespider YouBot cohere-ai AI2Bot Diffbot Timpibot SBIntuitionsBot","allows":"","llmsTxt":false},{"rank":682,"domain":"britannica.com","blocks":"","allows":"","llmsTxt":false},{"rank":684,"domain":"box.com","blocks":"","allows":"","llmsTxt":false},{"rank":686,"domain":"surveymonkey.com","blocks":"","allows":"","llmsTxt":true},{"rank":689,"domain":"ngenix.net","blocks":"","allows":"","llmsTxt":false},{"rank":691,"domain":"cambridge.org","blocks":"ChatGPT-User","allows":"","llmsTxt":false},{"rank":693,"domain":"perplexity.ai","blocks":"","allows":"","llmsTxt":false},{"rank":696,"domain":"xhamster.com","blocks":"","allows":"","llmsTxt":false},{"rank":698,"domain":"ring.com","blocks":"","allows":"GPTBot ChatGPT-User ClaudeBot anthropic-ai PerplexityBot Google-Extended Applebot-Extended cohere-ai","llmsTxt":true},{"rank":699,"domain":"kick.com","blocks":"","allows":"","llmsTxt":false},{"rank":700,"domain":"lemonde.fr","blocks":"ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web Google-Extended GoogleOther Google-CloudVertexBot Google-Agent Applebot-Extended meta-webindexer FacebookBot Amazonbot CCBot Bytespider MistralAI-User MistralAI-Index DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot KimiBot Kimi-SearchBot AI2Bot Bravebot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma AIWebIndex amazon-kendra amazon-QBusiness AmazonBuyForMe Anomura ApifyBot ApifyWebsiteContentCrawler Aranet-SearchBot atlassian-bot AzureAI-SearchBot bedrockbot bigsur.ai Brightbot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudVertexBot Cotoyogi CragCrawler Crawl4AI Crawlspace Datenbank Crawler Devin DoubaoBot ERNIEBot ExaBot Factset_spyderbot FirecrawlAgent FriendlyCrawler GeistHaus-PageFetcher Gemini-Deep-Research Google-NotebookLM GoogleAgent-Mariner GoogleAgent-URLContext HenkBot iAskBot iaskspider ICC-Crawler imageSpider kagi-fetcher Kangaroo Bot Kimi-User KlaviyoAIBot KunatoCrawler laion-huggingface-processor LCC LinerBot LinkupBot Manus-User MoonshotBot Mozilla-Tabstack netEstate Imprint Crawler NovaAct omgili omgilibot PanguBot PhindBot Poggio-Citations QualifiedBot Querit-SearchBot QueritBot QwenBot SBIntuitionsBot SemrushBot-OCOB Shap-User ShapBot Spider TavilyBot Terra Cotta TerraCotta TongyiBot TwinAgent UseAI VelenPublicWebCrawler Webzio-Extended WRTNBot YaK YandexAdditional YandexAdditionalBot YiyanBot ZanistaBot","allows":"","llmsTxt":false},{"rank":701,"domain":"cbsnews.com","blocks":"GPTBot panscient.com","allows":"","llmsTxt":false},{"rank":702,"domain":"yelp.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Meta-ExternalAgent Meta-ExternalFetcher CCBot","allows":"","llmsTxt":false},{"rank":704,"domain":"rustore.ru","blocks":"","allows":"","llmsTxt":false},{"rank":705,"domain":"fb.com","blocks":"GPTBot ClaudeBot PerplexityBot Google-Extended Applebot-Extended Amazonbot Brightbot Scrapy YaK","allows":"","llmsTxt":false},{"rank":709,"domain":"telefonica.de","blocks":"","allows":"","llmsTxt":false},{"rank":710,"domain":"360.cn","blocks":"","allows":"","llmsTxt":false},{"rank":711,"domain":"bandcamp.com","blocks":"GPTBot ClaudeBot Google-Extended Meta-ExternalAgent FacebookBot Amazonbot CCBot Bytespider Diffbot omgili","allows":"","llmsTxt":false},{"rank":713,"domain":"imgsmail.ru","blocks":"","allows":"","llmsTxt":false},{"rank":714,"domain":"amazon.es","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":717,"domain":"dailymail.co.uk","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai DeepSeekBot AI2Bot Diffbot Timpibot Ai2Bot-Dolma bedrockbot ChatGLM-Spider CloudflareBrowserRenderingCrawler Cotoyogi EchoboxBot FriendlyCrawler Gemini-Deep-Research GoogleOther-Image GoogleOther-Video iaskspider ICC-Crawler ImagesiftBot img2dataset ISSCyberRiskCrawler Kangaroo Bot KunatoCrawler LinerBot omgili omgilibot PanguBot QualifiedBot Scrapy Sidetrade indexer bot TikTokSpider VelenPublicWebCrawler WARDBot Webzio-Extended wpbot WRTNBot","allows":"GoogleOther","llmsTxt":false},{"rank":719,"domain":"stripchat.com","blocks":"","allows":"","llmsTxt":false},{"rank":721,"domain":"hihonorcloud.com","blocks":"","allows":"","llmsTxt":false},{"rank":722,"domain":"outbrain.com","blocks":"","allows":"","llmsTxt":false},{"rank":723,"domain":"ya.ru","blocks":"","allows":"","llmsTxt":false},{"rank":727,"domain":"itch.io","blocks":"","allows":"","llmsTxt":false},{"rank":729,"domain":"uber.com","blocks":"","allows":"","llmsTxt":false},{"rank":730,"domain":"pinimg.com","blocks":"YandexAdditional YandexAdditionalBot","allows":"FacebookBot DuckAssistBot Bravebot","llmsTxt":false},{"rank":731,"domain":"linode.com","blocks":"","allows":"","llmsTxt":false},{"rank":735,"domain":"amazon.it","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":737,"domain":"cnet.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer FacebookBot Amazonbot Amzn-SearchBot Amzn-User CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Bravebot Diffbot Timpibot AddSearchBot AgentDataBot AI2Bot-DeepResearchEval Ai2Bot-Dolma AIWebIndex amazon-QBusiness AmazonBuyForMe ApifyBot ApifyWebsiteContentCrawler atlassian-bot AzureAI-SearchBot bigsur.ai BixelBot Brightbot Channel3Bot ChatGLM-Spider Claude-Code Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Code Cotoyogi Crawl4AI Cursor Datenbank Crawler Devin Echobot Bot ExaBot ExaSearchBot Factset_spyderbot FirecrawlAgent GeistHaus-PageFetcher Hunyuan iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider kagi-fetcher Kangaroo Bot KeenableBot Kimi-Agent Kimi-User KlaviyoAIBot KunatoCrawler laion-huggingface-processor LCC LinerBot LinkupBot Manus-User netEstate Imprint Crawler NovaAct omgili omgilibot opencode PanguBot PhindBot Poggio-Citations qodercli QualifiedBot Reflectionbot SBIntuitionsBot Scrapy SemrushBot-OCOB Shap-User ShapBot Spider TavilyBot TerraCotta TongyiBot Trae TwinAgent UseAI VelenPublicWebCrawler Webzio-Extended WRTNBot xAI-Grok YandexAdditionalBot YiBot YiyanBot ZanistaBot","allows":"","llmsTxt":false},{"rank":738,"domain":"quantserve.com","blocks":"GoogleOther","allows":"","llmsTxt":false},{"rank":740,"domain":"viber.com","blocks":"","allows":"","llmsTxt":false},{"rank":741,"domain":"mediafire.com","blocks":"","allows":"","llmsTxt":false},{"rank":743,"domain":"asus.com","blocks":"","allows":"","llmsTxt":true},{"rank":745,"domain":"shopee.co.id","blocks":"","allows":"","llmsTxt":false},{"rank":746,"domain":"gumgum.com","blocks":"Brightbot","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot anthropic-ai PerplexityBot Google-Extended GoogleOther","llmsTxt":true},{"rank":747,"domain":"creativecdn.com","blocks":"","allows":"","llmsTxt":false},{"rank":750,"domain":"hugedomains.com","blocks":"","allows":"","llmsTxt":false},{"rank":752,"domain":"marriott.com","blocks":"","allows":"","llmsTxt":false},{"rank":753,"domain":"wpguardian.io","blocks":"","allows":"","llmsTxt":false},{"rank":755,"domain":"wpguardian.com","blocks":"","allows":"","llmsTxt":false},{"rank":758,"domain":"python.org","blocks":"","allows":"","llmsTxt":false},{"rank":759,"domain":"bild.de","blocks":"PerplexityBot Perplexity-User Applebot-Extended Amazonbot CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler AI2Bot Diffbot Timpibot Ai2Bot-Dolma aiHitBot Andibot Awario bedrockbot Brightbot 1.0 Cotoyogi Crawlspace Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FirecrawlAgent FriendlyCrawler iaskspider ICC-Crawler ImagesiftBot img2dataset ISSCyberRiskCrawler Kangaroo Bot LCC MyCentralAIScraperBot netEstate Imprint Crawler NovaAct omgili omgilibot Operator PanguBot Panscient panscient.com PhindBot Poseidon Research Crawler QualifiedBot QuillBot quillbot.com SBIntuitionsBot Scrapy SemrushBot-OCOB SemrushBot-SWA Sidetrade indexer bot TerraCotta Thinkbot TikTokSpider VelenPublicWebCrawler WARDBot Webzio-Extended wpbot YandexAdditional YandexAdditionalBot","allows":"","llmsTxt":false},{"rank":761,"domain":"globalsign.com","blocks":"GPTBot ClaudeBot Google-Extended Applebot-Extended Meta-ExternalAgent CCBot Bytespider","allows":"OAI-SearchBot ChatGPT-User Claude-SearchBot PerplexityBot","llmsTxt":false},{"rank":762,"domain":"rfc-editor.org","blocks":"","allows":"","llmsTxt":false},{"rank":764,"domain":"google.co.uk","blocks":"","allows":"","llmsTxt":false},{"rank":766,"domain":"homedepot.com","blocks":"","allows":"","llmsTxt":false},{"rank":769,"domain":"airbnb.com","blocks":"GPTBot ClaudeBot Applebot-Extended MistralAI-Training AI2Bot ImagesiftBot Webzio-Extended","allows":"OAI-SearchBot ChatGPT-User anthropic-ai PerplexityBot Meta-ExternalAgent cohere-ai","llmsTxt":false},{"rank":770,"domain":"mailchimp.com","blocks":"","allows":"","llmsTxt":true},{"rank":772,"domain":"craigslist.org","blocks":"","allows":"","llmsTxt":false},{"rank":774,"domain":"imgur.com","blocks":"","allows":"","llmsTxt":false},{"rank":775,"domain":"mlb.com","blocks":"","allows":"","llmsTxt":false},{"rank":776,"domain":"mikrotik.com","blocks":"","allows":"","llmsTxt":false},{"rank":777,"domain":"fontawesome.com","blocks":"","allows":"","llmsTxt":true},{"rank":780,"domain":"deepl.com","blocks":"","allows":"","llmsTxt":false},{"rank":781,"domain":"livejournal.com","blocks":"","allows":"","llmsTxt":false},{"rank":783,"domain":"welt.de","blocks":"PerplexityBot Perplexity-User Applebot-Extended Amazonbot CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler AI2Bot Diffbot Timpibot Ai2Bot-Dolma aiHitBot Andibot Awario bedrockbot Brightbot 1.0 Cotoyogi Crawlspace Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FirecrawlAgent FriendlyCrawler iaskspider ICC-Crawler ImagesiftBot img2dataset ISSCyberRiskCrawler Kangaroo Bot LCC MyCentralAIScraperBot netEstate Imprint Crawler NovaAct omgili omgilibot Operator PanguBot Panscient panscient.com PhindBot Poseidon Research Crawler QualifiedBot QuillBot quillbot.com SBIntuitionsBot Scrapy SemrushBot-OCOB SemrushBot-SWA Sidetrade indexer bot TerraCotta Thinkbot TikTokSpider VelenPublicWebCrawler WARDBot Webzio-Extended wpbot YandexAdditional YandexAdditionalBot","allows":"","llmsTxt":false},{"rank":784,"domain":"crwdcntrl.net","blocks":"","allows":"","llmsTxt":false},{"rank":785,"domain":"shopee.com.br","blocks":"","allows":"","llmsTxt":false},{"rank":786,"domain":"amazon.co.za","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":789,"domain":"allegro.pl","blocks":"","allows":"CCBot","llmsTxt":true},{"rank":792,"domain":"maricopa.gov","blocks":"","allows":"GPTBot OAI-SearchBot ClaudeBot Claude-SearchBot PerplexityBot Meta-ExternalAgent meta-webindexer Amazonbot CCBot Bytespider DuckAssistBot YouBot cohere-ai","llmsTxt":false},{"rank":793,"domain":"wattpad.com","blocks":"GPTBot","allows":"","llmsTxt":false},{"rank":795,"domain":"zillow.com","blocks":"Applebot-Extended Meta-ExternalAgent Amazonbot YouBot AI2Bot AI2Bot-DeepResearchEval Ai2Bot-Dolma amazon-kendra YandexAdditional YandexAdditionalBot","allows":"","llmsTxt":false},{"rank":796,"domain":"apnews.com","blocks":"GPTBot ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Applebot-Extended Amazonbot CCBot cohere-ai Timpibot","allows":"","llmsTxt":false},{"rank":798,"domain":"sentinelone.net","blocks":"","allows":"","llmsTxt":false},{"rank":800,"domain":"moloco.com","blocks":"Bytespider","allows":"CCBot","llmsTxt":true},{"rank":801,"domain":"google.com.br","blocks":"","allows":"","llmsTxt":false},{"rank":802,"domain":"ameblo.jp","blocks":"","allows":"","llmsTxt":false},{"rank":805,"domain":"zoho.com","blocks":"","allows":"","llmsTxt":true},{"rank":806,"domain":"recaptcha.net","blocks":"","allows":"","llmsTxt":false},{"rank":808,"domain":"flurry.com","blocks":"","allows":"","llmsTxt":false},{"rank":812,"domain":"rakuten.com","blocks":"","allows":"","llmsTxt":false},{"rank":813,"domain":"zoom.com","blocks":"","allows":"","llmsTxt":false},{"rank":814,"domain":"wp.pl","blocks":"GPTBot CCBot","allows":"OAI-SearchBot ChatGPT-User PerplexityBot Perplexity-User","llmsTxt":false},{"rank":818,"domain":"prnewswire.com","blocks":"","allows":"","llmsTxt":true},{"rank":820,"domain":"discordapp.com","blocks":"","allows":"","llmsTxt":false},{"rank":821,"domain":"fiverr.com","blocks":"","allows":"","llmsTxt":true},{"rank":822,"domain":"disneyplus.com","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User PerplexityBot Google-Extended Google-Agent Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher Amazonbot CCBot Bytespider DuckAssistBot YouBot DeepSeekBot GrokBot","llmsTxt":false},{"rank":823,"domain":"ted.com","blocks":"GPTBot ClaudeBot anthropic-ai Claude-Web Google-Extended Applebot-Extended Meta-ExternalAgent FacebookBot Amazonbot CCBot Bytespider cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot Diffbot Timpibot Brightbot FriendlyCrawler iaskspider ImagesiftBot img2dataset omgili omgilibot PanguBot Webzio-Extended","allows":"OAI-SearchBot ChatGPT-User Claude-SearchBot Claude-User PerplexityBot Perplexity-User MistralAI-User DuckAssistBot YouBot","llmsTxt":false},{"rank":824,"domain":"conviva.com","blocks":"Brightbot","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot anthropic-ai PerplexityBot Google-Extended GoogleOther","llmsTxt":false},{"rank":827,"domain":"people.com","blocks":"ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer FacebookBot Amazonbot Amzn-SearchBot Amzn-User CCBot Bytespider MistralAI-User MistralAI-Index DuckAssistBot YouBot cohere-ai cohere-training-data-crawler Bravebot Diffbot Timpibot amazon-kendra AmazonBuyForMe Brightbot 1.0 Diffbot-User ERNIEBot ExaSearchBot FirecrawlAgent FriendlyCrawler iAskBot iaskspider ImagesiftBot imageSpider ISSCyberRiskCrawler LinerBot LinkupBot omgili omgilibot Shap-User ShapBot VelenPublicWebCrawler Webzio-Extended","allows":"GPTBot OAI-SearchBot ChatGPT-User","llmsTxt":false},{"rank":828,"domain":"erome.com","blocks":"","allows":"","llmsTxt":false},{"rank":829,"domain":"gotinder.com","blocks":"","allows":"","llmsTxt":false},{"rank":830,"domain":"mixpanel.com","blocks":"","allows":"","llmsTxt":false},{"rank":831,"domain":"parklogic.com","blocks":"","allows":"","llmsTxt":false},{"rank":834,"domain":"usps.com","blocks":"","allows":"","llmsTxt":false},{"rank":835,"domain":"onlyfans.com","blocks":"","allows":"","llmsTxt":false},{"rank":838,"domain":"liftoff.io","blocks":"","allows":"","llmsTxt":true},{"rank":842,"domain":"vivo.com.cn","blocks":"","allows":"","llmsTxt":false},{"rank":843,"domain":"netlify.app","blocks":"","allows":"","llmsTxt":true},{"rank":844,"domain":"vkontakte.ru","blocks":"","allows":"","llmsTxt":false},{"rank":845,"domain":"consultant.ru","blocks":"","allows":"","llmsTxt":false},{"rank":846,"domain":"xvideos.com","blocks":"","allows":"","llmsTxt":false},{"rank":847,"domain":"adform.net","blocks":"","allows":"GoogleOther","llmsTxt":false},{"rank":850,"domain":"life360.com","blocks":"","allows":"GPTBot ChatGPT-User ClaudeBot PerplexityBot Google-Extended","llmsTxt":true},{"rank":851,"domain":"academia.edu","blocks":"GPTBot Google-Extended Applebot-Extended","allows":"OAI-SearchBot ChatGPT-User Claude-SearchBot Claude-User PerplexityBot MistralAI-User DuckAssistBot YouBot","llmsTxt":false},{"rank":852,"domain":"supabase.co","blocks":"","allows":"","llmsTxt":true},{"rank":853,"domain":"agoda.com","blocks":"","allows":"","llmsTxt":false},{"rank":855,"domain":"vivoglobal.com","blocks":"","allows":"","llmsTxt":false},{"rank":858,"domain":"synology.com","blocks":"","allows":"","llmsTxt":false},{"rank":859,"domain":"cdnvideo.ru","blocks":"","allows":"","llmsTxt":false},{"rank":861,"domain":"myspace.com","blocks":"","allows":"","llmsTxt":false},{"rank":862,"domain":"theverge.com","blocks":"ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot AI2Bot Diffbot Timpibot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot ApifyBot ApifyWebsiteContentCrawler Brightbot ChatGLM-Spider CloudVertexBot Cotoyogi Crawl4AI FirecrawlAgent FriendlyCrawler iaskspider ICC-Crawler ImagesiftBot img2dataset omgili omgilibot SBIntuitionsBot Scrapy TikTokSpider VelenPublicWebCrawler Webzio-Extended","allows":"GPTBot Amazonbot","llmsTxt":false},{"rank":863,"domain":"chess.com","blocks":"","allows":"","llmsTxt":false},{"rank":865,"domain":"unpkg.com","blocks":"","allows":"GPTBot ClaudeBot PerplexityBot Google-Extended Applebot-Extended Meta-ExternalAgent CCBot","llmsTxt":true},{"rank":867,"domain":"frontiersin.org","blocks":"","allows":"","llmsTxt":true},{"rank":868,"domain":"amazon.com.br","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":869,"domain":"one.one","blocks":"","allows":"","llmsTxt":true},{"rank":871,"domain":"flipkart.com","blocks":"","allows":"","llmsTxt":false},{"rank":872,"domain":"agora.io","blocks":"","allows":"","llmsTxt":true},{"rank":873,"domain":"cursor.sh","blocks":"","allows":"","llmsTxt":true},{"rank":874,"domain":"www.gov.br","blocks":"","allows":"","llmsTxt":false},{"rank":875,"domain":"pexels.com","blocks":"GPTBot ClaudeBot Applebot-Extended CCBot Bytespider","allows":"","llmsTxt":false},{"rank":877,"domain":"ebay.co.uk","blocks":"GPTBot ClaudeBot anthropic-ai PerplexityBot Applebot-Extended Meta-ExternalAgent Amazonbot CCBot Bytespider ChatGLM-Spider","allows":"OAI-SearchBot ChatGPT-User Claude-SearchBot Claude-User","llmsTxt":false},{"rank":879,"domain":"porkbun.com","blocks":"","allows":"","llmsTxt":true},{"rank":880,"domain":"cloudinary.com","blocks":"","allows":"","llmsTxt":true},{"rank":881,"domain":"pikabu.ru","blocks":"","allows":"","llmsTxt":false},{"rank":882,"domain":"ebay.de","blocks":"GPTBot ClaudeBot anthropic-ai PerplexityBot Applebot-Extended Meta-ExternalAgent Amazonbot CCBot Bytespider ChatGLM-Spider","allows":"","llmsTxt":false},{"rank":887,"domain":"biblegateway.com","blocks":"","allows":"","llmsTxt":false},{"rank":888,"domain":"no-ip.com","blocks":"","allows":"","llmsTxt":true},{"rank":890,"domain":"pendo.io","blocks":"","allows":"","llmsTxt":true},{"rank":891,"domain":"deloitte.com","blocks":"","allows":"","llmsTxt":false},{"rank":893,"domain":"arcgis.com","blocks":"","allows":"","llmsTxt":false},{"rank":898,"domain":"nbcnews.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer FacebookBot Amazonbot CCBot Bytespider MistralAI-User YouBot cohere-ai Diffbot Brightbot 1.0 ExaBot ExaSearchBot NagetBot omgili omgilibot QuillBot quillbot.com Scrapy ShapBot","allows":"","llmsTxt":false},{"rank":900,"domain":"worldbank.org","blocks":"","allows":"","llmsTxt":false},{"rank":902,"domain":"auth0.com","blocks":"","allows":"","llmsTxt":true},{"rank":903,"domain":"pornhub.com","blocks":"","allows":"","llmsTxt":false},{"rank":904,"domain":"daum.net","blocks":"GoogleOther","allows":"","llmsTxt":false},{"rank":906,"domain":"hbr.org","blocks":"GPTBot Google-Extended CCBot","allows":"","llmsTxt":false},{"rank":907,"domain":"figma.com","blocks":"GPTBot ClaudeBot Google-Extended FacebookBot CCBot Bytespider cohere-ai amazon-kendra omgili omgilibot","allows":"OAI-SearchBot ChatGPT-User Claude-SearchBot Claude-User PerplexityBot","llmsTxt":false},{"rank":909,"domain":"wyzecam.com","blocks":"","allows":"GPTBot OAI-SearchBot ClaudeBot PerplexityBot Google-Extended Meta-ExternalAgent CCBot","llmsTxt":true},{"rank":910,"domain":"princeton.edu","blocks":"","allows":"","llmsTxt":false},{"rank":911,"domain":"poki.com","blocks":"","allows":"","llmsTxt":false},{"rank":913,"domain":"fwmrm.net","blocks":"","allows":"","llmsTxt":true},{"rank":914,"domain":"bankofamerica.com","blocks":"","allows":"","llmsTxt":false},{"rank":915,"domain":"singular.net","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot anthropic-ai PerplexityBot Google-Extended Applebot-Extended Amazonbot CCBot Bytespider","llmsTxt":true},{"rank":916,"domain":"disqus.com","blocks":"","allows":"","llmsTxt":false},{"rank":917,"domain":"epa.gov","blocks":"","allows":"","llmsTxt":false},{"rank":918,"domain":"lg.com","blocks":"","allows":"","llmsTxt":true},{"rank":919,"domain":"latimes.com","blocks":"GPTBot ClaudeBot Bytespider Diffbot ImagesiftBot omgili omgilibot","allows":"","llmsTxt":false},{"rank":920,"domain":"networkadvertising.org","blocks":"","allows":"","llmsTxt":false},{"rank":926,"domain":"mercadolivre.com.br","blocks":"GPTBot ChatGPT-User ClaudeBot Claude-User PerplexityBot Perplexity-User Amazonbot","allows":"FacebookBot","llmsTxt":false},{"rank":928,"domain":"qualtrics.com","blocks":"","allows":"","llmsTxt":false},{"rank":929,"domain":"seedtag.com","blocks":"","allows":"GPTBot ChatGPT-User ClaudeBot PerplexityBot Bytespider","llmsTxt":true},{"rank":935,"domain":"classlink.com","blocks":"","allows":"","llmsTxt":true},{"rank":936,"domain":"patreon.com","blocks":"PerplexityBot Google-CloudVertexBot FacebookBot Timpibot","allows":"","llmsTxt":false},{"rank":940,"domain":"genius.com","blocks":"GPTBot ChatGPT-User ClaudeBot anthropic-ai Google-Extended Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot CCBot Bytespider YouBot cohere-ai Diffbot Timpibot ImagesiftBot omgili omgilibot Webzio-Extended","allows":"","llmsTxt":false},{"rank":941,"domain":"tailscale.com","blocks":"","allows":"","llmsTxt":true},{"rank":942,"domain":"jetbrains.com","blocks":"","allows":"","llmsTxt":false},{"rank":944,"domain":"note.com","blocks":"","allows":"","llmsTxt":false},{"rank":946,"domain":"state.gov","blocks":"","allows":"","llmsTxt":false},{"rank":950,"domain":"odoo.com","blocks":"","allows":"","llmsTxt":false},{"rank":951,"domain":"xiaohongshu.com","blocks":"","allows":"","llmsTxt":false},{"rank":963,"domain":"chaturbate.com","blocks":"","allows":"","llmsTxt":false},{"rank":964,"domain":"mercadolibre.com.ar","blocks":"GPTBot ChatGPT-User ClaudeBot Claude-User PerplexityBot Perplexity-User Amazonbot","allows":"FacebookBot","llmsTxt":false},{"rank":967,"domain":"amazon.com.au","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User Claude-Web PerplexityBot Perplexity-User Google-Extended GoogleOther Google-CloudVertexBot Meta-ExternalAgent Meta-ExternalFetcher meta-webindexer CCBot Bytespider MistralAI-User DuckAssistBot YouBot cohere-ai cohere-training-data-crawler DeepSeekBot GrokBot AI2Bot Diffbot Timpibot AddSearchBot AI2Bot-DeepResearchEval Ai2Bot-Dolma aiHitBot Andibot Anomura atlassian-bot Brightbot BuddyBot Channel3Bot ChatGLM-Spider Cloudflare-AutoRAG CloudflareBrowserRenderingCrawler Cotoyogi Crawl4AI Datenbank Crawler Devin Echobot Bot EchoboxBot Factset_spyderbot FriendlyCrawler Gemini-Deep-Research Google-Firebase Google-NotebookLM GoogleAgent-Mariner iAskBot iaskspider ICC-Crawler ImagesiftBot imageSpider img2dataset ISSCyberRiskCrawler Kangaroo Bot KlaviyoAIBot KunatoCrawler laion-huggingface-processor Linguee Bot LinkupBot Manus-User netEstate Imprint Crawler omgili PanguBot Panscient panscient.com PhindBot Poggio-Citations QualifiedBot SBIntuitionsBot Scrapy SemrushBot-SWA ShapBot Sidetrade indexer bot TavilyBot TerraCotta Thinkbot TwinAgent VelenPublicWebCrawler Webzio-Extended wpbot WRTNBot xAI-Grok YaK ZanistaBot","allows":"","llmsTxt":false},{"rank":968,"domain":"otto.de","blocks":"","allows":"GPTBot OAI-SearchBot ChatGPT-User PerplexityBot Perplexity-User","llmsTxt":true},{"rank":969,"domain":"washington.edu","blocks":"","allows":"","llmsTxt":false},{"rank":971,"domain":"docker.io","blocks":"","allows":"","llmsTxt":false},{"rank":972,"domain":"corriere.it","blocks":"GPTBot ChatGPT-User ClaudeBot anthropic-ai Claude-Web Google-Extended Google-CloudVertexBot Applebot-Extended Meta-ExternalAgent Amazonbot CCBot Bytespider YouBot Timpibot Diffbot-User Scrapy","allows":"OAI-SearchBot PerplexityBot Amzn-SearchBot","llmsTxt":false},{"rank":973,"domain":"nexusmods.com","blocks":"","allows":"","llmsTxt":false},{"rank":974,"domain":"eporner.com","blocks":"","allows":"","llmsTxt":false},{"rank":976,"domain":"ancestry.com","blocks":"Google-Extended","allows":"","llmsTxt":false},{"rank":980,"domain":"onet.pl","blocks":"Meta-ExternalAgent meta-webindexer Amazonbot Amzn-SearchBot Amzn-User Bytespider YouBot cohere-ai Diffbot Timpibot Ai2Bot-Dolma Diffbot-User omgilibot","allows":"","llmsTxt":false},{"rank":984,"domain":"app-ads-services.com","blocks":"","allows":"","llmsTxt":false},{"rank":985,"domain":"trendyol.com","blocks":"","allows":"","llmsTxt":true},{"rank":986,"domain":"spiegel.de","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web Applebot-Extended Meta-ExternalAgent FacebookBot CCBot Bytespider MistralAI-User YouBot cohere-ai cohere-training-data-crawler DeepSeekBot AI2Bot Diffbot Timpibot bigsur.ai CloudVertexBot Devin iaskspider ImagesiftBot Kangaroo Bot LCC LinerBot omgili omgilibot PanguBot Scrapy SemrushBot-SWA Webzio-Extended","allows":"","llmsTxt":false},{"rank":987,"domain":"change.org","blocks":"","allows":"Google-CloudVertexBot","llmsTxt":false},{"rank":988,"domain":"mailchi.mp","blocks":"","allows":"","llmsTxt":false},{"rank":989,"domain":"att.com","blocks":"","allows":"","llmsTxt":true},{"rank":991,"domain":"abovedomains.com","blocks":"","allows":"","llmsTxt":false},{"rank":992,"domain":"xnxx.com","blocks":"","allows":"","llmsTxt":false},{"rank":993,"domain":"instructure.com","blocks":"","allows":"","llmsTxt":true},{"rank":994,"domain":"jd.com","blocks":"","allows":"","llmsTxt":false},{"rank":995,"domain":"jamfcloud.com","blocks":"","allows":"","llmsTxt":false},{"rank":998,"domain":"huffpost.com","blocks":"GPTBot OAI-SearchBot ChatGPT-User ClaudeBot Claude-SearchBot Claude-User anthropic-ai Claude-Web PerplexityBot Perplexity-User Google-Extended Applebot-Extended Meta-ExternalAgent Meta-ExternalFetcher FacebookBot Amazonbot CCBot Bytespider DuckAssistBot cohere-ai Diffbot Timpibot Ai2Bot-Dolma omgilibot","allows":"","llmsTxt":false},{"rank":1000,"domain":"umich.edu","blocks":"","allows":"","llmsTxt":false}]}