{"about":"What well-known websites tell AI crawlers in robots.txt. robots.txt only; rechecked about weekly.","license":"CC BY 4.0","source":"https://aipaygen.com/ai-crawlers/sites","crawlers_tested":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"sites_read":347,"sites_unreadable":53,"block_at_least_one":121,"sites":[{"site":"aa.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"abcnews.go.com","kind":"News","blocked":["GPTBot","ChatGPT-User","anthropic-ai","Google-Extended","CCBot","Bytespider","FacebookBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"academia.edu","kind":"Education","blocked":["GPTBot","ClaudeBot","anthropic-ai","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","meta-externalagent","FacebookBot","cohere-ai","Diffbot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"admin.ch","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"adobe.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"agoda.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"ai21.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"airbnb.com","kind":"Travel","blocked":["GPTBot","ClaudeBot","Applebot-Extended","ImagesiftBot","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"airtable.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"aliexpress.com","kind":"Shopping","blocked":["GPTBot","ClaudeBot","Google-Extended","CCBot","Bytespider","Amazonbot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"aljazeera.com","kind":"News","blocked":["GPTBot","ChatGPT-User","ClaudeBot","anthropic-ai","PerplexityBot","Bytespider","cohere-ai"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"allrecipes.com","kind":"Reference","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"amazon.com","kind":"Shopping","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","meta-externalagent","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"amplitude.com","kind":"SaaS and business software","blocked":["Bytespider"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"amtrak.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"angular.dev","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"anthropic.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"apache.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"apnews.com","kind":"News","blocked":["GPTBot","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","CCBot","Amazonbot","Applebot-Extended","cohere-ai","Timpibot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"apple.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"archive.org","kind":"Reference","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"arstechnica.com","kind":"News","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"arxiv.org","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"asana.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"atlassian.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"auth0.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"axios.com","kind":"News","blocked":["CCBot","Bytespider","Amazonbot","FacebookBot","Diffbot","ImagesiftBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"bankofamerica.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"bankrate.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"bbc.com","kind":"News","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","cohere-ai","Diffbot","YouBot","DuckAssistBot","PetalBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"berkeley.edu","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"bhphotovideo.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"billboard.com","kind":"Entertainment and media","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","anthropic-ai","PerplexityBot","Google-Extended","CCBot","Bytespider","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","Timpibot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"binance.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"bitbucket.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"bloomberg.com","kind":"News","blocked":["GPTBot","ClaudeBot","anthropic-ai","PerplexityBot","Google-Extended","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","cohere-ai","Diffbot","YouBot","DuckAssistBot","PetalBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"booking.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"box.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"brilliant.org","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"britannica.com","kind":"Reference","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"bsky.app","kind":"Social and community","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"bundesregierung.de","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"businessinsider.com","kind":"News","blocked":["ClaudeBot","anthropic-ai","CCBot","Bytespider","Diffbot","Timpibot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"calendly.com","kind":"SaaS and business software","blocked":["CCBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"cam.ac.uk","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"canada.ca","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"canva.com","kind":"SaaS and business software","blocked":["GPTBot","ClaudeBot","anthropic-ai","CCBot","Bytespider","Applebot-Extended","meta-externalagent","FacebookBot","Diffbot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"capitalone.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"cbsnews.com","kind":"News","blocked":["GPTBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"cdc.gov","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"census.gov","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"character.ai","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"chase.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"chegg.com","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"chewy.com","kind":"Shopping","blocked":["Amazonbot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"citi.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"clickup.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"cloudflare.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"cnbc.com","kind":"Finance","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"cnn.com","kind":"News","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"codecademy.com","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"cohere.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"coinbase.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"columbia.edu","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"congress.gov","kind":"Government and public bodies","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"cornell.edu","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"costco.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"coursera.org","kind":"Education","blocked":["meta-externalagent"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"crates.io","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"creditkarma.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"crunchyroll.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"cursor.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"databricks.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"datadoghq.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"deepl.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"deepmind.google","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"dev.to","kind":"Social and community","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"developer.mozilla.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"deviantart.com","kind":"Social and community","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"dictionary.com","kind":"Reference","blocked":["GPTBot","CCBot","Applebot-Extended"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"digitalocean.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"discord.com","kind":"Social and community","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"disney.com","kind":"Entertainment and media","blocked":["GPTBot","Google-Extended"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"djangoproject.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"docker.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"docusign.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"dropbox.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"duolingo.com","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"dw.com","kind":"News","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","anthropic-ai","PerplexityBot","Google-Extended","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"ebay.com","kind":"Shopping","blocked":["GPTBot","ClaudeBot","anthropic-ai","PerplexityBot","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"economist.com","kind":"News","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Applebot-Extended","PetalBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"edx.org","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"elevenlabs.io","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"elpais.com","kind":"News","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","PerplexityBot","Perplexity-User","CCBot","Bytespider","Amazonbot","MistralAI-User","DuckAssistBot","PetalBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"engadget.com","kind":"News","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"espn.com","kind":"Entertainment and media","blocked":["GPTBot","ChatGPT-User","anthropic-ai","Google-Extended","CCBot","Bytespider","FacebookBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"etsy.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"europa.eu","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"expedia.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"facebook.com","kind":"Social and community","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"fda.gov","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"fidelity.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"fifa.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"figma.com","kind":"SaaS and business software","blocked":["GPTBot","ClaudeBot","Google-Extended","CCBot","Bytespider","FacebookBot","cohere-ai","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"flickr.com","kind":"Social and community","blocked":["GPTBot","ClaudeBot","anthropic-ai","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"flipkart.com","kind":"Shopping","blocked":["Amazonbot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"forbes.com","kind":"News","blocked":["GPTBot","ClaudeBot","anthropic-ai","PerplexityBot","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"foxnews.com","kind":"News","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"freecodecamp.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"freshworks.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"ft.com","kind":"News","blocked":["ClaudeBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"ftc.gov","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"gamespot.com","kind":"Entertainment and media","blocked":["GPTBot","ClaudeBot","anthropic-ai","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","cohere-ai","Diffbot","PetalBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"geeksforgeeks.org","kind":"Developer tools","blocked":["anthropic-ai","CCBot","Bytespider","cohere-ai"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"github.com","kind":"Developer tools","blocked":["Bytespider"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"gitlab.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"glassdoor.com","kind":"Reference","blocked":["GPTBot","ClaudeBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","FacebookBot","cohere-ai","Diffbot","YouBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"go.dev","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"goodreads.com","kind":"Reference","blocked":["GPTBot","CCBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"google.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"gouvernement.fr","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"gov.uk","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"groq.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"harvard.edu","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"hashnode.com","kind":"Social and community","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"healthline.com","kind":"Reference","blocked":["GPTBot","ClaudeBot","anthropic-ai","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","Diffbot","Timpibot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"heroku.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"hilton.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"hollywoodreporter.com","kind":"Entertainment and media","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","anthropic-ai","PerplexityBot","Google-Extended","CCBot","Bytespider","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","Timpibot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"homedepot.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"howstuffworks.com","kind":"Reference","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"hubspot.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"huffpost.com","kind":"News","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","DuckAssistBot","Timpibot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"huggingface.co","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"hulu.com","kind":"Entertainment and media","blocked":["anthropic-ai","Perplexity-User","GoogleOther","FacebookBot","cohere-ai","Diffbot","MistralAI-User","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"ibm.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"ign.com","kind":"Entertainment and media","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"ikea.com","kind":"Shopping","blocked":["Amazonbot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"imdb.com","kind":"Reference","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"imf.org","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"independent.co.uk","kind":"News","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"instagram.com","kind":"Social and community","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"intercom.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"intuit.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"investopedia.com","kind":"Reference","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"irs.gov","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"jasper.ai","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"jetbrains.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"jstor.org","kind":"Education","blocked":["GPTBot","ClaudeBot","Google-Extended","CCBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"kayak.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"kotaku.com","kind":"Entertainment and media","blocked":["CCBot","Bytespider","meta-externalagent","cohere-ai","Diffbot","Timpibot","ImagesiftBot","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"kraken.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"kubernetes.io","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"langchain.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"laravel.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"latimes.com","kind":"News","blocked":["GPTBot","ClaudeBot","Bytespider","Diffbot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"lemonde.fr","kind":"News","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"letterboxd.com","kind":"Social and community","blocked":["GPTBot","ClaudeBot","Google-Extended","CCBot","Bytespider","Applebot-Extended","meta-externalagent","FacebookBot","Diffbot","PetalBot","Timpibot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"linear.app","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"linkedin.com","kind":"Social and community","blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-User","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"llamaindex.ai","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"lobste.rs","kind":"Social and community","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"loc.gov","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"lonelyplanet.com","kind":"Travel","blocked":["GPTBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"loom.com","kind":"SaaS and business software","blocked":["GPTBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"lufthansa.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"lyft.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"mailchimp.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"marketwatch.com","kind":"Finance","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Amazonbot","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"marriott.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"mastodon.social","kind":"Social and community","blocked":["GPTBot","ClaudeBot","anthropic-ai","Google-Extended","CCBot","Bytespider","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","ImagesiftBot","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"max.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"medium.com","kind":"Social and community","blocked":["GPTBot","ClaudeBot","GoogleOther","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":true,"checked":"2026-11-16"},{"site":"meetup.com","kind":"Social and community","blocked":["GPTBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"mercadolibre.com","kind":"Shopping","blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-User","PerplexityBot","Perplexity-User","Amazonbot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"merriam-webster.com","kind":"Reference","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"metacritic.com","kind":"Entertainment and media","blocked":["GPTBot","OAI-SearchBot","CCBot","meta-externalagent","ImagesiftBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"microsoft.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"miro.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"mistral.ai","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"mit.edu","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"mixpanel.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"mlb.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"monday.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"mongodb.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"morningstar.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"namecheap.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nasa.gov","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nature.com","kind":"Education","blocked":["GPTBot","ChatGPT-User","ClaudeBot","anthropic-ai","PerplexityBot","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nba.com","kind":"Entertainment and media","blocked":["GPTBot","ChatGPT-User","ClaudeBot","anthropic-ai","PerplexityBot","Google-Extended","CCBot","Amazonbot","Applebot-Extended"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nbcnews.com","kind":"News","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","PetalBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nerdwallet.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"netflix.com","kind":"Entertainment and media","blocked":["anthropic-ai","Perplexity-User","GoogleOther","CCBot","Bytespider","Amazonbot","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"netlify.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"newegg.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"news.ycombinator.com","kind":"Social and community","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"newyorker.com","kind":"News","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nextdoor.com","kind":"Social and community","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nfl.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nginx.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nhs.uk","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nike.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"noaa.gov","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nodejs.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nordstrom.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"notion.so","kind":"AI companies","blocked":["Amazonbot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nyse.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"nytimes.com","kind":"News","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","DuckAssistBot","Timpibot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"oecd.org","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"okta.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"openai.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"packagist.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"paramountplus.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"paypal.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"peacocktv.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"perplexity.ai","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"pinecone.io","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"pinterest.com","kind":"Social and community","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","meta-externalagent","cohere-ai","Diffbot","YouBot","MistralAI-User","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"pitchfork.com","kind":"Entertainment and media","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"playstation.com","kind":"Entertainment and media","blocked":["PetalBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"poe.com","kind":"AI companies","blocked":["OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"polygon.com","kind":"Entertainment and media","blocked":["GPTBot","ChatGPT-User","ClaudeBot","anthropic-ai","PerplexityBot","Google-Extended","CCBot","Bytespider","Applebot-Extended","FacebookBot","Diffbot","YouBot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"postman.com","kind":"Developer tools","blocked":["Bytespider","Diffbot","Timpibot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"priceline.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"primevideo.com","kind":"Entertainment and media","blocked":["ChatGPT-User","Claude-User","PerplexityBot","Perplexity-User","GoogleOther","CCBot","Bytespider","meta-externalagent","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"princeton.edu","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"producthunt.com","kind":"Social and community","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"pypi.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"python.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"quora.com","kind":"Reference","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","PerplexityBot","Bytespider","Applebot-Extended","Diffbot","PetalBot","ImagesiftBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"rakuten.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"react.dev","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"reddit.com","kind":"Social and community","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"replicate.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"replit.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"researchgate.net","kind":"Education","blocked":["GoogleOther","Applebot-Extended"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"reuters.com","kind":"News","blocked":["GPTBot","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"revolut.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"rippling.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"robinhood.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"rollingstone.com","kind":"Entertainment and media","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","anthropic-ai","PerplexityBot","Google-Extended","CCBot","Bytespider","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","Timpibot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"rome2rio.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"rottentomatoes.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"rubygems.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"rubyonrails.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"runwayml.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"rust-lang.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"salesforce.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"sap.com","kind":"SaaS and business software","blocked":["anthropic-ai","Bytespider","Applebot-Extended","meta-externalagent","cohere-ai","Diffbot","YouBot","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"scale.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"science.org","kind":"Education","blocked":["ChatGPT-User","anthropic-ai","Google-Extended","CCBot","Applebot-Extended","meta-externalagent"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"sec.gov","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"seekingalpha.com","kind":"Finance","blocked":["GPTBot","ClaudeBot","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"segment.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"sentry.io","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"shein.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"shopify.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"shopify.dev","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"skyscanner.net","kind":"Travel","blocked":["meta-externalagent"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"slack.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"snapchat.com","kind":"Social and community","blocked":["GPTBot","ClaudeBot","anthropic-ai","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"snopes.com","kind":"Reference","blocked":["GPTBot","ChatGPT-User","ClaudeBot","anthropic-ai","PerplexityBot","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"snowflake.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"sofi.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"soundcloud.com","kind":"Social and community","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"southwest.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"spiegel.de","kind":"News","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","CCBot","Bytespider","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"spotify.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"springer.com","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"squarespace.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"stability.ai","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"stanford.edu","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"steampowered.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"strava.com","kind":"Social and community","blocked":["GPTBot","ClaudeBot","Google-Extended","meta-externalagent"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"stripe.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"substack.com","kind":"Social and community","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"supabase.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"target.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"techcrunch.com","kind":"News","blocked":["GPTBot","ClaudeBot","anthropic-ai","CCBot","Bytespider","Applebot-Extended","FacebookBot","cohere-ai","Diffbot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"ted.com","kind":"Education","blocked":["GPTBot","ClaudeBot","anthropic-ai","Google-Extended","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","Timpibot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"telegraph.co.uk","kind":"News","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"temu.com","kind":"Shopping","blocked":["Bytespider"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"theatlantic.com","kind":"News","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Amazonbot","meta-externalagent","cohere-ai","Diffbot","MistralAI-User","PetalBot","Timpibot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"theguardian.com","kind":"News","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","YouBot","DuckAssistBot","PetalBot","ImagesiftBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":true,"checked":"2026-11-16"},{"site":"theverge.com","kind":"News","blocked":["ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"threads.net","kind":"Social and community","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"tiktok.com","kind":"Social and community","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Applebot-Extended","meta-externalagent","MistralAI-User","DuckAssistBot","PetalBot","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"time.com","kind":"News","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"timeanddate.com","kind":"Reference","blocked":["PerplexityBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"together.ai","kind":"AI companies","blocked":["Google-Extended"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"tradingview.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"trainline.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"trello.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"tripadvisor.com","kind":"Reference","blocked":["GPTBot","ClaudeBot","Google-Extended","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","cohere-ai","YouBot","PetalBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"trivago.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"tumblr.com","kind":"Social and community","blocked":["ClaudeBot","anthropic-ai","Google-Extended","CCBot","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","YouBot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"twilio.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"twitch.tv","kind":"Social and community","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"typeform.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"uber.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"udacity.com","kind":"Education","blocked":["Applebot-Extended"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"un.org","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"usa.gov","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"usatoday.com","kind":"News","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"vanguard.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"variety.com","kind":"Entertainment and media","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","anthropic-ai","PerplexityBot","Google-Extended","CCBot","Bytespider","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","Timpibot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"vercel.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"vimeo.com","kind":"Social and community","blocked":["CCBot","Bytespider","FacebookBot","Diffbot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"visa.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"vox.com","kind":"News","blocked":["ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"vrbo.com","kind":"Travel","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"vuejs.org","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"w3schools.com","kind":"Developer tools","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"walmart.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"washingtonpost.com","kind":"News","blocked":["ClaudeBot","anthropic-ai","PerplexityBot","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","Diffbot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"wayfair.com","kind":"Shopping","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"weather.com","kind":"Reference","blocked":["GoogleOther","CCBot","Bytespider","Amazonbot","FacebookBot","Diffbot","YouBot","ImagesiftBot","omgili"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"weaviate.io","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"webmd.com","kind":"Reference","blocked":["GPTBot","ClaudeBot","CCBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"wellsfargo.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"whitehouse.gov","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"who.int","kind":"Reference","blocked":["CCBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"wikihow.com","kind":"Reference","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"wikipedia.org","kind":"Reference","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"wiktionary.org","kind":"Reference","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"wired.com","kind":"News","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"wise.com","kind":"Finance","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"wix.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"wordpress.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"workday.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"worldbank.org","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"worldometers.info","kind":"Reference","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"writer.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"wsj.com","kind":"News","blocked":["ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","CCBot","Bytespider","Amazonbot","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"wto.org","kind":"Government and public bodies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"x.ai","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"x.com","kind":"Social and community","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"xbox.com","kind":"Entertainment and media","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"xero.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"yale.edu","kind":"Education","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"yelp.com","kind":"Reference","blocked":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-User","Claude-SearchBot","anthropic-ai","PerplexityBot","Perplexity-User","Google-Extended","GoogleOther","CCBot","Bytespider","Amazonbot","Applebot-Extended","meta-externalagent","FacebookBot","cohere-ai","Diffbot","YouBot","MistralAI-User","DuckAssistBot","PetalBot","Timpibot","ImagesiftBot","omgili","AI2Bot","ElevenlabsCrawler","ElevenlabsBot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"you.com","kind":"AI companies","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"youtube.com","kind":"Social and community","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"zapier.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"zendesk.com","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"zillow.com","kind":"Reference","blocked":["Amazonbot","Applebot-Extended","meta-externalagent","YouBot","AI2Bot"],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"},{"site":"zoom.us","kind":"SaaS and business software","blocked":[],"charges_ai_crawlers":false,"publishes_ai_terms":false,"rsl_license":false,"checked":"2026-11-16"}]}