{"version":"2026-07-03","name":"Visa Atlas crawler and AI retrieval access audit","description":"Public audit of the robots policy, crawler-facing discovery surfaces, JSON headers and local log commands that let search and AI retrievers access Visa Atlas.","datasetUrl":"https://visaatlas.org/api/public/crawler-access","humanUrl":"https://visaatlas.org/research/crawler-ai-retrieval-access-audit-2026","robotsTxtUrl":"https://visaatlas.org/robots.txt","sitemapUrl":"https://visaatlas.org/sitemap.xml","license":"CC BY 4.0","attribution":"Visa Atlas","sourceDatasets":["/api/public/visibility-metrics","/api/public/indexable-pages","/api/public/search-index"],"methodology":{"robotsSourceOfTruth":"src/lib/crawler-access-audit.ts exports the allow and deny rule constants consumed by src/app/robots.ts; this feed, the research report and generated robots.txt share the same policy list.","visibilityMetricsVersion":"2026-07-03","liveProofPolicy":"This dataset proves access intent and discovery wiring. It does not prove that a crawler fetched the site; live proof requires server access logs or Search Console evidence.","privateDataPolicy":"Raw server logs, IP addresses, Search Console exports, model transcripts and unpublished referral analytics remain local and are not exposed by this public endpoint.","identityProofPolicy":"User-agent strings alone are not proof. Where official IP ranges or DNS verification methods exist, local WAF allow rules and verified-fetch claims must match the user-agent to those official identity sources before publishing aggregate evidence.","commands":["npm run audit:visibility-metrics","npm run audit:data-front-door","npm run audit:data-citation","npm run audit:indexable-page-manifest","npm run audit:rendered-seo","npm run audit:crawler-access","npm run audit:botlog","npm run seo:botlog -- logs.txt --json"],"caveats":["Allowed in robots.txt is not a ranking, indexing, citation or traffic guarantee.","Some AI systems use search-engine crawlers or robots tokens rather than a distinct user-agent in server logs.","User-agent strings can be spoofed, so verified-fetch claims need official IP range or reverse/forward DNS checks where the crawler publisher provides them.","Public JSON endpoints emit noindex, follow so dataset links remain discoverable without competing with canonical human pages in search results."]},"summary":{"policyRowCount":25,"explicitAllowCount":13,"inheritedAllowCount":4,"deniedCrawlerCount":8,"allowedCrawlerCount":17,"technicalCheckCount":8,"discoverySurfaceCount":29,"identityVerificationSourceCount":7,"officialIpRangeSourceCount":5},"crawlerPolicies":[{"id":"openai-gptbot","family":"OpenAI","userAgent":"GPTBot","category":"ai-crawler","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicitly allowed for OpenAI crawler access; server logs are still required to prove fetches."},{"id":"openai-oai-searchbot","family":"OpenAI","userAgent":"OAI-SearchBot","category":"ai-retrieval","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicitly allowed for ChatGPT search retrieval; citation presence is measured separately from access."},{"id":"openai-chatgpt-user","family":"OpenAI","userAgent":"ChatGPT-User","category":"ai-retrieval","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicitly allowed for user-triggered ChatGPT browsing and retrieval requests."},{"id":"anthropic-claudebot","family":"Anthropic","userAgent":"ClaudeBot","category":"ai-crawler","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicitly allowed for Anthropic crawler access."},{"id":"anthropic-claude-web","family":"Anthropic","userAgent":"Claude-Web","category":"ai-retrieval","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicitly allowed for Claude web retrieval."},{"id":"anthropic-ai","family":"Anthropic","userAgent":"anthropic-ai","category":"ai-crawler","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicitly allowed for Anthropic AI crawler access."},{"id":"perplexitybot","family":"Perplexity","userAgent":"PerplexityBot","category":"ai-crawler","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicitly allowed for Perplexity crawler access."},{"id":"perplexity-user","family":"Perplexity","userAgent":"Perplexity-User","category":"ai-retrieval","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicitly allowed for user-triggered Perplexity retrieval requests."},{"id":"google-extended","family":"Google","userAgent":"Google-Extended","category":"ai-training-token","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicit robots token allowance; live Gemini or Googlebot fetches must be read from server logs and Search Console."},{"id":"applebot-extended","family":"Apple","userAgent":"Applebot-Extended","category":"ai-training-token","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicit robots token allowance for Apple extended crawler controls."},{"id":"common-crawl-ccbot","family":"Common Crawl","userAgent":"CCBot","category":"common-crawl","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicitly allowed because open dataset reuse and downstream AI retrieval are part of the public data strategy."},{"id":"meta-externalagent","family":"Meta","userAgent":"Meta-ExternalAgent","category":"ai-crawler","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicitly allowed for Meta external agent access."},{"id":"amazonbot","family":"Amazon","userAgent":"Amazonbot","category":"ai-crawler","accessPolicy":"explicit-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Explicitly allowed for Amazon crawler access."},{"id":"googlebot","family":"Google","userAgent":"Googlebot","category":"search","accessPolicy":"inherited-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/sitemap.xml","/api/public/indexable-pages","/api/public/search-index"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Allowed by the wildcard rule and discovered through the sitemap, indexable page manifest and rendered HTML."},{"id":"bingbot","family":"Microsoft Bing","userAgent":"bingbot","category":"search","accessPolicy":"inherited-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/sitemap.xml","/api/public/indexable-pages","/api/public/search-index"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Allowed by the wildcard rule and discovered through the same sitemap and page-manifest surfaces as Googlebot."},{"id":"applebot","family":"Apple","userAgent":"Applebot","category":"search","accessPolicy":"inherited-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/sitemap.xml","/api/public/indexable-pages","/api/public/search-index"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Allowed by the wildcard rule; Applebot-Extended is also explicitly allowed."},{"id":"claude-searchbot","family":"Anthropic","userAgent":"Claude-SearchBot","category":"ai-retrieval","accessPolicy":"inherited-allow","robotsDirective":"allow","robotsPath":"/","evidenceSurfaces":["/robots.txt","/api/public/crawler-access","/research/crawler-ai-retrieval-access-audit-2026","/ai-agents","/llms.txt","/llms-full.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Allowed by the wildcard rule; monitored by the bot-log parser as an Anthropic retrieval user agent."},{"id":"semrushbot","family":"Competitive SEO crawler","userAgent":"SemrushBot","category":"competitive-tool","accessPolicy":"disallow","robotsDirective":"disallow","robotsPath":"/","evidenceSurfaces":["/robots.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Disallowed because the access strategy welcomes search and AI retrieval while blocking aggressive competitive scraping."},{"id":"ahrefsbot","family":"Competitive SEO crawler","userAgent":"AhrefsBot","category":"competitive-tool","accessPolicy":"disallow","robotsDirective":"disallow","robotsPath":"/","evidenceSurfaces":["/robots.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Disallowed because the access strategy welcomes search and AI retrieval while blocking aggressive competitive scraping."},{"id":"mj12bot","family":"Competitive SEO crawler","userAgent":"MJ12bot","category":"competitive-tool","accessPolicy":"disallow","robotsDirective":"disallow","robotsPath":"/","evidenceSurfaces":["/robots.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Disallowed because the access strategy welcomes search and AI retrieval while blocking aggressive competitive scraping."},{"id":"dotbot","family":"Competitive SEO crawler","userAgent":"DotBot","category":"competitive-tool","accessPolicy":"disallow","robotsDirective":"disallow","robotsPath":"/","evidenceSurfaces":["/robots.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Disallowed because the access strategy welcomes search and AI retrieval while blocking aggressive competitive scraping."},{"id":"dataforseobot","family":"Competitive SEO crawler","userAgent":"DataForSeoBot","category":"competitive-tool","accessPolicy":"disallow","robotsDirective":"disallow","robotsPath":"/","evidenceSurfaces":["/robots.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Disallowed because the access strategy welcomes search and AI retrieval while blocking aggressive competitive scraping."},{"id":"blexbot","family":"Competitive SEO crawler","userAgent":"BLEXBot","category":"competitive-tool","accessPolicy":"disallow","robotsDirective":"disallow","robotsPath":"/","evidenceSurfaces":["/robots.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Disallowed because the access strategy welcomes search and AI retrieval while blocking aggressive competitive scraping."},{"id":"seznambot","family":"Competitive SEO crawler","userAgent":"SeznamBot","category":"competitive-tool","accessPolicy":"disallow","robotsDirective":"disallow","robotsPath":"/","evidenceSurfaces":["/robots.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Disallowed because the access strategy welcomes search and AI retrieval while blocking aggressive competitive scraping."},{"id":"petalbot","family":"Competitive SEO crawler","userAgent":"PetalBot","category":"competitive-tool","accessPolicy":"disallow","robotsDirective":"disallow","robotsPath":"/","evidenceSurfaces":["/robots.txt"],"logCommand":"npm run seo:botlog -- logs.txt --json","note":"Disallowed because the access strategy welcomes search and AI retrieval while blocking aggressive competitive scraping."}],"technicalChecks":[{"id":"robots-policy","label":"robots.txt allows search and AI retrievers","status":"instrumented","evidenceSurfaces":["/robots.txt","src/app/robots.ts","src/lib/crawler-access-audit.ts"],"command":"npm run audit:visibility-metrics","note":"The generated robots file consumes the same allow and deny rule constants published by this dataset."},{"id":"sitemap-discovery","label":"Sitemap and page manifest expose indexable URLs","status":"instrumented","evidenceSurfaces":["/sitemap.xml","/api/public/indexable-pages","/api/public/search-index"],"command":"npm run audit:indexable-page-manifest","note":"The sitemap is advertised from robots.txt and mirrored into a JSON manifest for crawler reconciliation."},{"id":"api-follow-headers","label":"Public JSON emits X-Robots-Tag: noindex, follow","status":"instrumented","evidenceSurfaces":["/api/public","/api/openapi.json","next.config.ts"],"command":"npm run audit:data-front-door","note":"API routes stay out of search result pages while still allowing crawlers to follow discovery links."},{"id":"public-api-link-relations","label":"Public API responses advertise discovery Link headers","status":"instrumented","evidenceSurfaces":["/api","/api/openapi.json","/data","/use-our-data","/ai-agents","/llms.txt"],"command":"npm run audit:data-front-door","note":"Link headers connect JSON feeds to human docs, OpenAPI, license guidance and LLM indexes."},{"id":"data-page-alternate-json","label":"Human research and data pages expose alternate JSON headers","status":"instrumented","evidenceSurfaces":["/research/crawler-ai-retrieval-access-audit-2026","/api/public/crawler-access","next.config.ts"],"command":"npm run audit:data-citation","note":"The report page has an alternate JSON endpoint for crawlers and data consumers."},{"id":"llm-indexes","label":"LLM indexes point to crawler access evidence","status":"instrumented","evidenceSurfaces":["/llms.txt","/llms-full.txt","/ai-agents"],"command":"npm run audit:data-front-door","note":"Plain-text retrieval indexes include the report and JSON endpoint alongside the wider open-data graph."},{"id":"server-log-proof","label":"Live fetch evidence remains a local server-log workflow","status":"instrumented","evidenceSurfaces":["/visibility-metrics","/api/public/visibility-metrics"],"command":"npm run seo:botlog -- logs.txt --json","note":"GA4 cannot prove crawler fetches; server logs are required for live bot-hit evidence and are not published by this endpoint. The parser is regression-tested with npm run audit:botlog."},{"id":"identity-and-waf-verification","label":"Crawler identity and WAF rules require official IP or DNS checks","status":"instrumented","evidenceSurfaces":["/api/public/crawler-access","https://openai.com/gptbot.json","https://openai.com/searchbot.json","https://openai.com/chatgpt-user.json","https://www.perplexity.ai/perplexitybot.json","https://developers.google.com/static/crawling/ipranges/common-crawlers.json"],"command":"npm run audit:crawler-access","note":"User-agent strings can be spoofed. WAF allow rules and verified-fetch claims must pair the user-agent with official IP range JSON or Google reverse/forward DNS evidence where available."}],"identityVerificationSources":[{"id":"openai-gptbot-ip-ranges","crawlerFamily":"OpenAI","userAgents":["GPTBot"],"verificationMethod":"ip-range-json","officialDocsUrl":"https://platform.openai.com/docs/bots","ipRangeUrls":["https://openai.com/gptbot.json"],"wafAction":"If a CDN/WAF rule blocks AI crawlers, allow GPTBot only when the user-agent matches and the source IP is present in the official OpenAI GPTBot range file.","proofRequired":"Server-log proof must preserve timestamp, normalized path, user-agent, status code and a source IP matched against https://openai.com/gptbot.json before the hit is treated as verified OpenAI crawler access.","caveat":"The JSON range file is an identity aid, not evidence of indexing, ranking, answer use or training inclusion."},{"id":"openai-searchbot-ip-ranges","crawlerFamily":"OpenAI","userAgents":["OAI-SearchBot"],"verificationMethod":"ip-range-json","officialDocsUrl":"https://platform.openai.com/docs/bots","ipRangeUrls":["https://openai.com/searchbot.json"],"wafAction":"If a CDN/WAF rule blocks ChatGPT search retrieval, allow OAI-SearchBot only when the user-agent matches and the source IP is present in the official OpenAI search bot range file.","proofRequired":"Server-log proof must preserve timestamp, normalized path, user-agent, status code and a source IP matched against https://openai.com/searchbot.json before the hit is treated as verified ChatGPT search retrieval.","caveat":"Verified fetch access does not prove that ChatGPT cited Visa Atlas or sent human referral sessions."},{"id":"openai-chatgpt-user-ip-ranges","crawlerFamily":"OpenAI","userAgents":["ChatGPT-User"],"verificationMethod":"ip-range-json","officialDocsUrl":"https://platform.openai.com/docs/bots","ipRangeUrls":["https://openai.com/chatgpt-user.json"],"wafAction":"If a CDN/WAF rule blocks user-triggered ChatGPT browsing, allow ChatGPT-User only when the user-agent matches and the source IP is present in the official OpenAI ChatGPT-User range file.","proofRequired":"Server-log proof must preserve timestamp, normalized path, user-agent, status code and a source IP matched against https://openai.com/chatgpt-user.json before the hit is treated as verified user-triggered ChatGPT retrieval.","caveat":"User-triggered fetch evidence is separate from automated crawler access and from analytics referral attribution."},{"id":"perplexitybot-ip-ranges","crawlerFamily":"Perplexity","userAgents":["PerplexityBot"],"verificationMethod":"ip-range-json","officialDocsUrl":"https://docs.perplexity.ai/guides/bots","ipRangeUrls":["https://www.perplexity.ai/perplexitybot.json"],"wafAction":"If a CDN/WAF rule blocks Perplexity crawler access, allow PerplexityBot only when the user-agent matches and the source IP is present in the official PerplexityBot range file.","proofRequired":"Server-log proof must preserve timestamp, normalized path, user-agent, status code and a source IP matched against https://www.perplexity.ai/perplexitybot.json before the hit is treated as verified Perplexity crawler access.","caveat":"Verified PerplexityBot access is not the same thing as a Perplexity answer citation or referral session."},{"id":"googlebot-reverse-dns-ip-ranges","crawlerFamily":"Google","userAgents":["Googlebot"],"verificationMethod":"reverse-dns-and-ip-range","officialDocsUrl":"https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot","ipRangeUrls":["https://developers.google.com/static/crawling/ipranges/common-crawlers.json"],"wafAction":"If a CDN/WAF rule blocks Googlebot, allow Googlebot only after reverse DNS and forward DNS checks agree or the source IP is present in the official Google common-crawlers range file.","proofRequired":"Server-log proof must preserve timestamp, normalized path, user-agent, status code and either matching reverse/forward DNS for googlebot.com or an IP match against the official Google common crawler ranges.","caveat":"Verified Googlebot access does not prove indexing, ranking, Search Console coverage, Discover inclusion or AI Overview citation."},{"id":"google-extended-token","crawlerFamily":"Google","userAgents":["Google-Extended"],"verificationMethod":"robots-token-only","officialDocsUrl":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers","ipRangeUrls":[],"wafAction":"Keep Google-Extended as a robots token control, not a server-log bot identity; do not create WAF allow or proof rows from this token alone.","proofRequired":"The only public proof for Google-Extended here is robots.txt policy intent. Live Google fetch proof must use Googlebot, Search Console or verified Google crawler logs.","caveat":"Google-Extended is not counted as a loggable crawler hit by the local bot-log parser."},{"id":"applebot-extended-token","crawlerFamily":"Apple","userAgents":["Applebot-Extended"],"verificationMethod":"robots-token-only","officialDocsUrl":"https://support.apple.com/en-us/119829","ipRangeUrls":[],"wafAction":"Keep Applebot-Extended as a robots token control, not a server-log bot identity; do not create WAF allow or proof rows from this token alone.","proofRequired":"The only public proof for Applebot-Extended here is robots.txt policy intent. Live Apple fetch proof must come from actual Applebot log rows.","caveat":"Applebot-Extended is not counted as a loggable crawler hit by the local bot-log parser."}]}