--- title: AI Crawler Access Checker description: url: https://moxseo.com/tools/ai-crawler-access-checker date_modified: 2026-07-26 author: Admin language: en_US --- .mox-cw{--mox-bg:#fff;--mox-panel:#fff;--mox-ink:#0e0f12;--mox-ink-soft:#525866;--mox-line:#e8e8ea;--mox-primary:#ff5a1f;--mox-primary-d:#e8470a;--mox-accent:#ff5a1f;--mox-dark:#0e0f12;--mox-dark-2:#181a1f;--mox-good:#16a077;--mox-warn:#e8a20c;--mox-risk:#e34d5b;--mox-r:18px;--mox-r-sm:12px;--mox-font-d:'Sora',sans-serif;--mox-font-b:'Inter',sans-serif;--mox-font-m:'IBM Plex Mono',monospace;font-family:var(--mox-font-b);background:var(--mox-bg);color:var(--mox-ink)} .mox-cw *,.mox-cw *::before,.mox-cw *::after{box-sizing:border-box;margin:0;padding:0} .mox-cw a{text-decoration:none}.mox-cw button{font-family:inherit} .mox-cw .w{max-width:1180px;margin:0 auto;padding:0 22px} .mox-cw .sec{padding:82px 0} .mox-cw .sec--alt{background:#fafafb} .mox-cw .sec--dark{background:var(--mox-dark)} .mox-cw .sec--dark h2,.mox-cw .sec--dark h3,.mox-cw .sec--dark p,.mox-cw .sec--dark li{color:#f4f4f6} .mox-cw .eyebrow{display:inline-flex;align-items:center;gap:8px;font-family:var(--mox-font-m);font-size:12px;font-weight:600;letter-spacing:.14em;text-transform:uppercase;color:var(--mox-primary);margin-bottom:16px} .mox-cw .eyebrow::before{content:"";width:22px;height:2px;background:var(--mox-accent)} .mox-cw h1{font-family:var(--mox-font-d);font-size:clamp(32px,4.6vw,54px);font-weight:800;line-height:1.12;letter-spacing:-.02em} .mox-cw h2{font-family:var(--mox-font-d);font-size:clamp(26px,3.4vw,38px);font-weight:700;line-height:1.18;letter-spacing:-.015em;max-width:780px} .mox-cw h3{font-family:var(--mox-font-d);font-weight:700} .mox-cw .sub{font-size:17px;color:var(--mox-ink-soft);max-width:700px;margin-top:14px;line-height:1.7} .mox-cw .btn{display:inline-flex;align-items:center;justify-content:center;gap:8px;font-family:var(--mox-font-d);font-weight:700;font-size:15px;padding:15px 28px;border-radius:999px;border:2px solid transparent;cursor:pointer;transition:transform .25s,background .2s} .mox-cw .btn:hover{transform:translateY(-2px)} .mox-cw .btn--p{background:var(--mox-primary);color:#fff;box-shadow:0 10px 26px rgba(255,90,31,.32)} .mox-cw .btn--p:hover{background:var(--mox-primary-d)} .mox-cw .btn--ghost{background:transparent;border-color:var(--mox-line);color:var(--mox-ink)} .mox-cw .btn--dg{background:transparent;border-color:rgba(255,255,255,.25);color:#fff} .mox-cw .trust{background:var(--mox-dark);padding:14px 0} .mox-cw .trust__row{display:flex;flex-wrap:wrap;justify-content:center;gap:10px 28px;font-family:var(--mox-font-m);font-size:11.5px;letter-spacing:.06em;text-transform:uppercase;color:#a7acb5} .mox-cw .trust__row span{display:inline-flex;align-items:center;gap:8px} .mox-cw .trust__row span::before{content:"";width:6px;height:6px;border-radius:50%;background:var(--mox-accent)} .mox-cw .hero{padding:80px 0 70px;background:radial-gradient(900px 500px at 85% -10%,rgba(255,90,31,.12),transparent 60%),radial-gradient(700px 420px at -10% 30%,rgba(255,90,31,.08),transparent 55%),var(--mox-bg)} .mox-cw .hero__grid{display:grid;grid-template-columns:1.05fr .95fr;gap:56px;align-items:center} .mox-cw .lead{font-size:18px;color:var(--mox-ink-soft);margin:20px 0 28px;max-width:560px;line-height:1.7} .mox-cw .cta-row{display:flex;gap:14px;flex-wrap:wrap;margin-bottom:26px} .mox-cw .ticks{display:flex;flex-wrap:wrap;gap:10px 22px;font-size:14px;color:var(--mox-ink-soft)} .mox-cw .ticks span{display:inline-flex;align-items:center;gap:7px} .mox-cw .ticks span::before{content:"\2713";color:var(--mox-good);font-weight:700} .mox-cw .sample{background:var(--mox-dark);border-radius:24px;padding:30px;box-shadow:0 30px 70px rgba(14,15,18,.35);transform:rotate(1.2deg)} .mox-cw .sample__tag{font-family:var(--mox-font-m);font-size:11px;letter-spacing:.12em;text-transform:uppercase;color:var(--mox-accent);margin-bottom:18px} .mox-cw .sample__score{display:flex;align-items:baseline;gap:14px;border-bottom:1px solid rgba(255,255,255,.12);padding-bottom:18px;margin-bottom:18px} .mox-cw .sample__num{font-family:var(--mox-font-m);font-size:56px;font-weight:600;color:var(--mox-accent);line-height:1} .mox-cw .sample__lbl{font-size:14px;color:#a7acb5;max-width:160px} .mox-cw .sample__row{display:flex;justify-content:space-between;gap:16px;padding:9px 0;font-size:14px} .mox-cw .sample__row b{color:#fff;font-weight:600} .mox-cw .sample__row span{color:#85898f} .mox-cw .sample__note{margin-top:16px;font-size:12.5px;color:#85898f;border-left:3px solid var(--mox-accent);padding-left:12px;line-height:1.6} .mox-cw .g3{display:grid;grid-template-columns:repeat(3,1fr);gap:20px;margin-top:44px} .mox-cw .g2{display:grid;grid-template-columns:repeat(2,1fr);gap:20px;margin-top:44px} .mox-cw .card{background:var(--mox-panel);border:1px solid var(--mox-line);border-radius:var(--mox-r);padding:26px;transition:transform .3s,box-shadow .3s} .mox-cw .card:hover{transform:translateY(-4px);box-shadow:0 18px 40px rgba(14,15,18,.08)} .mox-cw .card h3{font-size:17px;margin:14px 0 8px} .mox-cw .card p{font-size:14.5px;color:var(--mox-ink-soft);line-height:1.65} .mox-cw .chip{display:inline-flex;font-family:var(--mox-font-m);font-size:11px;font-weight:600;letter-spacing:.08em;padding:5px 10px;border-radius:6px;background:rgba(255,90,31,.1);color:var(--mox-primary)} .mox-cw .bands{display:grid;grid-template-columns:repeat(3,1fr);gap:18px;margin-top:44px} .mox-cw .band{border-radius:var(--mox-r);padding:24px;background:var(--mox-panel);border:1px solid var(--mox-line);border-top:5px solid var(--mox-line)} .mox-cw .band h3{font-size:15.5px;margin:8px 0} .mox-cw .band p{font-size:13.5px;color:var(--mox-ink-soft);line-height:1.65} .mox-cw .band__t{font-family:var(--mox-font-m);font-weight:600;font-size:20px} .mox-cw .band--a{border-top-color:var(--mox-good)}.mox-cw .band--a .band__t{color:var(--mox-good)} .mox-cw .band--b{border-top-color:var(--mox-warn)}.mox-cw .band--b .band__t{color:var(--mox-warn)} .mox-cw .band--c{border-top-color:var(--mox-risk)}.mox-cw .band--c .band__t{color:var(--mox-risk)} .mox-cw .tool{background:var(--mox-panel);border:1px solid var(--mox-line);border-radius:24px;padding:34px;box-shadow:0 20px 60px rgba(14,15,18,.06);margin-top:38px} .mox-cw .form{display:flex;flex-wrap:wrap;gap:12px} .mox-cw .form input{flex:1 1 320px;min-width:250px;font-family:var(--mox-font-b);font-size:16px;color:var(--mox-ink);background:#fbfbfc;border:1.5px solid var(--mox-line);border-radius:999px;padding:15px 22px;outline:none;transition:border .2s,box-shadow .2s} .mox-cw .form input:focus{border-color:var(--mox-primary);box-shadow:0 0 0 4px rgba(255,90,31,.14)} .mox-cw .hint{font-size:13px;color:var(--mox-ink-soft);margin-top:12px} .mox-cw .faq{max-width:820px;margin:44px auto 0} .mox-cw .faq details{border-bottom:1px solid var(--mox-line);padding:18px 0} .mox-cw .faq summary{font-family:var(--mox-font-d);font-weight:700;font-size:16.5px;cursor:pointer;list-style:none;display:flex;justify-content:space-between;gap:16px} .mox-cw .faq summary::after{content:"+";color:var(--mox-primary);font-size:20px;line-height:1} .mox-cw .faq details[open] summary::after{content:"\2212"} .mox-cw .faq p{font-size:15px;color:var(--mox-ink-soft);line-height:1.7;margin-top:12px} .mox-cw .final{text-align:center;background:radial-gradient(700px 380px at 50% -20%,rgba(255,90,31,.16),transparent 60%),var(--mox-dark)} .mox-cw .final h2{margin:0 auto}.mox-cw .final .cta-row{justify-content:center;margin-top:30px} @media(max-width:900px){.mox-cw .hero__grid{grid-template-columns:1fr;gap:40px}.mox-cw .g3,.mox-cw .bands{grid-template-columns:1fr}.mox-cw .g2{grid-template-columns:1fr}.mox-cw .sec{padding:56px 0}.mox-cw .sample{transform:none}} 14 AI crawlers checkedReads public robots.txt onlyNo signupNothing stored Free tool # Find out which AI systems are allowed to read your site Your robots.txt decides whether ChatGPT, Claude, Perplexity, Gemini and Apple Intelligence can see your content. Most sites have never checked, and a surprising number are blocking the assistants they most want to appear in. [Check my site](#mox-run)[Check llms.txt too](https://moxseo.com/tools/llms-txt-generator/) Results in secondsShows the exact blocking ruleWorks on any public site Sample result 6/14AI crawlers fully allowed on this domain GPTBot · OpenAI**Allowed** ClaudeBot · Anthropic**Allowed** PerplexityBot · Perplexity**Partial** Google-Extended · Google**Blocked** CCBot · Common Crawl**Blocked** Google-Extended blocked by *Disallow: /* — this removes you from Gemini and AI Overviews training while leaving normal Google Search unaffected. The problem ## Your robots.txt was written for Google. AI crawlers are not Google. Most robots.txt files were set up years ago, by a developer, for one purpose: keeping search engines out of admin pages. Then a dozen AI crawlers appeared, each with its own user-agent, and almost nobody went back to check what those rules now imply. ACCIDENTAL ### Blanket rules catch everything A single *User-agent: ** with *Disallow: /* blocks every AI crawler alongside every search engine. Plugins and security suites add these silently, often during a staging-to-live migration that nobody audited afterwards. MISUNDERSTOOD ### Training and retrieval are different GPTBot trains models. OAI-SearchBot indexes for ChatGPT Search. Blocking the first does not remove you from the second, and blocking both removes you from answers entirely. Very few sites make that distinction deliberately. INVISIBLE ### Nothing tells you it happened There is no Search Console for AI visibility. If ClaudeBot has been blocked for eighteen months, no dashboard flags it, no email arrives, and your traffic reports look exactly the same as they always did. What gets checked ## Fourteen crawlers, grouped by who operates them and what they actually do ### OpenAI — GPTBot, OAI-SearchBot, ChatGPT-User GPTBot collects training data. OAI-SearchBot builds the index behind ChatGPT Search results. ChatGPT-User fetches a page live when someone pastes your link into a conversation. Blocking all three removes you from ChatGPT completely. ### Anthropic — ClaudeBot, Claude-SearchBot, Claude-User The same three-way split: training, search indexing, and live user-triggered fetches. Claude cites sources prominently in answers, so retrieval access matters more here than training access does. ### Perplexity — PerplexityBot, Perplexity-User Perplexity is answer-first and citation-heavy, which makes it one of the highest-value referral sources in AI search. Blocking PerplexityBot removes you from a channel that actually sends clicks. ### Google — Google-Extended A separate token that controls Gemini and AI Overviews training only. It does not affect Googlebot or your normal search rankings — a distinction many site owners get wrong in both directions. ### Apple — Applebot-Extended Governs whether your content trains Apple Intelligence. Applebot itself still crawls for Siri and Spotlight; the Extended token is training-only. ### Others — CCBot, Bytespider, Amazonbot, meta-externalagent Common Crawl feeds most open training datasets, so blocking CCBot has downstream effects far beyond one model. The rest cover ByteDance, Alexa and Rufus, and Meta AI respectively. The tool ## Check your robots.txt against every major AI crawler Enter any domain. We fetch its public robots.txt, resolve which rules apply to each crawler, and show you the exact directive responsible for every verdict. Check access Reads only the public robots.txt and llms.txt. No crawling, no signup, nothing stored. NEW RESEARCH ### We ran this check across 506 of the world’s top websites Training crawlers are blocked roughly twice as often as retrieval crawlers. Full findings, methodology and open dataset. [Read the study →](https://moxseo.com/ai-crawler-blocking-study/) Reading your results ## What each verdict actually means for your AI visibility Allowed ### The crawler can read everything No rule blocks this user-agent. Your content is eligible for training, retrieval and citation, depending on what that particular crawler does. Partial ### Some paths are blocked Usually harmless when it is only */wp-admin/*. Worth investigating when the disallowed paths include your blog, docs or product pages — that is where citations come from. Blocked ### Sitewide Disallow: / applies This crawler sees nothing. If it was deliberate, fine. If it was inherited from a staging config or a security plugin, you are invisible in that assistant for no reason. The honest version ## Blocking AI crawlers is a legitimate choice. Doing it by accident is not. ### Reasons to allow AI assistants are becoming a discovery channel. Perplexity and ChatGPT Search cite sources and send referral traffic. If you publish content to be found, blocking the systems people increasingly ask first works against you. Retrieval crawlers in particular tend to reward good structure rather than exploit it. ### Reasons to block If your content is your product — original research, paid courses, licensed material, a proprietary database — training crawlers extract value without attribution or payment. Publishers with licensing deals often block by default and negotiate access separately. That is a sound commercial position. The point of this tool is not to push you either way. It is to make sure whichever position you are in is the one you chose. Common questions ## AI crawler access, answered plainly Does blocking Google-Extended hurt my Google rankings? No. Google-Extended controls Gemini and AI Overviews training only. Googlebot handles Search and is governed separately. You can block one and keep the other, and many publishers do exactly that. If I block GPTBot, am I out of ChatGPT? Not entirely. GPTBot is the training crawler. OAI-SearchBot builds the index used for ChatGPT Search, and ChatGPT-User fetches pages when a person shares a link. To be genuinely absent you would need to block all three. Do AI crawlers actually obey robots.txt? The major ones from OpenAI, Anthropic, Google, Apple and Perplexity publicly commit to honouring it, and independent testing broadly supports that. Some smaller scrapers ignore it entirely. robots.txt is a request, not a firewall — if you need enforcement, that happens at server or WAF level. What is llms.txt and do I need one? robots.txt says what crawlers may not read. llms.txt tells them what is worth reading and how your site is organised. It is a young convention with real adoption momentum. Our [llms.txt tool](https://moxseo.com/tools/llms-txt-generator/) validates yours or builds a starter file from your sitemap. How often should I re-check this? After any migration, security plugin change, or CDN configuration update — those are the three things that most often rewrite robots.txt without anyone noticing. Quarterly is sensible otherwise, since new AI user-agents appear regularly. Next step ## Knowing you are crawlable is the floor, not the ceiling Access gets you eligible. Being chosen as the cited source is a different problem — one of structure, entity clarity and demonstrable expertise. [AI SEO services](https://moxseo.com/services/seo/ai/)[See all free tools](https://moxseo.com/tools/) (function(){ var inp=document.getElementById('cw-in'),btn=document.getElementById('cw-go'),out=document.getElementById('cw-out'); if(!inp||!btn||!out)return; var C={allowed:'#16a077',partial:'#e8a20c',blocked:'#e34d5b'}; var L={allowed:'ALLOWED',partial:'PARTIAL',blocked:'BLOCKED'}; function esc(s){var d=document.createElement('div');d.textContent=s==null?'':s;return d.innerHTML;} function note(m,c){return ''+m+'';} function run(){ var d=(inp.value||'').trim(); if(!d){out.innerHTML=note('Enter a domain to check.');return;} btn.disabled=true;btn.textContent='Checking...'; out.innerHTML=note('Fetching robots.txt for '+esc(d)+'...'); fetch('/wp-json/moxtools/v1/crawlers?domain='+encodeURIComponent(d)) .then(function(r){return r.json().then(function(j){return {s:r.status,j:j};});}) .then(function(res){ btn.disabled=false;btn.textContent='Check access'; var j=res.j; if(res.s!==200||j.error){out.innerHTML=note(esc(j.error||'Something went wrong.'),'#e34d5b');return;} var c=j.counts||{},tot=(c.allowed||0)+(c.partial||0)+(c.blocked||0); var h=''; h+='Result for '+esc(j.host)+''; h+=''+(c.allowed||0)+'/'+tot+'AI crawlers fully allowed on this domain'; h+=''; [['allowed','Allowed'],['partial','Partial'],['blocked','Blocked']].forEach(function(k){ h+=''+(c[k[0]]||0)+''+k[1]+''; }); h+=''; h+='robots.txt '+(j.robots_found?('found · '+j.robots_bytes+' bytes · '+j.groups_found+' user-agent group(s)'):'not found — all crawlers permitted by default')+' · llms.txt '+(j.llms_txt?'present':'absent')+''; var rows=''; (j.rows||[]).forEach(function(r,i){ rows+=''; rows+=''+L[r.status]+''; rows+=''+esc(r.bot)+''+esc(r.owner)+' · '+esc(r.note)+''; rows+=''+esc(r.reason)+''; }); rows+=''; var nx=''; if((c.blocked||0)>0){nx+='**'+c.blocked+' crawler(s) fully blocked.** If that was not deliberate, the fix is usually one line in robots.txt. ';} else if((c.partial||0)===tot){nx+='**Everything is partially restricted.** Check whether the disallowed paths include content you actually want cited. ';} else {nx+='**Access looks healthy.** ';} nx+=(j.llms_txt?'You already have an llms.txt — ':'You have no llms.txt — ')+'['+(j.llms_txt?'validate it here':'generate one here')+'](https://moxseo.com/tools/llms-txt-generator/).'; out.innerHTML=h+rows+nx; }) .catch(function(){btn.disabled=false;btn.textContent='Check access';out.innerHTML=note('Request failed. Please try again.','#e34d5b');}); } btn.addEventListener('click',run); inp.addEventListener('keydown',function(e){if(e.key==='Enter'){e.preventDefault();run();}}); })(); /* .mox-cw base: light surface */ .mox-cw h1,.mox-cw h2,.mox-cw h3,.mox-cw h4{color:var(--mox-ink)!important} .mox-cw p,.mox-cw li,.mox-cw td{color:var(--mox-ink-soft)!important} .mox-cw .sub,.mox-cw .lead,.mox-cw .hint{color:var(--mox-ink-soft)!important} .mox-cw .card h3{color:var(--mox-ink)!important} .mox-cw .card p{color:var(--mox-ink-soft)!important} .mox-cw .band h3{color:var(--mox-ink)!important} .mox-cw .band p{color:var(--mox-ink-soft)!important} .mox-cw .faq summary{color:var(--mox-ink)!important} .mox-cw .faq p{color:var(--mox-ink-soft)!important} .mox-cw .eyebrow,.mox-cw .chip{color:var(--mox-primary)!important} .mox-cw a{color:var(--mox-primary)!important} .mox-cw .btn--p{color:#fff!important} .mox-cw .btn--ghost{color:var(--mox-ink)!important} .mox-cw .band--a .band__t{color:var(--mox-good)!important} .mox-cw .band--b .band__t{color:var(--mox-warn)!important} .mox-cw .band--c .band__t{color:var(--mox-risk)!important} /* .mox-cw dark surfaces */ .mox-cw .trust__row,.mox-cw .trust__row span{color:#a7acb5!important} .mox-cw .sec--dark h1,.mox-cw .sec--dark h2,.mox-cw .sec--dark h3,.mox-cw .sec--dark p,.mox-cw .sec--dark li{color:#f4f4f6!important} .mox-cw .sec--dark .sub,.mox-cw .sec--dark .eyebrow{color:#a7acb5!important} .mox-cw .final h1,.mox-cw .final h2,.mox-cw .final h3{color:#fff!important} .mox-cw .final p,.mox-cw .final .sub{color:#a7acb5!important} .mox-cw .final .eyebrow{color:var(--mox-accent)!important} .mox-cw .final .btn--dg{color:#fff!important} .mox-cw .sample__tag{color:var(--mox-accent)!important} .mox-cw .sample__num{color:var(--mox-accent)!important} .mox-cw .sample__lbl,.mox-cw .sample__note{color:#85898f!important} .mox-cw .sample__row span{color:#85898f!important} .mox-cw .sample__row b{color:#fff!important} .mox-cw .ticks span{color:var(--mox-ink-soft)!important}