"use strict"; const {Fragment: _Fragment, jsx: _jsx, jsxs: _jsxs} = arguments[0]; const {useMDXComponents: _provideComponents} = arguments[0]; function _createMdxContent(props) { const _components = { a: "a", code: "code", h2: "h2", h3: "h3", hr: "hr", li: "li", p: "p", pre: "pre", strong: "strong", table: "table", tbody: "tbody", td: "td", th: "th", thead: "thead", tr: "tr", ul: "ul", ..._provideComponents(), ...props.components }; return _jsxs(_Fragment, { children: [_jsxs(_components.p, { children: ["Most \"AI SEO\" advice is just standard SEO. Some configurations are genuinely specific to how ", _jsx(_components.a, { href: "/resources/glossary/ai-agent", children: "AI agents" }), " interact with your site. This checklist separates the two and provides actual configurations to implement."] }), "\n", _jsx(_components.p, { children: "This is not a full audit guide; it is a configuration reference. If you implement these items, your technical foundation will be compatible with current AI navigation patterns. Treat it as a checklist to verify readiness, not a narrative to read once and forget." }), "\n", _jsx(_components.h2, { children: "Robots.txt Configuration" }), "\n", _jsxs(_components.p, { children: [_jsx(_components.a, { href: "/resources/glossary/robots-txt-ai", children: "Robots.txt" }), " is the first decision point for AI agents. It determines whether crawlers can discover your content for indexing and training."] }), "\n", _jsx(_components.h3, { children: "The decision: Allow or block?" }), "\n", _jsx(_components.p, { children: "The recommendation is to allow access unless you have specific intellectual property protection needs, such as paywalled content or proprietary documentation." }), "\n", _jsx(_components.p, { children: "Blocking AI crawlers removes your content from AI search visibility. For most businesses, preventing indexing on a platform where users are actively seeking your services is a negative trade-off." }), "\n", _jsx(_components.p, { children: "Note the distinction: Blocking crawlers stops your content from being indexed or trained on. It does not necessarily prevent AI agents from visiting your site in real-time when a user asks them to. ChatGPT browsing and crawler indexing are different things." }), "\n", _jsx(_components.h3, { children: "AI crawler user-agents to know about" }), "\n", _jsx(_components.p, { children: "You should permit the following user-agents if you want visibility across these platforms:" }), "\n", _jsxs(_components.table, { children: [_jsx(_components.thead, { children: _jsxs(_components.tr, { children: [_jsx(_components.th, { children: "User-Agent" }), _jsx(_components.th, { children: "Crawler" }), _jsx(_components.th, { children: "Block?" }), _jsx(_components.th, { children: "Notes" })] }) }), _jsxs(_components.tbody, { children: [_jsxs(_components.tr, { children: [_jsx(_components.td, { children: "GPTBot" }), _jsx(_components.td, { children: "OpenAI" }), _jsx(_components.td, { children: "No" }), _jsx(_components.td, { children: "Required for ChatGPT search visibility" })] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "ClaudeBot" }), _jsx(_components.td, { children: "Anthropic" }), _jsx(_components.td, { children: "No" }), _jsx(_components.td, { children: "Required for Claude visibility" })] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "PerplexityBot" }), _jsx(_components.td, { children: "Perplexity" }), _jsx(_components.td, { children: "No" }), _jsx(_components.td, { children: "Required for Perplexity visibility" })] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "CCBot" }), _jsx(_components.td, { children: "Common Crawl" }), _jsx(_components.td, { children: "No" }), _jsx(_components.td, { children: "Used for training data; block only for IP protection" })] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "Googlebot" }), _jsx(_components.td, { children: "Google" }), _jsx(_components.td, { children: "No" }), _jsx(_components.td, { children: "Blocks all Google indexing if blocked" })] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "Amazonbot" }), _jsx(_components.td, { children: "Amazon" }), _jsx(_components.td, { children: "No" }), _jsx(_components.td, { children: "Blocks Alexa results if blocked" })] }), _jsxs(_components.tr, { children: [_jsx(_components.td, { children: "anthropic-ai" }), _jsx(_components.td, { children: "Anthropic" }), _jsx(_components.td, { children: "No" }), _jsx(_components.td, { children: "Alternative identifier" })] })] })] }), "\n", _jsxs(_components.p, { children: ["Blocking specific agents like ", _jsx(_components.code, { children: "GPTBot" }), " or ", _jsx(_components.code, { children: "ClaudeBot" }), " prevents discovery via those platforms' indexes."] }), "\n", _jsx(_components.h3, { children: "Common robots.txt configurations" }), "\n", _jsx(_components.p, { children: _jsx(_components.strong, { children: "Recommended: Allow all" }) }), "\n", _jsx(_components.pre, { children: _jsx(_components.code, { className: "language-text", children: "User-agent: *\nAllow: /\nSitemap: https://yoursite.com/sitemap.xml\n" }) }), "\n", _jsx(_components.p, { children: _jsx(_components.strong, { children: "Allow all except specific paths (for paywalled content)" }) }), "\n", _jsx(_components.pre, { children: _jsx(_components.code, { className: "language-text", children: "User-agent: *\nAllow: /\nDisallow: /members-only/\nDisallow: /private/\nSitemap: https://yoursite.com/sitemap.xml\n" }) }), "\n", _jsx(_components.p, { children: _jsx(_components.strong, { children: "Allow all but throttle aggressive crawlers" }) }), "\n", _jsx(_components.pre, { children: _jsx(_components.code, { className: "language-text", children: "User-agent: *\nAllow: /\nCrawl-delay: 5\n\nUser-agent: PerplexityBot\nCrawl-delay: 10\n" }) }), "\n", _jsx(_components.p, { children: _jsx(_components.strong, { children: "Block AI crawlers (only if IP protection is critical)" }) }), "\n", _jsx(_components.pre, { children: _jsx(_components.code, { className: "language-text", children: "User-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: *\nAllow: /\n" }) }), "\n", _jsx(_components.h3, { children: "Verification" }), "\n", _jsxs(_components.p, { children: ["Check your file at ", _jsx(_components.code, { children: "yoursite.com/robots.txt" }), ". Use Google Search Console to test bot access rules. Ensure you haven't accidentally blocked standard crawlers while targeting AI agents."] }), "\n", _jsx(_components.h2, { children: "Crawlability and Rendering" }), "\n", _jsx(_components.p, { children: "Content must be fetchable and rendered. AI agents, like standard search engines, struggle to execute complex JavaScript dynamically during initial fetches." }), "\n", _jsx(_components.h3, { children: "Static HTML vs. JavaScript rendering" }), "\n", _jsxs(_components.p, { children: ["Test your site by disabling JavaScript in your browser. If critical content disappears—navigation menus, product descriptions, pricing information, or contact forms—you have a rendering issue. Basic ", _jsx(_components.a, { href: "/resources/glossary/ai-crawling", children: "AI crawlers" }), " cannot execute JavaScript to retrieve this information."] }), "\n", _jsx(_components.h3, { children: "Recommended approach: SSR or Static Generation" }), "\n", _jsx(_components.p, { children: "Content should reside in HTML, not load via client-side JavaScript. For dynamic sites, use Server-Side Rendering (SSR) or Static Site Generation (SSG)." }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "SSR:" }), " Content generated on the server and sent as HTML."] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "SSG:" }), " Pages pre-built at build time, served as static files."] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Hybrid:" }), " Static for key pages (pricing, contact), dynamic for secondary content."] }), "\n"] }), "\n", _jsxs(_components.p, { children: ["Frameworks handle this differently: Next.js uses ", _jsx(_components.code, { children: "getStaticProps" }), " or ", _jsx(_components.code, { children: "getServerSideProps" }), "; Vue uses Nuxt for SSR; SvelteKit defaults to SSR. Hugo defaults to static."] }), "\n", _jsx(_components.h3, { children: "Testing: What AI agents see" }), "\n", _jsxs(_components.p, { children: [_jsx(_components.strong, { children: "Method 1: Screaming Frog" }), "\nSet to \"JavaScript: Enabled\". Crawl your site. Compare word count in raw HTML vs. rendered content. If rendered content exceeds HTML by 50% or more, you have JS rendering issues that will block AI discovery."] }), "\n", _jsxs(_components.p, { children: [_jsx(_components.strong, { children: "Method 2: Manual check" }), "\nAsk an AI agent: \"Visit [yoursite.com/key-page]. What content can you see?\" Compare this to what you see in a browser. Major discrepancies indicate rendering failures."] }), "\n", _jsx(_components.h3, { children: "What's actually critical" }), "\n", _jsx(_components.p, { children: "Not all JavaScript needs fixing. Only content that matters for discovery must be in HTML:" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Navigation:" }), " Must be in HTML or rendered."] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Product/service descriptions:" }), " Must be in HTML."] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Pricing:" }), " Must be in HTML."] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Contact information:" }), " Must be in HTML."] }), "\n"] }), "\n", _jsx(_components.p, { children: "Nice-to-haves can remain JS-loaded: Analytics, chat widgets, and expandable \"View more\" sections." }), "\n", _jsx(_components.h3, { children: "Crawl delay and rate limiting" }), "\n", _jsx(_components.p, { children: "If you have high traffic, rate-limit appropriately to prevent server strain. Reasonable rate limits prevent issues with multiple AI crawlers plus standard bots." }), "\n", _jsx(_components.pre, { children: _jsx(_components.code, { className: "language-text", children: "User-agent: *\nCrawl-delay: 5\nRequest-rate: 1/5\n" }) }), "\n", _jsx(_components.h2, { children: "Schema Markup Essentials" }), "\n", _jsxs(_components.p, { children: [_jsx(_components.a, { href: "/resources/glossary/schema-markup", children: "Schema markup" }), " provides semantic context, reducing ", _jsx(_components.a, { href: "/resources/glossary/ai-hallucination", children: "hallucination" }), " risk and helping AI extract structured information accurately."] }), "\n", _jsx(_components.h3, { children: "Why schema matters for AI" }), "\n", _jsx(_components.p, { children: "Instead of parsing raw text like \"£99/month\", schema explicitly states \"this is a price for a software product with a 30-day trial.\" This structure improves extraction accuracy." }), "\n", _jsx(_components.h3, { children: "Essential schema types for AI" }), "\n", _jsx(_components.p, { children: _jsx(_components.strong, { children: "High impact (implement these):" }) }), "\n", _jsxs(_components.p, { children: [_jsx(_components.strong, { children: "Organization schema" }), " (on homepage)"] }), "\n", _jsx(_components.pre, { children: _jsx(_components.code, { className: "language-json", children: "{\n \"@context\": \"https://schema.org\",\n \"@type\": \"Organization\",\n \"name\": \"Your Company\",\n \"url\": \"https://yoursite.com\",\n \"logo\": \"https://yoursite.com/logo.png\",\n \"contactPoint\": {\n \"@type\": \"ContactPoint\",\n \"contactType\": \"Customer Service\",\n \"email\": \"hello@yoursite.com\"\n }\n}\n" }) }), "\n", _jsxs(_components.p, { children: [_jsx(_components.strong, { children: "Product schema" }), " (on product pages)"] }), "\n", _jsx(_components.pre, { children: _jsx(_components.code, { className: "language-json", children: "{\n \"@context\": \"https://schema.org/\",\n \"@type\": \"Product\",\n \"name\": \"Product Name\",\n \"description\": \"Product description\",\n \"price\": \"99.99\",\n \"priceCurrency\": \"GBP\",\n \"url\": \"https://yoursite.com/products/xyz\"\n}\n" }) }), "\n", _jsxs(_components.p, { children: [_jsx(_components.strong, { children: "Article/BlogPosting schema" }), " (on blog posts)"] }), "\n", _jsx(_components.pre, { children: _jsx(_components.code, { className: "language-json", children: "{\n \"@context\": \"https://schema.org\",\n \"@type\": \"BlogPosting\",\n \"headline\": \"Title\",\n \"datePublished\": \"2026-03-22\",\n \"author\": {\n \"@type\": \"Person\",\n \"name\": \"Author Name\"\n }\n}\n" }) }), "\n", _jsx(_components.p, { children: _jsx(_components.strong, { children: "Medium impact:" }) }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "LocalBusiness schema" }), " (if you have physical locations)"] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "BreadcrumbList schema" }), " (aids navigation understanding)"] }), "\n"] }), "\n", _jsx(_components.p, { children: _jsx(_components.strong, { children: "Lower impact:" }) }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Event schema" }), " (if you have events)"] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "FAQ schema" }), " (helps with Q&A, but not critical)"] }), "\n"] }), "\n", _jsx(_components.h3, { children: "Implementation and Testing" }), "\n", _jsxs(_components.p, { children: ["Use JSON-LD. Add a ", _jsx(_components.code, { children: "

Technical AEO Checklist: Robots.txt, Schema, Site Architecture