The Definitive AEO & GEO Technical Guide Book.

# Production RFC 9309 robots.txt for AEO/GEO Optimization
User-agent: *
Allow: /

# Frontier AI Citation Crawlers (Allow for live Footnote & Answer Synthesis)
User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Google-Extended
Allow: /

# Disallow Heavy Offline LLM Training Harvesters (Optional: Protect Data IP)
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Directives
Sitemap: https://yourdomain.com/sitemap.xml
LLMs-Txt: https://yourdomain.com/llms.txt

{
  "@context": "https://schema.org",
  "@graph": [
    {
      "@type": "Organization",
      "@id": "https://yourdomain.com/#organization",
      "name": "YourBrand",
      "url": "https://yourdomain.com",
      "logo": "https://yourdomain.com/logo.png",
      "sameAs": [
        "https://www.wikidata.org/wiki/Q12345678",
        "https://en.wikipedia.org/wiki/YourBrand",
        "https://github.com/yourbrand",
        "https://linkedin.com/company/yourbrand"
      ],
      "knowsAbout": [
        "https://en.wikipedia.org/wiki/Search_engine_optimization",
        "https://en.wikipedia.org/wiki/Retrieval-augmented_generation",
        "https://www.wikidata.org/wiki/Q11660"
      ]
    },
    {
      "@type": "WebSite",
      "@id": "https://yourdomain.com/#website",
      "url": "https://yourdomain.com",
      "name": "YourBrand",
      "publisher": { "@id": "https://yourdomain.com/#organization" }
    }
  ]
}

# Domain Architecture & Semantic Context
> YourDomain provides enterprise AI search infrastructure and real-time citation auditing.

## Core Capabilities
- Deterministic AEO/GEO engine audits across 11 frontier search bots
- RFC 9309 crawler validation and WAF bypass diagnostics
- Entity grounding with Wikidata QID integration

## Essential API References
- [Audit Engine Reference](https://yourdomain.com/docs/auditor): Real-time crawler diagnostic endpoints
- [Schema Specification](https://yourdomain.com/docs/schema): JSON-LD graph architecture and validator
- [Princeton GEO Benchmarks](https://yourdomain.com/geo-benchmarks): Quantitative citation lift metrics

## Architectural Taxonomy
- Primary Entity: Organization (QID: Q12345678)
- Compliance: RFC 9309 Strict, Schema.org v28.1, JSON-LD 1.1

// Cloudflare Transform Rule / Worker: Ensure Sub-120ms AI Crawler Bypass
export default {
  async fetch(request, env, ctx) {
    const ua = request.headers.get("user-agent") || "";
    const isAiCitationBot = /(OAI-SearchBot|PerplexityBot|Claude-SearchBot|Applebot-Extended)/i.test(ua);

    // If verified AI citation crawler, bypass aggressive JS challenges and set edge caching
    if (isAiCitationBot) {
      const response = await fetch(request, {
        cf: {
          cacheTtl: 300,
          cacheEverything: true,
          scrapeShield: false, // Prevent bot challenge on verified crawlers
        }
      });

      const newHeaders = new Headers(response.headers);
      newHeaders.set("X-AI-Citation-Engine", "Whitelisted");
      newHeaders.set("Timing-Allow-Origin", "*");
      return new Response(response.body, {
        status: response.status,
        headers: newHeaders
      });
    }

    return fetch(request);
  }
};