# Global crawl policy
User-agent: *
Allow: /
Allow: /privacy
Allow: /terms
Allow: /support
Allow: /licensing
Allow: /features/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /llms/
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /node_modules/
Disallow: /app
Disallow: /sign-in
Disallow: /sign-up
Disallow: /onboarding
Disallow: /oauth/
Disallow: /openrouter/
Disallow: /mobile-drive-picker

# ── OpenAI ──
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# ── Anthropic ──
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# ── Google AI ──
User-agent: Google-Extended
Allow: /

# ── Apple ──
User-agent: Applebot-Extended
Allow: /

# ── Perplexity ──
User-agent: PerplexityBot
Allow: /

# ── Meta ──
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# ── Amazon ──
User-agent: Amazonbot
Allow: /

# ── Cohere ──
User-agent: cohere-ai
Allow: /

# ── Bytedance ──
User-agent: Bytespider
Allow: /

# ── Common Crawl (used by many AI training sets) ──
User-agent: CCBot
Allow: /

# ── You.com ──
User-agent: YouBot
Allow: /

# Sitemaps
Sitemap: https://nanthai.tech/sitemap.xml
Sitemap: https://nanthai.tech/sitemap-edge.xml