# --- Sitemaps -------------------------------------------------------
Sitemap: https://fcfkbvculokjmzycgxbj.supabase.co/functions/v1/llm-discovery/public/sitemap.xml?domain=www.lightsite.ai
Sitemap: https://lightsite.ai/sitemap.xml
Sitemap: https://lightsite.ai/ai-sitemap.xml

# Default: allow the public site; block sensitive areas
User-agent: *
Allow: /
Disallow: /client-login/
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /admin/
Disallow: /*?*sessionid=*
Disallow: /*?*preview=*
Disallow: /*?*utm_*

# --- AI & LLM crawlers: explicitly allowed + key discovery files ----
User-agent: GPTBot
Allow: /
Allow: /ai.json
Allow: /ai/*
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/wacp.json
Allow: /api-docs.json
Allow: /ai-sitemap.xml

User-agent: Google-Extended
Allow: /
Allow: /ai.json
Allow: /ai/*
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/wacp.json
Allow: /api-docs.json
Allow: /ai-sitemap.xml

User-agent: ClaudeBot
Allow: /
Allow: /ai.json
Allow: /ai/*
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/wacp.json
Allow: /api-docs.json
Allow: /ai-sitemap.xml

User-agent: Anthropic-AI
Allow: /
Allow: /ai.json
Allow: /ai/*
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/wacp.json
Allow: /api-docs.json
Allow: /ai-sitemap.xml

User-agent: CCBot
Allow: /
Allow: /ai.json
Allow: /ai/*
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/wacp.json
Allow: /api-docs.json
Allow: /ai-sitemap.xml

User-agent: PerplexityBot
Allow: /
Allow: /ai.json
Allow: /ai/*
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/wacp.json
Allow: /api-docs.json
Allow: /ai-sitemap.xml

# --- Major web search crawlers ----------
User-agent: Googlebot
Allow: /
Allow: /ai.json
Allow: /ai/*
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/wacp.json
Allow: /api-docs.json

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# --- Highly indexable docs & API specs ------------------------------
User-agent: *
Allow: /docs/
Allow: /docs/*
Allow: /ai.json$
Allow: /ai/*.json$
Allow: /.well-known/ai-plugin.json$
Allow: /.well-known/wacp.json$
Allow: /api-docs.json$