# DropContent robots.txt
User-agent: *
Allow: /

# Block API endpoints (no crawlable content, conserves crawl budget)
Disallow: /api/

# Block private app routes (require auth, returns duplicate content to crawlers)
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /site/
Disallow: /account
Disallow: /account/
Disallow: /onboarding
Disallow: /onboarding/
Disallow: /admin/
Disallow: /client-portal/

# Sitemap location
Sitemap: https://dropcontent.co/sitemap.xml

# LLMs.txt - AI content index
# Summary: https://dropcontent.co/llms.txt
# Full content: https://dropcontent.co/llms-full.txt

# Allow AI crawlers to access public content
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /dashboard
Disallow: /site/

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /dashboard
Disallow: /site/

User-agent: Claude-Web
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /dashboard
Disallow: /site/

User-agent: Anthropic-AI
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /dashboard
Disallow: /site/

User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /dashboard
Disallow: /site/

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /dashboard
Disallow: /site/

User-agent: Cohere-AI
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /dashboard
Disallow: /site/