# https://coasty.ai/robots.txt
# Coasty allows AI training and search indexing. We're an AI-first product —
# we want crawlers to find, index, retrieve, and cite our content.
#
# Discovery surfaces for agents:
#   - https://coasty.ai/llms.txt
#   - https://coasty.ai/llms-full.txt
#   - https://coasty.ai/sitemap.xml
#   - https://coasty.ai/api/discovery
#   - https://coasty.ai/.well-known/openapi.json
#   - https://coasty.ai/.well-known/mcp/server-card.json

User-agent: *
Allow: /
Disallow: /api/
Disallow: /c/
Disallow: /account
Disallow: /onboarding
Disallow: /auth/error
Disallow: /c/*/edit
Disallow: /p/*/settings
Sitemap: https://coasty.ai/sitemap.xml

# ─── OpenAI ──────────────────────────────────────────────────────────────
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /c/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /c/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /c/

# ─── Anthropic ───────────────────────────────────────────────────────────
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /c/

User-agent: ClaudeUser
Allow: /
Disallow: /api/
Disallow: /c/

User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /c/

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /c/

# ─── Google ──────────────────────────────────────────────────────────────
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /c/

User-agent: Googlebot
Allow: /
Disallow: /api/
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /
Disallow: /api/

# ─── Apple ───────────────────────────────────────────────────────────────
User-agent: Applebot
Allow: /
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/

# ─── Meta ────────────────────────────────────────────────────────────────
User-agent: FacebookBot
Allow: /
Disallow: /api/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/

# ─── Perplexity ──────────────────────────────────────────────────────────
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /c/

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /c/

# ─── Mistral ─────────────────────────────────────────────────────────────
User-agent: MistralAI-User
Allow: /
Disallow: /api/
Disallow: /c/

# ─── Cohere ──────────────────────────────────────────────────────────────
User-agent: cohere-ai
Allow: /
Disallow: /api/

# ─── Common Crawl ────────────────────────────────────────────────────────
User-agent: CCBot
Allow: /
Disallow: /api/

# ─── Diffbot ─────────────────────────────────────────────────────────────
User-agent: Diffbot
Allow: /
Disallow: /api/

# ─── You.com ─────────────────────────────────────────────────────────────
User-agent: YouBot
Allow: /
Disallow: /api/

# ─── Microsoft Bing ──────────────────────────────────────────────────────
User-agent: Bingbot
Allow: /
Disallow: /api/
Crawl-delay: 0

# ─── Other major search crawlers ─────────────────────────────────────────
User-agent: Slurp
Allow: /
Disallow: /api/
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Crawl-delay: 1

User-agent: Yandex
Allow: /
Disallow: /api/
Crawl-delay: 2

User-agent: Baiduspider
Allow: /
Disallow: /api/
Crawl-delay: 2

User-agent: Yeti
Allow: /
Disallow: /api/
Crawl-delay: 1

User-agent: SeznamBot
Allow: /
Disallow: /api/
Crawl-delay: 1

# ─── Opt-out: aggressive scrapers ────────────────────────────────────────
# Bytedance/TikTok crawler — historically scrapes aggressively, no useful
# inbound traffic for our segment.
User-agent: Bytespider
Disallow: /

# SEO/competitor scrapers — no value to us, high server cost.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: BLEXBot
Disallow: /