# robots.txt for carvetka.com
# Updated based on Google Search best practices

# Allow all search engines
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://carvetka.com/sitemap.xml

# Block access to admin and private areas
Disallow: /dashboard/
Disallow: /auth/
Disallow: /api/
Disallow: /_next/
Disallow: /admin/

# Block redirect/login pages (these redirect to app.carvetka.com)
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

# Block thank you and confirmation pages (avoid thin content in index)
Disallow: /thank-you
Disallow: /confirm

# Allow important public pages explicitly
Allow: /
Allow: /pricing
Allow: /features
Allow: /faq
Allow: /contact
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /blog/
Allow: /changelog/
Allow: /resources/
Allow: /tools/

# Allow query parameters for tracking (but they should use canonical)
Allow: /*?ref=*
Allow: /*?utm_*

# Respect crawl rate
Crawl-delay: 1

# Specific rules for GoogleBot (most important)
User-agent: Googlebot
Disallow: /dashboard/
Disallow: /auth/
Disallow: /api/
Allow: /
Allow: /blog/
Allow: /resources/
Allow: /tools/

# Specific rules for Googlebot-Image
User-agent: Googlebot-Image
Allow: /
Allow: /optimized/
Allow: /images/

# Bing
User-agent: Bingbot
Crawl-delay: 1
Disallow: /dashboard/
Disallow: /auth/
Disallow: /api/
Allow: /

# AI search bots — explicitly allowed so these platforms can cite Carvetka
# These bots power ChatGPT search, Perplexity, Claude, Google AI Overviews, and Copilot
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

# Block Common Crawl (used for AI training datasets, not search citation)
User-agent: CCBot
Disallow: /

# Block bad bots and scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /