# https://www.robotstxt.org/robotstxt.html

User-agent: *
Allow: /

# Next.js internals (allow static assets for rendering)
Allow: /_next/static/
Disallow: /_next/
Disallow: /api/

# Pages with no SEO value
Disallow: /search
Disallow: /filter
Disallow: /demo/

# Vercel debug params
Disallow: /*?_vercel_toolbar=1

# Throttle aggressive bots
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

# AI training bots — block (no benefit, consumes server resources)
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

# AI search/retrieval bots — allow (they cite the site with links)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

# Sitemaps
Sitemap: https://usfiredept.com/sitemap.xml