# Brizzly robots.txt
# Last updated: April 13, 2025

# Allow Google to crawl the site with some restrictions
User-agent: Googlebot
Allow: /$
Allow: /sitemap.xml$
Allow: /sitemap-static.xml$
# Allow limited user profile access (rate-limited)
Allow: /*/
# Disallow admin and sensitive areas
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /settings/
Disallow: /messages/
Disallow: /search/
Disallow: /subscriptions/
# Crawl delay to prevent excessive user profile enumeration
Crawl-delay: 5

# Similar rules for Google Mobile
User-agent: Googlebot-Mobile
Allow: /$
Allow: /sitemap.xml$
Allow: /sitemap-static.xml$
Allow: /*/
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /settings/
Disallow: /messages/
Disallow: /search/
Disallow: /subscriptions/
Crawl-delay: 5

# Block OpenAI crawlers
User-agent: GPTBot
Disallow: /

# Block Claude/Anthropic crawlers
User-agent: Claude-Web
Disallow: /

# Block Common Crawl (often used by AI training)
User-agent: CCBot
Disallow: /

# Block other AI training bots
User-agent: anthropic-ai
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Bytespider
Disallow: /

# Default rules for all other bots - more restrictive
User-agent: *
Allow: /$
Allow: /sitemap.xml$
Allow: /privacy$
Allow: /tos$
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /*/
Disallow: /settings/
Disallow: /messages/
Disallow: /search/
Disallow: /subscriptions/
Crawl-delay: 10

# Sitemap index (points to segmented sitemaps)
Sitemap: https://brizzly.com/sitemap.xml