User-agent: *


User-agent: *Disallow: /*.html

# Allow public pages
Allow: /

# Block OpenAI's GPTBot (used by ChatGPT browsing & training)
User-agent: GPTBot
Disallow: /

# Block Google's AI crawler (Bard/Gemini)
User-agent: Google-Extended
Disallow: /

# Block Common AI scrapers (Anthropic, Perplexity, etc.)
User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /