# General access for all bots
User-agent: *
Disallow: /private/
Disallow: /tmp/
Disallow: /cache/

# Sitemap for standard crawlers
Sitemap: https://noldus.com/sitemap.xml

# Custom field (non-standard, but LLMs may check it)
LLMS: https://noldus.com/llms.json

# --- Explicit allowances for known AI bots ---

# OpenAI GPTBot (ChatGPT browsing)
User-agent: GPTBot
Allow: /

# OpenAI ChatGPT Plugins
User-agent: ChatGPT-User
Allow: /

# Anthropic's Claude
User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google AI crawlers (future-proofing)
User-agent: Google-Extended
Allow: /

# Common LLM-related bots
User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /