# robots.txt for ukfile.co.uk
# Policy: allow search + AI-referencing crawlers (cite us, send traffic);
# block AI training-only crawlers (scrape for model training, no traffic back)
# and SEO scrapers.

# --- Search engines ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Applebot
Allow: /

# --- AI search / referencing (cite sources, send users) ---
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Buscadores no-Google con indice propio o heredado de Bing
# Documentado en ~/.claude/skills/launch-site/multi-search-engine-seo.md
User-agent: Bravebot
Allow: /

User-agent: Qwantbot
Allow: /

User-agent: MojeekBot
Allow: /

User-agent: Kagibot
Allow: /

User-agent: Yandex
Allow: /

# --- AI training-only crawlers: blocked ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Amazonbot
Disallow: /

# --- SEO scrapers: blocked ---
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: *
Allow: /

Sitemap: https://ukfile.co.uk/sitemap-index.xml