# robots.txt for https://datalumio.co/
# Last updated: May 2026
# Reference: https://www.robotstxt.org/robotstxt.html

# ---------------------------------------------------------------------------
# Default rule: allow all well-behaved search engine crawlers
# ---------------------------------------------------------------------------
User-agent: *
Allow: /

# Block access to admin, private, and system paths
Disallow: /admin/
Disallow: /login/
Disallow: /dashboard/
Disallow: /api/
Disallow: /private/
Disallow: /wp-admin/
Disallow: /cgi-bin/
Disallow: /*.json$
Disallow: /*.xml$

# Allow sitemap XML explicitly
Allow: /sitemap.xml

# ---------------------------------------------------------------------------
# Major search engines — explicitly allowed (best practice)
# ---------------------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# ---------------------------------------------------------------------------
# AI Training Crawlers — blocked (2025–2026 standard practice)
# These bots scrape content for LLM training datasets.
# Block if you do not want your content used for AI model training.
# ---------------------------------------------------------------------------

# OpenAI
User-agent: GPTBot
Allow: /

# OpenAI web browsing (ChatGPT)
User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Google AI (Gemini / Vertex AI training)
User-agent: Google-Extended
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: meta-externalagent
Allow: /

# Amazon Alexa / Bedrock
User-agent: Amazonbot
Allow: /

# Apple Applebot-Extended (AI training)
User-agent: Applebot-Extended
Allow: /

# Common Crawl (used widely for training datasets)
User-agent: CCBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Bytedance / TikTok AI
User-agent: Bytespider
Allow: /

# Diffbot (data extraction for AI)
User-agent: Diffbot
Disallow: /

# Omgili / Webz.io (news data for AI)
User-agent: omgili
Allow: /

# DataForSEO (scraping)
User-agent: DataForSeoBot
Allow: /

# PetalBot (Huawei AI)
User-agent: PetalBot
Allow: /

# Scrapy (generic scraper framework)
User-agent: Scrapy
Allow: /

# ---------------------------------------------------------------------------
# Sitemap location
# ---------------------------------------------------------------------------
Sitemap: https://datalumio.co/sitemap.xml