# robots.txt for Gothic Codex
# https://www.gothiccodex.com/robots.txt
#
# Each group is one contiguous block: its User-agent lines on consecutive
# lines, then its Allow / Disallow rules, with no blank line inside the
# group. RFC 9309 parsers ignore blank lines, but older ones (Python's
# urllib.robotparser among them) end the group at the first blank line,
# drop a User-agent run that has no rule yet, and ignore rules that follow.
# Comments are skipped by both, so they may annotate the rules.

# ============================================
# Default: Allow all search engines and bots
# ============================================
# Account pages (/login, /settings, /profile, /my-events) stay crawlable
# on purpose: each carries <meta name="robots" content="noindex, nofollow">,
# and a crawler has to fetch the page to see it. A Disallow here would hide
# the noindex, and Google can still index a blocked URL from outside links.
User-agent: *
Allow: /
# API endpoints (not meant for crawling)
Disallow: /api/
Disallow: /admin/
Disallow: /.env*
# Internal/development files and directories
Disallow: /node_modules/
Disallow: /build/
Disallow: /test-results/
Disallow: /playwright-report/
Disallow: /.git/
Disallow: /migrations/
Disallow: /supabase/
Disallow: /Documentation/
Disallow: /design-system
Disallow: /tests/
Disallow: /scripts/*.ts$
Disallow: /scripts/*.tsx$
# Verification files (IndexNow, Yandex)
Allow: /5c400e87591a4217871048fafd228c0d.txt
# Static assets
Allow: /data/
Allow: /fonts/
Allow: /public/
Allow: /styles/
Allow: /*.css$
Allow: /*.js$
Allow: /*.woff2$
Allow: /*.woff$
Allow: /*.ttf$
Allow: /*.svg$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.webp$
Allow: /*.ico$
# Manifest and config files
Allow: /site.webmanifest
Allow: /browserconfig.xml
Allow: /sitemap.xml
Allow: /sitemap
Allow: /robots.txt
# LLM guidance files
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/llms.txt
# Favicon files, explicitly, for search engines
Allow: /favicon.ico
Allow: /favicon.svg
Allow: /favicon-16.png
Allow: /favicon-180.png
Allow: /favicon-192.png
Allow: /favicon-512.png
Allow: /safari-pinned-tab.svg

# ============================================
# AI Search Crawlers — Explicitly allow (GEO)
# These crawlers power AI search, chatbots,
# and generative engines (information retrieval)
# NOT used for AI model training
# ============================================
# ChatGPT-User      OpenAI (ChatGPT, SearchGPT)
# OAI-SearchBot     OpenAI's search engine, so it can cite the site
# Claude-SearchBot  Anthropic's dedicated search crawler (Claude-Web is
#                   retired; the training crawler ClaudeBot is blocked below)
# Claude-User       Anthropic's user-directed agent for real-time browsing
# PerplexityBot     Perplexity, to read and link the site
# Perplexity-User   Perplexity's bot for live user queries
# DuckDuckBot-AI    DuckDuckGo AI answers
# DuckAssistBot     DuckDuckGo's AI-summarized answer feature
# DuckDuckBot       DuckDuckGo (AI-powered answers)
# cohere-ai         Cohere
# MistralAI-User    Mistral AI
# Applebot          Apple (Siri and Apple Intelligence)
# Bingbot           Microsoft / Bing AI (Copilot)
# YouBot            You.com AI Search
# Grok              xAI
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckDuckBot-AI
User-agent: DuckAssistBot
User-agent: DuckDuckBot
User-agent: cohere-ai
User-agent: MistralAI-User
User-agent: Applebot
User-agent: Bingbot
User-agent: YouBot
User-agent: Grok
Allow: /
# API endpoints (not meant for crawling)
Disallow: /api/
Disallow: /admin/
Disallow: /.env*
# Internal/development files and directories
Disallow: /node_modules/
Disallow: /build/
Disallow: /test-results/
Disallow: /playwright-report/
Disallow: /.git/
Disallow: /migrations/
Disallow: /supabase/
Disallow: /Documentation/
Disallow: /design-system
Disallow: /tests/
Disallow: /scripts/*.ts$
Disallow: /scripts/*.tsx$

# ============================================
# AI Training Crawlers — Block
# These crawlers collect data to train AI
# models. Content must not be used for
# AI training purposes.
# ============================================

# OpenAI Training
User-agent: GPTBot
Disallow: /

# Google AI Training
User-agent: Google-Extended
Disallow: /

# Anthropic Training
User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

# Meta AI Training
User-agent: Meta-ExternalAgent
Disallow: /

# Meta says FacebookBot crawls pages to improve its language models
# (link previews use facebookexternalhit, which stays allowed above)
User-agent: FacebookBot
Disallow: /

# Apple AI Training
User-agent: Applebot-Extended
Disallow: /

# Amazon: Amazonbot's crawl data may train Amazon's AI models
User-agent: Amazonbot
Disallow: /

# Common Crawl — primary source for AI training datasets (GPT, BERT, etc.)
User-agent: CCBot
Disallow: /

# Diffbot — scrapes web content to build ML training datasets
User-agent: Diffbot
Disallow: /

# ByteDance / TikTok AI training
User-agent: Bytespider
Disallow: /

# Omgili / Webz.io — feeds data into AI/ML pipelines
User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

# DataForSeo — bulk data collection for ML/AI
User-agent: DataForSeoBot
Disallow: /

# AI2Bot — Allen Institute for AI training
User-agent: AI2Bot
Disallow: /

# Timpibot — used for AI training datasets
User-agent: Timpibot
Disallow: /

# PetalBot — Huawei search/AI training
User-agent: PetalBot
Disallow: /

# Crawl-delay (optional - adjust if needed)
# Crawl-delay: 1

# ============================================
# Sitemap
# ============================================
Sitemap: https://www.gothiccodex.com/sitemap.xml

# LLM discovery files
# https://www.gothiccodex.com/llms.txt
# https://www.gothiccodex.com/.well-known/llms.txt