# Crawler policy for sebastienrousseau.com
# Human-readable version: https://sebastienrousseau.com/about/#bot-policy
# License of crawled content: CC BY-4.0 — attribution required.

User-agent: *
Allow: /

# -- Web search engines -----------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Yandex
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Applebot
Allow: /

# -- Social / link-preview --------------------------------------------------
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# -- SEO audit / link-graph crawlers ---------------------------------------
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

# -- AI retrieval (cite-on-query) ------------------------------------------
# These crawlers fetch on user-query and surface citations. Highest-signal
# bots for the site's distribution strategy.
User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: YouBot
Allow: /

# -- AI training crawlers ---------------------------------------------------
# Broad-ingest model-training bots. Allowed under CC BY-4.0; attribution
# requested per /about/#bot-policy. Flip any line to `Disallow: /` to
# opt out of that specific corpus.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: AmazonbotCommerce
Allow: /

User-agent: Bytespider
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: meta-externalfetcher
Allow: /

User-agent: facebook-externalhit-llama
Allow: /

# -- Specialised indexers ---------------------------------------------------
User-agent: ImagesiftBot
Allow: /

User-agent: Diffbot
Allow: /

# -- Sitemaps ---------------------------------------------------------------
Sitemap: https://sebastienrousseau.com/sitemap.xml
Sitemap: https://sebastienrousseau.com/news-sitemap.xml
Sitemap: https://sebastienrousseau.com/fr/news-sitemap.xml

# -- Machine-readable surfaces ---------------------------------------------
# llms.txt        — navigation index for LLM ingestion (llmstxt.org)
# llms-ctx.txt    — compact agent-context format (URLs + one-line context)
# llms-full.txt   — full article corpus, navigation-stripped
# api/agents/     — JSON API: posts, topics, person, organization
# Bot policy      — https://sebastienrousseau.com/about/#bot-policy

# llms.txt:       https://sebastienrousseau.com/llms.txt
# llms-ctx.txt:   https://sebastienrousseau.com/llms-ctx.txt
# llms-full.txt:  https://sebastienrousseau.com/llms-full.txt
# Agent API:      https://sebastienrousseau.com/api/agents/index.json