# =============================================================================
# Hello Deutsch — crawler policy
# -----------------------------------------------------------------------------
# Two jobs:
#   1. Keep the lesson media out of image search and out of scraper reach.
#   2. Refuse the AI training and "browse this site for me" crawlers outright.
#
# robots.txt is honoured voluntarily. The crawlers that ignore it are handled
# at the CloudFront edge instead (tools/preview/provision.sh) — this file is
# the polite half of the same policy.
# =============================================================================

# --- AI training and answer-engine crawlers ---------------------------------
# The whole course would otherwise end up inside a model or an AI summary, and
# nobody would need the site that wrote it.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Amazonbot
User-agent: FacebookBot
User-agent: meta-externalagent
User-agent: cohere-ai
User-agent: Diffbot
User-agent: Omgilibot
User-agent: ImagesiftBot
User-agent: Timpibot
Disallow: /

# --- offline copiers and generic fetchers -----------------------------------
User-agent: HTTrack
User-agent: wget
User-agent: curl
User-agent: WebCopier
User-agent: WebZIP
User-agent: Teleport
User-agent: SiteSnagger
User-agent: python-requests
User-agent: Scrapy
Disallow: /

# --- everyone else -----------------------------------------------------------
# Ordinary search engines may index the pages. They may not index the media,
# the raw content data or the admin portal.
User-agent: *
# THE LINK-PREVIEW CARD IS EXPLICITLY ALLOWED, and it has to be: /img/ is
# disallowed below to keep lesson art out of image search, and a crawler that
# honours robots.txt — Facebook's does — will refuse to fetch a preview image
# from a disallowed path. The card lives at the root for the same reason, so
# this line is belt and braces rather than the only thing holding it up.
Allow: /share-card.png
# THE FAVICONS ARE ALLOWED FOR EXACTLY THE SAME REASON, and they were not
# always: they lived at img/owl-32.png, inside the disallow below, so Google's
# favicon fetcher never asked for them and the search result showed a grey
# globe instead of Dr Woo. They live at the root now, next to the share card.
# /favicon.ico is the fallback Google looks for when a declared icon fails.
Allow: /favicon.ico
Allow: /favicon-192.png
Allow: /apple-touch-icon.png
Disallow: /img/
Disallow: /uploads/
Disallow: /audio/
Disallow: /data/
Disallow: /admin/
Disallow: /course/content/
Allow: /

# THIS LINE WENT NOWHERE UNTIL 2026-09-18. It was written when the file was,
# and nothing ever produced sitemap.xml on the app site — the only one in the
# repo belonged to landing/, a stack disabled since the apex cutover — so every
# crawler that followed it got a 403. tools/build/build_sitemap.py generates it
# now, discovering the pages rather than listing them, and reads THIS file to
# decide what may appear in it. Adding a Disallow below therefore removes those
# URLs from the sitemap too; the two cannot drift apart.
Sitemap: https://hellodeutsch.com/sitemap.xml