# =============================================================
# silentfox.space — robots.txt
# Spec: https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
# Updated: 2026-05-26
# =============================================================

# ── Search engine indexers ──────────────────────────────────────
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: Applebot

# SvelteKit bundles JS + CSS under /_app/. Without an explicit Allow,
# some crawl infrastructure treats paths starting with _ as private.
# Googlebot must render these assets to evaluate page quality correctly.
Allow: /_app/
Allow: /

# App shell routes — auth-only, zero indexable content.
# These serve the SPA fallback HTML with no unique <title>, <h1>, or
# meta description. Indexing them triggers thin-content penalties.
Disallow: /auth/
Disallow: /now/
# /home/ is the app's Today surface and the single most-linked internal
# target on the site — 695 public pages carry a link to it. It is an app
# route, so adapter-static never emits a file for it, and every crawl of
# those 695 links resolved to the 404 fallback. Same category as /now/
# and /you/; it was simply missed. /archive/, /decide/, /memory/ and
# /state/ are the same: auth-only surfaces with nothing to index.
Disallow: /home/
Disallow: /archive/
Disallow: /decide/
Disallow: /memory/
Disallow: /state/
Disallow: /forecast/
Disallow: /charts/
Disallow: /cycles/
Disallow: /psyche/
Disallow: /you/
Disallow: /onboarding/
Disallow: /insights/
Disallow: /decisions/
Disallow: /synastry/
Disallow: /saved/
Disallow: /share/
Disallow: /upgrade/
Disallow: /settings/
Disallow: /api/
Disallow: /cdn-cgi/

# ── AI SEARCH / INFERENCE — ALLOWED (drives AI Overview / ChatGPT / Perplexity citations) ──
# These bots surface pages in AI-generated answers and citations; allowing them
# is required for visibility in AI-powered search products.
# Applebot-Extended is the same category as Google-Extended, not a separate
# crawler: it governs whether content Applebot already indexed may be used in
# Apple Intelligence / Siri's generative features, same as Google-Extended does
# for Gemini/AI Overviews. Added 2026-09-02 — the one AI-search control token
# missing from an otherwise complete list.
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /

# ── AI TRAINING / SCRAPERS — BLOCKED ────────────────────────────
# Training crawlers harvest content for model training datasets.
# GPTBot and CCBot are explicitly disallowed (training, not search).
User-agent: GPTBot
User-agent: CCBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: Bytespider
User-agent: Omgilibot
User-agent: Diffbot
User-agent: img2dataset
User-agent: Timpibot
User-agent: YouBot
Disallow: /

# ── Junk / content-scraping bots ────────────────────────────────
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: PetalBot
User-agent: Sogou Spider
User-agent: Exabot
User-agent: ia_archiver
Disallow: /

# ── Default: everything else ───────────────────────────────────
User-agent: *
Allow: /_app/
Allow: /
Disallow: /auth/
Disallow: /now/
Disallow: /home/
Disallow: /archive/
Disallow: /decide/
Disallow: /memory/
Disallow: /state/
Disallow: /forecast/
Disallow: /charts/
Disallow: /cycles/
Disallow: /psyche/
Disallow: /you/
Disallow: /onboarding/
Disallow: /insights/
Disallow: /decisions/
Disallow: /synastry/
Disallow: /saved/
Disallow: /share/
Disallow: /upgrade/
Disallow: /settings/
Disallow: /api/
Disallow: /cdn-cgi/

Sitemap: https://silentfox.space/sitemap.xml