# WealthWise OS - Robots.txt
# https://wealthwiseos.com
#
# ============================================================================
# GROUPING RULES — READ BEFORE EDITING
#
# Under the Robots Exclusion Protocol (RFC 9309 sec. 2.2.1), a crawler obeys
# EXACTLY ONE group: the one whose User-agent token matches it most
# specifically. It does NOT merge that group with `User-agent: *`, and it does
# NOT fall back to `*` for rules its own group omits.
#
# Two consequences, both of which have already caused live bugs here:
#
#   1. Putting Disallow rules after a specific agent scopes them to that agent
#      alone. A prior revision placed them after `User-agent: Applebot-Extended`,
#      so every other crawler received no Disallow at all.
#
#   2. Giving a named agent a group containing only `Allow: /` is WORSE than
#      not naming it. That agent then obeys only `Allow: /` and ignores every
#      Disallow below, so naming Googlebot that way grants it the private
#      surfaces the `*` group is trying to protect.
#
# Therefore: consecutive User-agent lines share the single rule block that
# follows them. Every crawler that is allowed to crawl is listed in the ONE
# group below, so the Disallow set applies to all of them by construction.
#
# To welcome a new crawler, add a `User-agent:` line to that group. Do NOT
# create a new group for it. The only standalone groups in this file are
# full-site blocks (`Disallow: /`), where no private-path rules are needed
# because nothing is crawlable.
# ============================================================================

# ========================================================================
# Allowed crawlers.
#
# `*` covers every crawler without a more specific token. The named agents
# below are listed explicitly so intent is documented — and because
# Google-Extended and Applebot-Extended are opt-out tokens where an explicit
# presence records a deliberate decision rather than a default.
#
# AI answer engines are allowed on purpose: blocking them does not protect the
# content, it only removes WealthWise from the answers they generate. Bulk
# training-corpus scrapers are declined separately, further down.
# ========================================================================
User-agent: *

# Traditional search
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot

# OpenAI — ChatGPT
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User

# Anthropic — Claude
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User

# Google — Gemini app and Vertex AI grounding.
# (AI Overviews are served from the core Googlebot index, not this agent.)
User-agent: Google-Extended

# Apple — Siri / Spotlight AI answers
User-agent: Applebot
User-agent: Applebot-Extended

# Microsoft Copilot
User-agent: Microsoft-Preview

# Meta AI
User-agent: meta-externalagent

# Mistral, You.com, DuckAssist
User-agent: MistralAI-User
User-agent: YouBot
User-agent: DuckAssistBot

# Amazon
User-agent: Amazonbot

# Social / link-preview crawlers — needed for rich unfurls
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Slackbot-LinkExpanding
User-agent: Discordbot

# ---- shared rule block for every agent listed above ----
Allow: /

# Private / authenticated surfaces. These serve the SPA shell (which renders as
# homepage content), so crawling them produces duplicate-content copies of "/"
# with no unique value. Also enforced by X-Robots-Tag in public/_headers.
Disallow: /api/
Disallow: /vault/
Disallow: /workspace/
Disallow: /profile/
Disallow: /seo/
Disallow: /analysis/
Disallow: /auth/
Disallow: /email-confirmed/

# Household invite redemption. The URL only means anything with a one-time
# token in its query string, it is single-use, and an indexed copy would put
# a live invite in a search result. Nothing links to it from the site.
Disallow: /family/join

# Never index query-string permutations of the app shell.
Disallow: /*?mode=

# ========================================================================
# Bulk training-corpus scrapers — declined.
#
# These build redistributable datasets rather than answering user queries, so
# blocking them costs no citation surface. Full-site blocks, so they need no
# private-path rules.
# ========================================================================
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

# ========================================================================
# Sitemap
# ========================================================================
Sitemap: https://wealthwiseos.com/sitemap.xml