# ════════════════════════════════════════════════════════════════════════
#  robots.txt — kinrehab.com
#  Last Update: 2026-08-18
#  Policy:
#    - Allow AI training crawlers + search engines to index PUBLIC content
#    - Block admin panel, REST API, internal assets, internal storage
#
#  IMPORTANT — keep these crawlable for SEO/AEO/GEO:
#    - /upload/seo/      → og:image / twitter:image (social share preview,
#                          AI vision context for ChatGPT/Claude/Perplexity)
#    - /upload/news/     → news thumbnails (article hero + card grid)
#    - /upload/service/  → service card images (referenced from CMS bodies)
#    - /assets/frontend/ → public CSS/JS — Googlebot needs these to render
#                          pages correctly during indexing
#
#  ⚠️ robots.txt rules do NOT inherit between User-agent blocks. The most
#     specific matching block wins; the `*` block does NOT apply to named
#     bots below. That's why the Disallow list is repeated under each
#     user-agent — same content, explicit per-bot.
#
#  2026-08-18 — added 5 live-fetch / answer-engine agents.
#     Training crawlers (GPTBot, ClaudeBot, CCBot) are NOT the same agents
#     that fetch a page while a user is asking. Those are the ones below.
#     They were already allowed via `*`, but naming them keeps them safe if
#     anyone ever adds a Disallow under `*`.
# ════════════════════════════════════════════════════════════════════════

# ── Default (all bots without their own block below) ─────────────────────
User-agent: *
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

Sitemap: https://kinrehab.com/sitemap.xml

# ── AI training crawlers — opt-in to public content, blocked from admin ──
User-agent: GPTBot
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

User-agent: ClaudeBot
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

User-agent: CCBot
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

User-agent: PerplexityBot
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

User-agent: Google-Extended
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

User-agent: Applebot-Extended
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

# ── Answer-engine indexers + live fetch (added 2026-08-18) ───────────────
# OAI-SearchBot   = builds the ChatGPT Search index (different job from GPTBot)
User-agent: OAI-SearchBot
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

# ChatGPT-User    = fetches a page in real time while a user is asking
User-agent: ChatGPT-User
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

# Perplexity-User = same idea on the Perplexity side
User-agent: Perplexity-User
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

# meta-externalagent = Meta AI inside Messenger / WhatsApp / Instagram
User-agent: meta-externalagent
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

# Amazonbot = Alexa / Rufus answers
User-agent: Amazonbot
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

# ── Search engine crawlers ───────────────────────────────────────────────
User-agent: Bingbot
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/

User-agent: Googlebot
Allow: /
Disallow: /portal/
Disallow: /api/
Disallow: /assets/admin/
Disallow: /storage/