# SAM.gov Hunter — robots.txt (v5.8.0, 2026-05)
# Strategy: ALLOW LLM training + retrieval bots, ALLOW search engines,
# BLOCK SEO scrapers and known-abusive crawlers. Protect /api, /admin,
# /account, /partner-dashboard, /active-bids* from any indexing.

# ─── Search engines ────────────────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /partner-dashboard/
Disallow: /active-bids
Disallow: /active-bids-kanban
Disallow: /proposal/

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /partner-dashboard/
Disallow: /active-bids
Disallow: /active-bids-kanban
Disallow: /proposal/

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# ─── LLM training crawlers ────────────────────────────
# Lets us appear in next-gen ChatGPT/Claude/Gemini/Llama training corpora.
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /partner-dashboard/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /partner-dashboard/

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /partner-dashboard/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /admin/

# ─── Live-retrieval / citation crawlers ───────────────
# These drive referral traffic when an LLM cites samgov-hunter.com.
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: Amzn-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: GoogleOther
Allow: /
Disallow: /api/
Disallow: /admin/

# ─── Social link previews ─────────────────────────────
User-agent: FacebookExternalHit
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

# ─── Blocked: SEO scrapers ────────────────────────────
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ─── Blocked: known abusive crawlers ──────────────────
User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# ─── Default for everything else ──────────────────────
# Block API + admin + tenant data routes for any bot we
# haven't explicitly allowed above.
User-agent: *
Disallow: /api/
Disallow: /admin/
Disallow: /partner-dashboard/
Disallow: /active-bids
Disallow: /active-bids-kanban
Disallow: /proposal/

Sitemap: https://www.samgov-hunter.com/sitemap.xml