# All crawlers welcome — including AI crawlers (explicitly allowed below).
# JSON API and admin endpoints are fenced defensively (they never appear in the sitemap).
# Query-string variants (?format=, request_id=) duplicate clean URLs that already carry
# rel=canonical; they are blocked so crawl budget stays on the canonical pages in the sitemap.
# Next.js navigation prefetches each page's RSC payload as /<path>.txt?_rsc=<hash>, with a
# hash that differs per referring page. Rendering crawlers followed them (89% of
# OAI-SearchBot fetches, 2026-09-17..24). Only .txt requests with a query are fenced, so
# /llms.txt, /ads.txt and the IndexNow key file stay crawlable.
# Every sound page links Studio with its own ?base= or ?q= query (dozens per page). Each one is a
# new URL that renders the same app and pulls its scripts; they took about 41% of OAI-SearchBot
# fetches (2026-10-22..28) while family pages went unindexed. /studio itself stays crawlable.
# Rendering /studio pulls /api/config and /generate-starts.json, JSON for the app: with /studio and
# its icon they still took 23-61% of OAI-SearchBot fetches a day (2026-10-24..30) while set and
# guide pages were fetched 10 and 13 times in the week.
User-agent: *
Allow: /
Disallow: /api/v1/
Disallow: /api/admin/
Disallow: /admin/
Disallow: /*?*request_id=
Disallow: /*?format=
Disallow: /*&format=
Disallow: /*.txt?
Disallow: /studio?
Disallow: /api/config
Disallow: /generate-starts.json

# A crawler that matches a named group reads only that group, so each named group repeats
# the fences above instead of inheriting them from "*".
User-agent: GPTBot
Allow: /
Disallow: /api/v1/
Disallow: /api/admin/
Disallow: /admin/
Disallow: /*?*request_id=
Disallow: /*?format=
Disallow: /*&format=
Disallow: /*.txt?
Disallow: /studio?
Disallow: /api/config
Disallow: /generate-starts.json

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/v1/
Disallow: /api/admin/
Disallow: /admin/
Disallow: /*?*request_id=
Disallow: /*?format=
Disallow: /*&format=
Disallow: /*.txt?
Disallow: /studio?
Disallow: /api/config
Disallow: /generate-starts.json

User-agent: ChatGPT-User
Allow: /
Disallow: /api/v1/
Disallow: /api/admin/
Disallow: /admin/
Disallow: /*?*request_id=
Disallow: /*?format=
Disallow: /*&format=
Disallow: /*.txt?
Disallow: /studio?
Disallow: /api/config
Disallow: /generate-starts.json

User-agent: ClaudeBot
Allow: /
Disallow: /api/v1/
Disallow: /api/admin/
Disallow: /admin/
Disallow: /*?*request_id=
Disallow: /*?format=
Disallow: /*&format=
Disallow: /*.txt?
Disallow: /studio?
Disallow: /api/config
Disallow: /generate-starts.json

User-agent: PerplexityBot
Allow: /
Disallow: /api/v1/
Disallow: /api/admin/
Disallow: /admin/
Disallow: /*?*request_id=
Disallow: /*?format=
Disallow: /*&format=
Disallow: /*.txt?
Disallow: /studio?
Disallow: /api/config
Disallow: /generate-starts.json

User-agent: Google-Extended
Allow: /
Disallow: /api/v1/
Disallow: /api/admin/
Disallow: /admin/
Disallow: /*?*request_id=
Disallow: /*?format=
Disallow: /*&format=
Disallow: /*.txt?
Disallow: /studio?
Disallow: /api/config
Disallow: /generate-starts.json

User-agent: CCBot
Allow: /
Disallow: /api/v1/
Disallow: /api/admin/
Disallow: /admin/
Disallow: /*?*request_id=
Disallow: /*?format=
Disallow: /*&format=
Disallow: /*.txt?
Disallow: /studio?
Disallow: /api/config
Disallow: /generate-starts.json

User-agent: Bingbot
Allow: /
Disallow: /api/v1/
Disallow: /api/admin/
Disallow: /admin/
Disallow: /*?*request_id=
Disallow: /*?format=
Disallow: /*&format=
Disallow: /*.txt?
Disallow: /studio?
Disallow: /api/config
Disallow: /generate-starts.json

Sitemap: https://sfxmint.com/sitemap.xml