# https://www.robotstxt.org/robotstxt.html
#
# STRUCTURE MATTERS HERE. Per RFC 9309 section 2.2.1 (and Google's own docs) a
# crawler obeys ONLY the most specific group whose user-agent matches it, and
# ignores `User-agent: *` entirely. This file used to declare ~35 individual
# groups that each contained a bare `Allow: /` and no Disallow lines, which
# meant Googlebot, Bingbot, GPTBot, ClaudeBot and every other named crawler was
# exempt from the `Disallow: /api/` and the UTM/ref parameter rules below —
# i.e. the rules applied to nobody who mattered. Fixed 8 Sep 2026.
#
# The named agents are kept (they are a deliberate opt-in signal for AI
# crawlers) but STACKED into one group so a single rule block covers them all,
# mirroring the structure visylix.com already used. If you add a crawler, add
# its User-agent line to the stack — do not give it its own group unless you
# also repeat every Disallow.
User-agent: *
Allow: /
# Block API endpoints (HubSpot proxies, Turnstile verify, IndexNow trigger, etc.)
Disallow: /api/
# Block UTM and tracking parameter URLs
Disallow: /*?utm_source=*
Disallow: /*?utm_campaign=*
Disallow: /*?utm_medium=*
Disallow: /*?ref=*
# ── Search engine crawlers ────────────────────────
User-agent: Googlebot
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
# ── AI / LLM crawlers (2026 complete list) ────────
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Bytespider
User-agent: cohere-ai
User-agent: Amazonbot
User-agent: YouBot
User-agent: PhindBot
User-agent: DeepSeekBot
User-agent: CCBot
User-agent: GeminiBot
User-agent: MistralBot
User-agent: Grok
User-agent: xAI-Grok
User-agent: AI2Bot
User-agent: Diffbot
User-agent: Webz.io
User-agent: iaskspider
User-agent: PetalBot
# ── Social / link preview crawlers ────────────────
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai.txt
Disallow: /api/
Disallow: /*?utm_source=*
Disallow: /*?utm_campaign=*
Disallow: /*?utm_medium=*
Disallow: /*?ref=*
Sitemap: https://aptibit.com/sitemap.xml
# Note: feed.xml is the RSS feed (discovered via the in ),
# not a sitemap. Declaring it as Sitemap: made crawlers try to parse RSS as a sitemap
# and fail ("Sitemap in the wrong format"). RSS auto-discovery handles it correctly.
#
# AI crawler context, updated March 2026
# See https://llmstxt.org for the llms.txt standard
# llms.txt: https://aptibit.com/llms.txt
# llms-full.txt: https://aptibit.com/llms-full.txt
# ai.txt: https://aptibit.com/ai.txt