User-agent: *
Allow: /
# Keep crawlers out of the faceted/paginated/sorted URL variants. Every canonical page is path-based
# and stays indexable; the query string is only ever pagination, sorting, date and filter params,
# which generate thousands of near-duplicate URLs per stock and waste crawl budget.
Disallow: /*?
# EXCEPT the most-shorted ranking's variants: Google picked up param'd most-shorted URLs from
# links and, unable to crawl them, indexed them URL-only — no title, no snippet, zero clicks
# guaranteed. The blanket Disallow also hid the page's self-referencing canonical, so Google
# could never consolidate the variants onto the clean URL. This longest-match Allow lets it
# read the canonical; the variant count is bounded (dates × sorts × pages), not per-stock.
Allow: /stocks/most-shorted?
# AND except the static bundles. They are served with a ?v=<hash> cache-buster, so the blanket
# Disallow above matches them and stops Google fetching the CSS/JS it renders the page with —
# which degrades the layout and mobile-usability assessment even though the HTML is
# server-rendered. Longest-match wins: these patterns are longer than "/*?" and so take precedence.
Allow: /dist/
Allow: /images/
# CSV exports are data downloads, not pages to index.
Disallow: /*.csv$

# Explicitly welcome AI/LLM crawlers to the canonical (path-based) pages.
#
# These share ONE group deliberately. A named user-agent group does NOT inherit from "*" — the
# most specific matching group wins outright — so while each agent had its own copy of the rules,
# later edits to "*" silently skipped them all (that is how "Allow: /stocks/most-shorted?" came to
# apply to Googlebot but not to any of these). Consecutive User-agent lines share the rules that
# follow, so the rules are written once and every agent below gets identical treatment.
# Whatever changes in "*" above must change here too, or delete this block — it grants nothing
# that "*" does not already grant.
User-agent: GPTBot
# OpenAI's search crawler, separate from GPTBot training and ChatGPT-User fetches.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Anthropic's current crawlers. Claude-Web, listed here until now, is retired.
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
# Perplexity's indexing crawler, then its user-initiated fetch agent.
User-agent: PerplexityBot
User-agent: Perplexity-User
# Google's AI (Gemini / Vertex) training crawler, separate from Googlebot search indexing.
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: cohere-ai
Allow: /
Disallow: /*?
Allow: /stocks/most-shorted?
Allow: /dist/
Allow: /images/
Disallow: /*.csv$

Sitemap: https://equibles.com/sitemap.xml