# =====================================================================
# robots.txt — highradius.com
# Purpose: Maximize discoverability across Search (Google, Bing) and
#          AI Answer Engines (OpenAI, Anthropic, Perplexity, Gemini)
#          while universally protecting gated, private, and duplicate paths.
# Last updated: 2026-07-01
# =====================================================================

# ---------------------------------------------------------------------
# GROUP 1: ALL PERMITTED SEO & AI CRAWLERS (Search, Retrieval, & Training)
# Binds all high-value bots to the same strict crawl budget rules.
# ---------------------------------------------------------------------
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: YouBot
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: Amazonbot
Allow: /
Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php

# --- Gated, Internal, and Authenticated App Paths ---
Disallow: /login/
Disallow: /signin/
Disallow: /account/
Disallow: /dashboard/
Disallow: /app/
Disallow: /portal/
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /api/

# --- Staging, Development, and Tooling Outpoints ---
Disallow: /staging/
Disallow: /dev/
Disallow: /test/
Disallow: /internal/

# --- Lead-Gen / Transactional Cleanups (Avoid duplicate form indexing) ---
Disallow: /thank-you/
Disallow: /thankyou/
Disallow: /cart/
Disallow: /checkout/
Disallow: /*?lead=*
Disallow: /*?form=*

# --- Crawl Budget Savers: Search, Sorting, and Trackers ---
Disallow: /search/
Disallow: /*?s=
Disallow: /*?q=
Disallow: /*?filter=
Disallow: /*?sort=
Disallow: /*utm_*
Disallow: /*?ref=*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*

# ---------------------------------------------------------------------
# GROUP 2: EXPLICITLY BLOCKED LOW-VALUE AI SCRAPERS
# High crawl overhead with zero referral value or traffic generation.
# ---------------------------------------------------------------------
User-agent: CCBot
User-agent: Bytespider
User-agent: Diffbot
User-agent: cohere-ai
User-agent: omgili
User-agent: omgilibot
Disallow: /

# ---------------------------------------------------------------------
# GROUP 3: CATCH-ALL DEFAULT RULE
# Applies the protection matrix to any future or unlisted crawlers.
# ---------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /login/
Disallow: /signin/
Disallow: /account/
Disallow: /dashboard/
Disallow: /app/
Disallow: /portal/
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /api/
Disallow: /staging/
Disallow: /dev/
Disallow: /test/
Disallow: /internal/
Disallow: /thank-you/
Disallow: /thankyou/
Disallow: /cart/
Disallow: /checkout/
Disallow: /*?lead=*
Disallow: /*?form=*
Disallow: /search/
Disallow: /*?s=
Disallow: /*?q=
Disallow: /*?filter=
Disallow: /*?sort=
Disallow: /*utm_*
Disallow: /*?ref=*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*

# ---------------------------------------------------------------------
# SECTION 4: DEPLOYMENT SPECIFIC MAPS & MASTER SITEMAPS INDEX
# ---------------------------------------------------------------------
# LLM Index Discovery
# llms.txt: https://www.highradius.com/llms.txt

# Global & Product Child Sitemaps
Sitemap: https://www.highradius.com/sitemap.xml
Sitemap: https://www.highradius.com/global-pages-sitemap.xml
Sitemap: https://www.highradius.com/software-pages-o2c-sitemap.xml
Sitemap: https://www.highradius.com/software-pages-ap-sitemap.xml
Sitemap: https://www.highradius.com/software-pages-r2r-sitemap.xml
Sitemap: https://www.highradius.com/software-pages-treasury-sitemap.xml
Sitemap: https://www.highradius.com/software-pages-b2b-sitemap.xml

# Resources & Inbound Marketing Assets
Sitemap: https://www.highradius.com/resources/resource-pages-sitemap.xml
Sitemap: https://www.highradius.com/resources/blog-pages-sitemap.xml

# Regional & International Locales
Sitemap: https://www.highradius.com/en-gb/sitemap.xml
Sitemap: https://www.highradius.com/fr/sitemap.xml
Sitemap: https://www.highradius.com/de/sitemap.xml