# =========================================
# Wild Boar ATV Parts - robots.txt
# Updated: 2026-05-20
# =========================================


User-agent: *
Allow: /

# --- Internal / low-value pages ---
Disallow: /checkout/
Disallow: /account/
Disallow: /login
Disallow: /register
Disallow: /wishlist
Disallow: /compare
Disallow: /search

# --- Parameter URLs (both ?param= and &param= variants) ---
Disallow: /*?search=
Disallow: /*&search=
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?order=
Disallow: /*&order=
Disallow: /*?limit=
Disallow: /*&limit=
Disallow: /*?filter=
Disallow: /*&filter=
Disallow: /*?tracking=
Disallow: /*&tracking=
Disallow: /*?utm_
Disallow: /*&utm_
Disallow: /*?gclid=
Disallow: /*&gclid=
Disallow: /*?fbclid=
Disallow: /*&fbclid=
Disallow: /*?tag=
Disallow: /*&tag=

# --- OpenCart internal routes ---
Disallow: /index.php?route=

# Note: /cart/ is intentionally NOT blocked. Legacy /cart/ URLs now
# 301-redirect to clean product URLs - Googlebot must crawl them to
# discover the redirects and clear the 404s from the index.

# =========================================
# AI crawlers - throttled (content allowed)
# =========================================

# OAI-AdsBot validates ChatGPT ad landing pages - allow, do NOT throttle (added jun24-2026, OpenAI advertiser guidance)
User-agent: OAI-AdsBot
Allow: /

User-agent: GPTBot
Crawl-delay: 10

User-agent: OAI-SearchBot
Crawl-delay: 10

User-agent: ChatGPT-User
Crawl-delay: 10

User-agent: ClaudeBot
Crawl-delay: 10

User-agent: Claude-User
Crawl-delay: 10

User-agent: PerplexityBot
Crawl-delay: 10

User-agent: Perplexity-User
Crawl-delay: 10

User-agent: DuckAssistBot
Crawl-delay: 10

# =========================================
# AI training crawlers - allowed
# =========================================

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# =========================================
# High-volume bots - strict throttle
# =========================================

User-agent: Amazonbot
Crawl-delay: 30

User-agent: meta-externalagent
Crawl-delay: 30

User-agent: meta-externalfetcher
Crawl-delay: 30

User-agent: CCBot
Crawl-delay: 30

# =========================================
# Ads & Social crawlers - full access (you run Google + Meta ads)
# =========================================

User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: meta-externalads
Allow: /

User-agent: facebookexternalhit
Allow: /

# =========================================
# Sitemap
# =========================================


Sitemap: https://www.wildboaratvparts.com/sitemap.xml
Sitemap: https://www.wildboaratvparts.com/blog/sitemap_index.xml
Sitemap: https://www.wildboaratvparts.com/llms.txt