# ============================================================
# robots.txt  -  ModelCars.com
# ============================================================
# NOTE ON HOW THIS FILE WORKS:
# A crawler obeys exactly ONE user-agent group  -  the most
# specific match  -  and ignores all others, including "*".
# So any bot given its own group must have EVERY rule repeated
# inside it. That is why there are no "Allow: /" groups here:
# a bot you want to allow is already allowed by "*", and giving
# it its own group would strip the duplicate-URL protections.
# ============================================================


# --- Sitemap (global directive) ---
Sitemap: https://modelcars.com/sitemap.xml


# ============================================================
# FULLY BLOCKED  -  aggressive crawlers, no commercial value
# ============================================================

# ByteDance (TikTok). Extremely aggressive, runs a real browser
# engine so it shows up in GA4. Prime suspect for the
# Singapore / Vietnam / China session volume.
User-agent: Bytespider
Disallow: /

# Huawei. Same profile  -  heavy crawl, APAC origin, no buyers.
User-agent: PetalBot
Disallow: /

# --- Chinese search engines  -  store does not ship to China ---
User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider-image
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Sogou inst spider
Disallow: /

User-agent: Sogou spider2
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: HaosouSpider
Disallow: /

User-agent: Yisouspider
Disallow: /

# --- Russian search engines  -  store does not ship to Russia ---
User-agent: Yandex
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: YandexImages
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

# Backlink / SEO scrapers with no search value
User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Timpibot
Disallow: /

# Generic scraping frameworks
User-agent: Nutch
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: python-requests
Disallow: /


# ============================================================
# NOTE: On-demand audit bots (AhrefsSiteAudit, SiteAuditBot) are
# deliberately NOT given their own groups. They only run when you
# or your agency trigger an audit, so rate-limiting them just makes
# your own audits slower. They inherit the full "*" rules below.
#
# RATE-LIMITED  -  SEO tools your agency may legitimately use.
# Path rules repeated because these groups override "*".
# ============================================================

User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /admin
Disallow: /account
Disallow: /orders
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /password
Disallow: /search
Disallow: /policies/
Disallow: /sf_private_access_tokens
Disallow: /cdn/
Disallow: /*?*preview_theme_id*
Disallow: /*?*preview_script_id*
Disallow: /*?*oseid=*
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: /collections/*filter*
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /products/*-remote$
Disallow: /*/products/*-remote$
Disallow: /collections/*/products/*-remote$

User-agent: SemrushBot
Crawl-delay: 10
Disallow: /admin
Disallow: /account
Disallow: /orders
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /password
Disallow: /search
Disallow: /policies/
Disallow: /sf_private_access_tokens
Disallow: /cdn/
Disallow: /*?*preview_theme_id*
Disallow: /*?*preview_script_id*
Disallow: /*?*oseid=*
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: /collections/*filter*
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /products/*-remote$
Disallow: /*/products/*-remote$
Disallow: /collections/*/products/*-remote$

User-agent: MJ12bot
Crawl-delay: 10
Disallow: /admin
Disallow: /account
Disallow: /orders
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /password
Disallow: /search
Disallow: /policies/
Disallow: /sf_private_access_tokens
Disallow: /cdn/
Disallow: /*?*preview_theme_id*
Disallow: /*?*preview_script_id*
Disallow: /*?*oseid=*
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: /collections/*filter*
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /products/*-remote$
Disallow: /*/products/*-remote$
Disallow: /collections/*/products/*-remote$


# ============================================================
# EVERYONE ELSE  -  Googlebot, Bingbot, DuckDuckBot, GPTBot,
# ClaudeBot, PerplexityBot, Applebot, Amazonbot, FacebookBot,
# etc. all land here and inherit the full protection set.
# ============================================================

User-agent: *

# Admin & customer accounts
Disallow: /admin
Disallow: /account
Disallow: /orders

# Cart & checkout
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /password

# Internal search & policies
Disallow: /search
Disallow: /policies/

# Shopify system URLs
Disallow: /sf_private_access_tokens
Disallow: /cdn/
Disallow: /*?*preview_theme_id*
Disallow: /*?*preview_script_id*
Disallow: /*?*oseid=*

# Collection duplicate-URL prevention  -  DO NOT REMOVE
# (sorted, tag-filtered, and faceted variants of the same page)
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: /collections/*filter*

# Blog duplicate-URL prevention
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*

# Shopify cross-region duplicate product URLs
# (robots.txt supports only * and $  -  character classes do not work)
Disallow: /products/*-remote$
Disallow: /*/products/*-remote$
Disallow: /collections/*/products/*-remote$