# Catalogue pages may be indexed, but automated crawlers should avoid
# customer-specific and expensive dynamic pages.
User-agent: *
Disallow: /download/
Disallow: /index.php?main_page=account
Disallow: /index.php?main_page=advanced_search
Disallow: /index.php?main_page=advanced_search_result
Disallow: /index.php?main_page=checkout
Disallow: /index.php?main_page=login
Disallow: /index.php?main_page=shopping_cart

# Google advertising and shopping crawlers may access the full site. A very
# high emergency server limit remains in place to stop spoofed user agents.
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: Storebot-Google
User-agent: Google-Shopping
Allow: /

# Google Search is allowed, but asked to crawl conservatively. Google may
# ignore Crawl-delay; the server also enforces a burst limit with HTTP 429.
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: GoogleOther
Crawl-delay: 3
Disallow: /download/
Disallow: /index.php?main_page=account
Disallow: /index.php?main_page=advanced_search
Disallow: /index.php?main_page=advanced_search_result
Disallow: /index.php?main_page=checkout
Disallow: /index.php?main_page=login
Disallow: /index.php?main_page=shopping_cart

# AI discovery tools can index public products and categories. Crawl-delay is
# advisory; the server also enforces a burst limit and returns HTTP 429.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
Crawl-delay: 5
Disallow: /download/
Disallow: /index.php?main_page=account
Disallow: /index.php?main_page=advanced_search
Disallow: /index.php?main_page=advanced_search_result
Disallow: /index.php?main_page=checkout
Disallow: /index.php?main_page=login
Disallow: /index.php?main_page=shopping_cart

# Low-value SEO crawlers are blocked by Apache before PHP starts.
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: DotBot
User-agent: MJ12bot
Disallow: /