# Soft crawl budget for known bots (Crawl-delay is advisory; server also soft-limits via mod_qos → 429,
# then fail2ban after ~30 ignored 429s). Google ignores Crawl-delay; Awario and many others respect it.

# --- Major AI / search (welcome; polite pace) ---
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: OAI-AdsBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: Googlebot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-InspectionTool
User-agent: Google-CloudVertexBot
User-agent: Storebot-Google
User-agent: bingbot
User-agent: BingPreview
User-agent: AdIdxBot
User-agent: MicrosoftPreview
User-agent: Yandex
User-agent: YandexBot
User-agent: DuckDuckBot
User-agent: DuckAssistBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Amazonbot
User-agent: CCBot
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: MistralAI-User
Crawl-delay: 1
Allow: /
Disallow: /adm/
Disallow: /cache/
Disallow: /docs/
Disallow: /files/
Disallow: /includes/
Disallow: /language/
Disallow: /store/
Disallow: /styles/
Disallow: /common.php
Disallow: /config.php
Disallow: /cron.php
Disallow: /mcp.php
Disallow: /memberlist.php
Disallow: /posting.php
Disallow: /report.php
Disallow: /search.php
Disallow: /style.php
Disallow: /ucp.php
Disallow: /viewonline.php
Disallow: /*viewtopic.php?p=
Disallow: /*viewtopic.php?f=
Disallow: /*viewtopic.php?sid=
Disallow: /*viewtopic.php?view=
Disallow: /*viewtopic.php?watch=
Disallow: /*viewforum.php?f=*&sid=
Allow: /viewforum.php?f=
Allow: /viewtopic.php?t=

# --- Hungrier SEO / brand / research crawlers (slower please) ---
User-agent: AwarioBot
User-agent: AwarioSmartBot
User-agent: AwarioRssBot
User-agent: Reflectionbot
User-agent: Konturbot
User-agent: PetalBot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
User-agent: DotBot
User-agent: MJ12bot
User-agent: DataForSeoBot
User-agent: BLEXBot
User-agent: Barkrowler
User-agent: ZoominfoBot
User-agent: Brightbot
User-agent: ExaSearchBot
User-agent: SeznamBot
User-agent: MojeekBot
User-agent: Qwantify
User-agent: Baiduspider
User-agent: Sogou
User-agent: 360Spider
User-agent: GenomeCrawlerd
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: TikTokSpider
Crawl-delay: 2
Allow: /
Disallow: /adm/
Disallow: /cache/
Disallow: /docs/
Disallow: /files/
Disallow: /includes/
Disallow: /language/
Disallow: /store/
Disallow: /styles/
Disallow: /common.php
Disallow: /config.php
Disallow: /cron.php
Disallow: /mcp.php
Disallow: /memberlist.php
Disallow: /posting.php
Disallow: /report.php
Disallow: /search.php
Disallow: /style.php
Disallow: /ucp.php
Disallow: /viewonline.php
Disallow: /*viewtopic.php?p=
Disallow: /*viewtopic.php?f=
Disallow: /*viewtopic.php?sid=
Disallow: /*viewtopic.php?view=
Disallow: /*viewtopic.php?watch=
Disallow: /*viewforum.php?f=*&sid=
Allow: /viewforum.php?f=
Allow: /viewtopic.php?t=

# --- Everyone else ---
User-agent: *
Crawl-delay: 2
Disallow: /adm/
Disallow: /cache/
Disallow: /docs/
Disallow: /files/
Disallow: /includes/
Disallow: /language/
Disallow: /store/
Disallow: /styles/
Disallow: /common.php
Disallow: /config.php
Disallow: /cron.php
Disallow: /mcp.php
Disallow: /memberlist.php
Disallow: /posting.php
Disallow: /report.php
Disallow: /search.php
Disallow: /style.php
Disallow: /ucp.php
Disallow: /viewonline.php
Disallow: /*viewtopic.php?p=
Disallow: /*viewtopic.php?f=
Disallow: /*viewtopic.php?sid=
Disallow: /*viewtopic.php?view=
Disallow: /*viewtopic.php?watch=
Disallow: /*viewforum.php?f=*&sid=
Allow: /viewforum.php?f=
Allow: /viewtopic.php?t=

Sitemap: https://esp-forum.com/sitemap.xml

# https://esp-forum.com/
# AI overview (not a Sitemap directive; fetch directly):
# https://esp-forum.com/llms.txt
# https://esp-forum.com/.well-known/llms.txt