# A vain attempt to stop robots from walking through the structure
User-Agent: *
Disallow: /cgi-bin/
Disallow: /ispcol/
Disallow: /drafts/
Disallow: /ietf/
Disallow: /papers/
Disallow: /tools/
Crawl-delay: 10


# https://www.semrush.com/bot/
User-agent: SemrushBot
Disallow: /

# https://www.semrush.com/bot/
User-agent: SiteAuditBot
Disallow: /

# https://www.semrush.com/bot/
User-agent: SemrushBot-BA
Disallow: / 

# https://www.semrush.com/bot/
User-agent: SemrushBot-SI
Disallow: / 

# https://www.semrush.com/bot/
User-agent: SemrushBot-SWA
Disallow: / 

# https://www.semrush.com/bot/
User-agent: SplitSignalBot
Disallow: / 

# https://www.semrush.com/bot/
User-agent: SemrushBot-OCOB
Disallow: / 

# https://www.semrush.com/bot/
User-agent: SemrushBot-FT
Disallow: / 

# https://www.semrush.com/bot/
User-agent: RyteBot
Disallow: / 

# https://www.semrush.com/bot/
User-agent: SemrushBot-ESI
Disallow: / 

# https://developer.amazon.com/amazonbot
User-agent: Amazonbot
Disallow: /

# https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
User-agent: meta-externalagent
Disallow: /

# https://developers.openai.com/api/docs/bots
User-agent: GPTBot
Disallow: /

# https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
User-agent: ClaudeBot
Disallow: /