User-agent: *

Allow: /
 
# Amazon bots
User-agent: Amazonbot
Allow: /
 
# Anthropic AI bots
User-agent: Anthropic-ai
Allow: /
 
# Apple bots
User-agent: Applebot-Extended
Allow: /
 
# Awario bots
User-agent: AwarioRssBot
Allow: /
User-agent: AwarioSmartBot
Allow: /
 
# ByteDance bot
User-agent: Bytespider
Allow: /
 
# Common Crawl bot
User-agent: CCBot
Allow: /
 
# OpenAI and ChatGPT bots
User-agent: ChatGPT-User
Allow: /
User-agent: GPTBot
Allow: /
 
# Claude bots (Anthropic)
User-agent: Claude
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /
 
# Cohere AI bot
User-agent: Cohere-ai
Allow: /
 
# DataForSeo bot
User-agent: DataForSeoBot
Allow: /
 
# Meta (Facebook) bots
User-agent: FacebookBot
Allow: /
User-agent: Meta AI
Allow: /
 
# Google bots
User-agent: Google-Extended
Allow: /
User-agent: Google Gemini
Allow: /
 
# Grok bot (Samsung)
User-agent: Grok
Allow: /
 
# Imagesift bot
User-agent: ImagesiftBot
Allow: /
 
# Magpie bot
User-agent: Magpie-crawler
Allow: /
 
# Microsoft bots
User-agent: Microsoft Copilot
Allow: /
 
# Omgili bots
User-agent: Omgili
Allow: /
User-agent: Omgilibot
Allow: /
 
# Peer39 crawlers
User-agent: Peer39_crawler
Allow: /
User-agent: Peer39_crawler/1.0
Allow: /
 
# Perplexity bot
User-agent: PerplexityBot
Allow: /
 
# You.com bot
User-agent: YouBot
Allow: /

User-agent: *

Disallow: /

Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /wp-includes
Disallow: /wp-content/plugins
Disallow: /wp-content/cache
Disallow: /wp-content/themes
Disallow: /comments
Disallow: /category/*/*
Disallow: */trackback
Disallow: */feed
Disallow: */comments
Disallow: /*?*
Disallow: /*?
Disallow: /websites

User-Agent: trendictionbot 
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Qwantify
Disallow: /

User-agent: LinksCrawler
Disallow: /

User-agent: DeuSu
Disallow: /

User-agent: oBot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: MSIE8.0
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: archive.org_bot 
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: special_archiver
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DuckDuckGo-Favicons-Bot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: ccbot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: netEstate
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: dotbot
Disallow: /