User-Agent: *
Disallow: /admin
Disallow: /admin_blog
Disallow: /cgi_bin
Disallow: /PHPMailer
Disallow: /.well-known
Disallow: /uploaded_slider


# ===============================
# Allowed (important search bots)
# ===============================
User-agent: Googlebot
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: YandexBot
Allow: /

# ===============================
# Aggressive crawlers (BLOCK)
# ===============================
User-agent: AhrefsBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DuckDuckBot
Disallow: /

User-agent: crawler
Disallow: /

User-agent: Apache-HttpClient
Disallow: /

User-agent: Unknown
Disallow: /

# ===============================
# Throttle (useful but heavy)
# ===============================

User-agent: crawl
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 5

User-agent: GPTBot
Crawl-delay: 5

User-agent: DataForSeoBot
Crawl-delay: 5

User-agent: Amazonbot
Crawl-delay: 5


