# ===============================
# Default: Apply 60s delay to all bots
# ===============================
User-agent: *
Crawl-delay: 900
Disallow: /test/
Disallow: /common/
Disallow: /art/services/print.php
Disallow: /article/ref_content
Disallow: /article/liv_e
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /xmlrpc.php

# ===============================
# Google (Ignores Crawl-delay in file)
# ===============================
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
Crawl-delay: 600
Disallow: /rss/
Disallow: /search/
# To limit Google's crawl rate, use Google Search Console > Settings.

# ===============================
# Bing (Honors Crawl-delay)
# ===============================
User-agent: Bingbot
Crawl-delay: 600
Disallow:

# ===============================
# Social & Indexer Bots (Honors Delay)
# ===============================
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: LinkedInBot
User-agent: redditbot
Crawl-delay: 900
Disallow:

# ===============================
# BLOCK AI / SEO SCRAPERS (Aggressive Bots)
# ===============================
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: CCBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: SemrushBot
User-agent: PetalBot
User-agent: Amazonbot
User-agent: trendictionbot
Crawl-delay: 900
Disallow: /

# ===============================
# Sitemaps
# ===============================
Sitemap: https://www.ibtimes.co.in/news.xml
Sitemap: https://www.ibtimes.co.in/sitemap.xml