# -------------------------------
# General rules for all bots
# -------------------------------
User-agent: *
Disallow: /isiteadmin
Disallow: /apply
Disallow: /careers.html
Disallow: /disclaimer.html
Disallow: /terms-of-service.html
Disallow: /blog/author
Disallow: /whats-happening.html?
Disallow: /enquiry_ajax.php
Disallow: /thank-you.html
Disallow: /send_newsletter.html


# -------------------------------
# AI Search Bots - allowed on public pages
# -------------------------------
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

# -------------------------------
# Normal SEO crawlers
# -------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# -------------------------------
# Sitemap
# -------------------------------
Sitemap: https://www.indiainternets.com/sitemap.xml