# ============================================================
# Privacy Insight Solutions — robots.txt
# Last updated: 2026-02-23
# ============================================================

# ── Google ──────────────────────────────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /cdn-cgi/

User-agent: Googlebot-Image
Allow: /

# ── Bing / Microsoft ────────────────────────────────────────
User-agent: Bingbot
Allow: /
Disallow: /cdn-cgi/
Crawl-delay: 2

User-agent: msnbot
Allow: /
Disallow: /cdn-cgi/

# ── Other reputable crawlers ─────────────────────────────────
User-agent: DuckDuckBot
Allow: /
Disallow: /cdn-cgi/

User-agent: Baiduspider
Allow: /
Disallow: /cdn-cgi/
Crawl-delay: 5

User-agent: YandexBot
Allow: /
Disallow: /cdn-cgi/
Crawl-delay: 5

# ── SEO audit bots (slow them down, don't ban) ──────────────
# These are legitimate but aggressively crawl and waste budget.
User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /cdn-cgi/

User-agent: SemrushBot
Crawl-delay: 10
Disallow: /cdn-cgi/

User-agent: MJ12bot
Crawl-delay: 10
Disallow: /cdn-cgi/

User-agent: DotBot
Crawl-delay: 10
Disallow: /cdn-cgi/

# ── AI training scrapers — block entirely ───────────────────
# These offer no SEO benefit and consume bandwidth.
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

# ── Default: allow everything else ──────────────────────────
User-agent: *
Allow: /
Disallow: /blog-article.html

# ── Sitemap ─────────────────────────────────────────────────
Sitemap: https://privacyinsightsolutions.com/sitemap.xml