# =========================================================================
# robots.txt for ensmart.ai
# EnSmart Technologies — SmartNova Platform
# Last revised: May 2026
# =========================================================================
#
# Policy summary:
#   1. All public pages are crawlable by every search engine and AI bot.
#   2. Internal/admin paths (APIs, lang JSON, internal helpers, sitemap
#      generator) are blocked from every crawler.
#   3. Three abusive crawlers (MJ12bot, DotBot, BLEXBot) are fully banned.
#   4. AhrefsBot and SemrushBot are throttled to 10-second crawl delay.
#
# Note on AI search bots (May 2026 revision):
#   Google-Extended, PerplexityBot, GPTBot, ClaudeBot, anthropic-ai,
#   ChatGPT-User, OAI-SearchBot, Applebot-Extended, CCBot, Bytespider,
#   Meta-ExternalAgent — and any other AI search/training crawler — are
#   NOT given separate User-agent blocks below. By omission, they inherit
#   the universal User-agent: * rules, meaning they can crawl every public
#   page on ensmart.ai except the internal paths listed below.
#
#   Previously these AI bots had User-agent blocks that restricted them
#   to /bms-library/ only. That was over-protective and cut off
#   AI Overview / Perplexity / ChatGPT search visibility for product,
#   solution, case-study, and blog pages — pages that are intentionally
#   public and are key to brand discoverability through AI search.
# =========================================================================

User-agent: *
Allow: /
Disallow: /blog-api.php
Disallow: /scanner-api.php
Disallow: /scanner/data/
Disallow: /scanner/files/
Disallow: /assets/private/
Disallow: /sitemap-generator.php
Disallow: /bms-library/_lib_*.php
Disallow: /bms-library/lang/
Disallow: /bms-library/ask
Disallow: /bms-library/search

# Block faceted / parametrised duplicates (canonical points to the clean URL)
Disallow: /bms-library/category/*?sort=
Disallow: /bms-library/category/*?lang=
Disallow: /bms-library/qa/*?lang=
Disallow: /bms-library/?lang=

# Admin-only / lead-tracking endpoints (also marked noindex in HTML)
Disallow: /leads-dashboard.php
Disallow: /leads-api.php
Disallow: /contact-us/contact-api.php
Disallow: /contact-us/data/
Disallow: /case-studies/case-study-api.php
Disallow: /case-studies/data/
Disallow: /data/

# =========================================================================
# Bad bots — fully blocked (excessive load, no SEO benefit)
# =========================================================================

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# =========================================================================
# Aggressive but useful crawlers — throttled
# =========================================================================

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

# =========================================================================
# Sitemaps
# =========================================================================

Sitemap: https://ensmart.ai/sitemap.xml
Sitemap: https://ensmart.ai/bms-library/sitemap.xml