# suganta.com robots.txt — corrected April 2026

# Google search & AI overviews
User-agent: Googlebot
User-agent: Google-Extended
Allow: /
Allow: /storage/sitemaps/
Disallow: /admin/
Disallow: /api/
Allow: /*?page=
Disallow: /*?

# ChatGPT Search retrieval (allows recommendations)
User-agent: OAI-SearchBot
Allow: /
Allow: /storage/sitemaps/

# Other AI search engines
User-agent: PerplexityBot
User-agent: YouBot
User-agent: Applebot-Extended
Allow: /
Allow: /storage/sitemaps/

# Block AI training scrapers (no traffic benefit)
User-agent: GPTBot
User-agent: CCBot
Disallow: /

# SEO crawlers — slow down instead of full block
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MajesticBot
User-agent: DotBot

# All other bots
User-agent: *
Allow: /storage/sitemaps/
Disallow: /admin/
Disallow: /api/
Allow: /*?page=
Disallow: /*?

# Main Sitemap
Sitemap: https://www.suganta.com/sitemap.xml

# Additional Sitemaps
Sitemap: https://www.suganta.com/sitemap/Cities/cities.xml
Sitemap: https://www.suganta.com/sitemap/jobs/jobs-sitemap-1.xml.gz
Sitemap: https://www.suganta.com/sitemap/subject-city-area/subject-city-area-parent-sitemap-001.xml.gz
Sitemap: https://www.suganta.com/sitemaps/city-edutech/city-edutech-sitemap-index.xml.gz
Sitemap: https://www.suganta.com/sitemap/subject-city-area/subject-city-area-sitemap-index.xml.gz
Sitemap: https://www.suganta.com/sitemaps/kaalo-ai-sitemap.xml
Sitemap: https://www.suganta.com/sitemap/blog/blog-web-stories-sitemap.xml