User-agent: *
Allow: /
Disallow: /admin/
Disallow: /dashboard/
Disallow: /login/
Disallow: /register/
Disallow: /storage/
Disallow: /private/
Disallow: /vendor/
Disallow: /.env
Disallow: /config/
Disallow: /bootstrap/
Disallow: /tests/

# Enhanced SEO directives
Allow: /resources/
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /images/
Allow: /css/
Allow: /js/
Allow: /fonts/

# Search engine specific optimizations
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

# Block resource-intensive crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Sitemap location
Sitemap: https://lorem.bunthu.com/sitemap.xml