User-agent: *
Disallow: /dashboard/
Disallow: /config/
Disallow: /classes/
Disallow: /vendor/
Disallow: /.htaccess
Disallow: /*.php
Disallow: /error_log
Disallow: /*.sql
Disallow: /*.env
Disallow: /*.lock
Disallow: /*.json
Disallow: /maintenance/
Disallow: /sql/
Disallow: /backups/
Disallow: /uploads/temp/
# Prevent crawling of legacy AMP company URLs (redirected to /amp/company/ pattern)
# Old pattern was causing 118,000+ GSC duplicate canonical issues
Disallow: /company/*/amp

# Crawl budget hardening: block utility/auth/account/transaction routes.
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /verify-email-pending
Disallow: /logout
Disallow: /account/
Disallow: /search/analytics
Disallow: /user-settings
Disallow: /subscribe-checkout
Disallow: /subscribe-success
Disallow: /subscribe-cancel
Disallow: /add-business

# Crawl budget hardening: tracking/noise query parameters.
Disallow: /*utm_
Disallow: /*gclid=
Disallow: /*fbclid=
Disallow: /*_ga=
Disallow: /*manual_search=

# Block suspicious user agents and vulnerability scanners
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MagicBot
Disallow: /

User-agent: YandexBot
Disallow: /

# Explicitly keep /dashboard/ blocked for named crawlers, while allowing public pages.
User-agent: GPTBot
Disallow: /dashboard/
Allow: /

User-agent: ChatGPT-User
Disallow: /dashboard/
Allow: /

User-agent: Claudebot
Disallow: /dashboard/
Allow: /

User-agent: anthropic-ai
Disallow: /dashboard/
Allow: /

User-agent: PerplexityBot
Disallow: /dashboard/
Allow: /

User-agent: perplexity-ai
Disallow: /dashboard/
Allow: /

User-agent: CCBot
Disallow: /dashboard/
Allow: /

User-agent: Copilot
Disallow: /dashboard/
Allow: /

User-agent: copilotbot
Disallow: /dashboard/
Allow: /

User-agent: Googlebot
Disallow: /dashboard/
Disallow: /account/
Disallow: /search/analytics
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /verify-email-pending
Disallow: /logout
Disallow: /user-settings
Disallow: /subscribe-checkout
Disallow: /subscribe-success
Disallow: /subscribe-cancel
Disallow: /add-business
Disallow: /*utm_
Disallow: /*gclid=
Disallow: /*fbclid=
Disallow: /*_ga=
Disallow: /*manual_search=
Allow: /

User-agent: Bingbot
Disallow: /dashboard/
Disallow: /account/
Disallow: /search/analytics
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /verify-email-pending
Disallow: /logout
Disallow: /user-settings
Disallow: /subscribe-checkout
Disallow: /subscribe-success
Disallow: /subscribe-cancel
Disallow: /add-business
Disallow: /*utm_
Disallow: /*gclid=
Disallow: /*fbclid=
Disallow: /*_ga=
Disallow: /*manual_search=
Allow: /

User-agent: facebookexternalhit
Disallow: /dashboard/
Allow: /

User-agent: ia_archiver
Disallow: /dashboard/
Allow: /

User-agent: AppleBot
Disallow: /dashboard/
Allow: /

User-agent: AppleBot-Extended
Disallow: /dashboard/
Allow: /

User-agent: Bytespider
Disallow: /dashboard/
Allow: /

User-agent: DataForSeoBot
Disallow: /dashboard/
Allow: /

User-agent: Omgilibot
Disallow: /dashboard/
Allow: /

User-agent: Diffbot
Disallow: /dashboard/
Allow: /

Crawl-delay: 0
Request-rate: 100/1s

# Sitemap index is the single authoritative entry; it lists every sub-sitemap
# (static, companies, blog, categories, hs-codes, amp when enabled, ai-sitemap).
Sitemap: https://www.haiuae.com/sitemap_index.xml