Sitemap: https://www.gccdomestic.com/sitemap.xml
Sitemap: https://www.gccdomestic.com/sitemap-index.xml
Sitemap: https://www.gccdomestic.com/video-sitemap.xml


# GCC Domestic - robots.txt
# Optimized for search engine crawling

User-agent: *
Allow: /

# Allow Next.js assets so Google can render pages properly
# Static assets have X-Robots-Tag: noindex via nginx (won't be indexed)
Allow: /_next/static/
Allow: /_next/image


# Block OG image routes from crawling (crawl budget waste — 3,000+ URLs)
# OG/Twitter card images must be fetchable by social crawlers (X/Twitter obeys robots.txt)
Allow: /api/og/
Allow: /*/opengraph-image
Allow: /*/twitter-image

# Block admin and private areas
# /en/agency/ 301-redirects to /agency/ — block the redirect source
Disallow: /en/agency/
# /ar/agency/ is the real Arabic version (different content) — do NOT block

# Block double-prefix paths (bug)
Disallow: /en/en/
Disallow: /ar/ar/
Disallow: /en/ar/
Disallow: /ar/en/

# Block capitalized country paths (should be lowercase: /ksa/, /uae/, etc.)
Disallow: /KSA/
Disallow: /Bahrain/
Disallow: /Qatar/
Disallow: /UAE/
Disallow: /Kuwait/
Disallow: /Oman/

# Block double agency segment (bug)
Disallow: /agency/agency/

# ?lang= URLs redirect to clean canonical URLs (no duplicate content).

# Block ?agency= query params on request-worker
Disallow: /*?agency=*
Disallow: /*&agency=*

# Block zero-result filter/param URLs (GSC 2026-06-11: 2,850 soft-404s, all ?city=/&q=/&type= combos
# with no results; also burns crawl budget). Clean directory pages stay fully crawlable.
Disallow: /*?city=
Disallow: /*&city=
Disallow: /*?q=
Disallow: /*&q=
Disallow: /*?type=
Disallow: /*&type=
Disallow: /*?nationality=
Disallow: /*&nationality=
Disallow: /*?source=
Disallow: /*&source=

# Block agency private/dashboard sub-pages (not for public indexing)
Disallow: /agency/*/accounting
Disallow: /agency/*/ai
Disallow: /agency/*/analytics
Disallow: /agency/*/applications
Disallow: /agency/*/availability
Disallow: /agency/*/bookings
Disallow: /agency/*/calendar
Disallow: /agency/*/cameras
Disallow: /agency/*/card-style
Disallow: /agency/*/client-requests
Disallow: /agency/*/communications
Disallow: /agency/*/compliance
Disallow: /agency/*/contracts
Disallow: /agency/*/dashboard
Disallow: /agency/*/deliveries
Disallow: /agency/*/document-generator
Disallow: /agency/*/documents
Disallow: /agency/*/export
Disallow: /agency/*/insights
Disallow: /agency/*/integrations
Disallow: /agency/*/interviews
Disallow: /agency/*/jobs
Disallow: /agency/*/labour-status
Disallow: /agency/*/marketing
Disallow: /agency/*/messages
Disallow: /agency/*/mobile-dashboard
Disallow: /agency/*/notifications
Disallow: /agency/*/office
Disallow: /agency/*/offers
Disallow: /agency/*/partner-agents
Disallow: /agency/*/partners
Disallow: /agency/*/pipeline
Disallow: /agency/*/profile
Disallow: /agency/*/queue
Disallow: /agency/*/salary-calendar
Disallow: /agency/*/settings
Disallow: /agency/*/staff
Disallow: /agency/*/tasks
Disallow: /agency/*/whatsapp-bot
Disallow: /agency/*/worker-allocations
Disallow: /agency/*/workers

# Block /posts/ route (internal, not public)
Disallow: /posts/
Disallow: /posts?

# Block app-specific routes
Disallow: /app-home
Disallow: /app-login
Disallow: /app-register
Disallow: /app-dashboard
Disallow: /app-workers
Disallow: /app-notifications
Disallow: /app-settings
Disallow: /app-download
Disallow: /app-mobile
Disallow: /agency-mobile
Disallow: /family-mobile
Disallow: /staff-mobile
Disallow: /booking
Disallow: /apply
Disallow: /add-contact
Disallow: /client
Disallow: /comparison
Disallow: /compare

Disallow: /admin/
Disallow: /api/admin/
Disallow: /dashboard
Disallow: /*/dashboard
Disallow: /owner/
Disallow: /staff/
Disallow: /family/
Disallow: /verification/
Disallow: /queue/

# Block API endpoints
Disallow: /api/backend/
Disallow: /api/auth/
Disallow: /api/admin/

# Block filter URLs with query parameters (prevent duplicate indexing)
Disallow: /*?city=
Disallow: /*&city=
Disallow: /*?country=
Disallow: /*&country=
Disallow: /*?q=
Disallow: /*&q=
Disallow: /*?type=
Disallow: /*&type=
Disallow: /*?nationality=
Disallow: /*&nationality=
Disallow: /*?status=
Disallow: /*&status=
# ?lang= URLs redirect to clean canonical URLs via nginx (301).
Disallow: /*?page=
Disallow: /*&page=
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?rate=
Disallow: /*&rate=
Disallow: /*?new=
Disallow: /*&new=
Disallow: /request-worker
Disallow: /en/request-worker
Disallow: /ar/request-worker
Disallow: /forgot-password
Disallow: /en/forgot-password
Disallow: /ar/forgot-password

# Allow public pages
Allow: /agencies/
Allow: /workers/
Allow: /agency/
Allow: /verification
Allow: /login/
Allow: /register/
Allow: /partner/

# Crawl-delay for optimal server performance
Crawl-delay: 1

# Popular search engines - Allow all
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# AI crawlers — ALLOW all (appear in AI search results: ChatGPT, Claude, Perplexity, Gemini, etc.)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: cohere-ai
Allow: /

# Block only spam/scraping bots
User-agent: Amazonbot
Disallow: /

User-agent: MJ12bot
Disallow: /