# Hotel Golden Lagoon - Robots.txt
# Optimized for Search Engines and AI Bots

# Allow all bots to crawl everything
User-agent: *
Allow: /

# Specific permissions for major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# AI Bots and Social Media Crawlers
User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Gemini
Allow: /

User-agent: Bard
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity
Allow: /

User-agent: Grok
Allow: /

User-agent: GrokBot
Allow: /

User-agent: Anthropic
Allow: /

User-agent: FacebookExternalHit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# Disallow sensitive directories for security
Disallow: /backend/config.php
Disallow: /backend/security.log
Disallow: /backend/rate_limits.json
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.bak$

# Allow important SEO files
Allow: /sitemap.xml
Allow: /robots.txt

# Crawl delay (optional - can be removed if not needed)
Crawl-delay: 1

# Sitemap location
Sitemap: https://hotelgoldenlagoon.com/sitemap.xml
# Local development sitemap
Sitemap: /sitemap.xml