# Standard robots.txt for BuffetDisplay - Multilingual Hospitality Tech
# Optimized for search engines while blocking low-authority crawlers

# Global rules - allow most content
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /.well-known/
Disallow: /tmp/
Crawl-delay: 1

# Block low-authority and aggressive bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: YandexBot
Crawl-delay: 3

# Allow major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 2

# Allow AI crawlers
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Allow social media crawlers
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Multilingual content - explicitly allow language directories
Allow: /fr/
Allow: /es/
Allow: /de/
Allow: /it/
Allow: /pt/
Allow: /zh/
Allow: /ja/
Allow: /ko/
Allow: /ar/
Allow: /id/

# Important business pages
Allow: /features
Allow: /integrations
Allow: /use-cases
Allow: /compliance
Allow: /demo
Allow: /contact

# Host declaration
Host: buffetdisplay.com

# Sitemap location
Sitemap: https://buffetdisplay.com/sitemap.xml