# robots.txt for Taste of Pakistan Restaurant
# Website: https://tasteofpakistan.online

# Allow all crawlers by default
User-agent: *
Allow: /

# Disallow private/admin directories (if any exist in future)
Disallow: /admin/
Disallow: /private/

# Disallow legal pages from indexing
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# Disallow llms.txt from search engine indexing (noindex directive via robots.txt)
# This file is for AI agents to understand the site, not for search engine indexing
Disallow: /llms.txt

# Disallow any temporary or backup files
Disallow: /*.bak$
Disallow: /*.tmp$
Disallow: /*~$

# Disallow markdown files
Disallow: /*.md$

# Allow specific important pages (redundant but explicit)
Allow: /index.html
Allow: /menu.html
Allow: /about.html
Allow: /our-food.html
Allow: /contact-us.html
Allow: /catering.html
Allow: /edmonton-pakistani-restaurant.html
Allow: /rewards-signup.html
Allow: /blog/
Allow: /tags/

# Sitemap location
Sitemap: https://tasteofpakistan.online/sitemap.xml

# Crawl delay (optional - prevents server overload)
# Crawl-delay: 1

# Specific bot instructions

# Google bot - allow everything except disallowed above
User-agent: Googlebot
Allow: /
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# Bing bot - allow everything except disallowed above
User-agent: Bingbot
Allow: /
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# AI Crawlers - Allow access to website content
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Block bad bots and scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Social media crawlers - allow for previews
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /