# Robots.txt for Himanshu Ahuja Portfolio
# Website: https://himanshuahuja.in
# Last Updated: 2025-10-24

# Global directives for all bots
User-agent: *
Allow: /
Crawl-delay: 1

# Disallow certain files and directories
Disallow: /backup/css/
Disallow: /backup/scripts/
Disallow: /backup/images/testimonials/
Disallow: /backup/pd/
Disallow: /backup/rd/
Disallow: /qr/style.css
Disallow: /ttt/style.css
Disallow: /*.txt$

# Allow important pages and directories
Allow: /
Allow: /qr/
Allow: /ttt/
Allow: /upi.html
Allow: /social-preview.html
Allow: /sitemap.html
Allow: /404.html
Allow: /manifest.json
Allow: /HimanshuAhujaResume.pdf
Allow: /images/
Allow: /styles.css

# Google-specific directives
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Bing-specific directives
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# AI and Generative Engine Bots
User-agent: GPTBot
Allow: /
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2

User-agent: CCBot
Allow: /
Crawl-delay: 2

User-agent: anthropic-ai
Allow: /
Crawl-delay: 2

User-agent: Claude-Web
Allow: /
Crawl-delay: 2

# Social Media Crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# Sitemaps
Sitemap: https://himanshuahuja.in/sitemap.xml

# Host directive
Host: himanshuahuja.in

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# Disallow specific paths if needed (currently allowing all)
# Disallow: /backup/
# Disallow: /temp/

# Crawl-delay (optional, in seconds)
Crawl-delay: 1