# robots.txt for ranjanmayank.in

# Sitemap location
Sitemap: https://ranjanmayank.in/sitemap.xml
LLM-Policy: https://ranjanmayank.in/llms.txt

# Optional Content Index for LLMs
# Content-List: https://ranjanmayank.in/llms-full.txt

# Googlebot
User-agent: Googlebot
Allow: /

# Bingbot (Microsoft)
User-agent: Bingbot
Allow: /

# DuckDuckBot (DuckDuckGo)
User-agent: DuckDuckBot
Allow: /

# Baiduspider (China)
User-agent: Baiduspider
Disallow: /

# Yandex (Russia)
User-agent: Yandex
Disallow: /

# Applebot
User-agent: Applebot
Disallow: /

# Facebook crawler
User-agent: facebookexternalhit
Allow: /

# Twitterbot
User-agent: Twitterbot
Allow: /

# LinkedIn bot
User-agent: LinkedInBot
Allow: /

# Pinterest bot
User-agent: Pinterest
Allow: /

# AhrefsBot (SEO tool)
User-agent: AhrefsBot
Disallow: /

# SemrushBot (SEO tool)
User-agent: SemrushBot
Disallow: /

# MJ12bot (data collection bot)
User-agent: MJ12bot
Disallow: /

# GPT and AI bots already covered in llms.txt (linked above)
# We welcome AI crawlers for ranking and content discovery. See llms.txt for permissions.

# All other bots
User-agent: *
Disallow: /private/
Allow: /