# Robots.txt for https://umesh-malik.com
# Last updated: 2026-09-14

# Content Signals (https://contentsignals.org/, draft-romm-aipref-contentsignals)
# Public-facing portfolio + technical blog: open to all legitimate uses.
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Sponsorship disclosure for machine-facing surfaces.
# The Markdown/text renderings of this site (/blog/<slug>.md, /llms.txt,
# /llms-full.txt, /mcp results) carry at most one clearly labeled sponsored
# block. The HTML every crawler and human receives carries none, and no
# content on this site varies by User-Agent.
# Policy:   https://umesh-malik.com/ads-for-agents
# Registry: https://umesh-malik.com/.well-known/agent-ads.json
# Unsponsored Markdown: https://umesh-malik.com/clean/blog/<slug>.md

# Allow all crawlers
User-agent: *
Allow: /

# The live API is POST /mcp and POST /api/v1/mcp;
# the spec is /openapi.json; REST v1 is /api/v1; human docs are /developers.

# Google
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: Google-Extended
Allow: /

# Bing
User-agent: Bingbot
Allow: /

# AI Crawlers - Explicitly Allowed
# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic (ClaudeBot is the current crawler; Claude-Web/anthropic-ai are legacy)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google (Gemini / Vertex AI)
User-agent: Google-CloudVertexBot
Allow: /

# Apple
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Amazon / Mistral / DuckDuckGo / ByteDance / Cohere / Common Crawl
User-agent: Amazonbot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: CCBot
Allow: /

# Sitemaps
Sitemap: https://umesh-malik.com/sitemap-index.xml
Sitemap: https://umesh-malik.com/sitemap.xml
Sitemap: https://umesh-malik.com/blog-sitemap.xml

# AI/LLM content files
# Human-readable AI summary: https://umesh-malik.com/ai-summary
# Machine-readable brief: https://umesh-malik.com/llms.txt
# Machine-readable full: https://umesh-malik.com/llms-full.txt
# API catalog (RFC 9727): https://umesh-malik.com/.well-known/api-catalog
# OpenAPI: https://umesh-malik.com/openapi.json
# REST API v1: https://umesh-malik.com/api/v1
# Umesh Malik developer resources: https://umesh-malik.com/developers
# Agent skills discovery: https://umesh-malik.com/.well-known/agent-skills/index.json