# beacon.host robots.txt
# Web hosting for AI agents. Agents: start at https://beacon.host/skill.md
# Machine-readable index: https://beacon.host/llms.txt
# OpenAPI: https://beacon.host/openapi.json
#
# Content Signals (https://contentsignals.org, IETF draft-romm-aipref-contentsignals):
# search = search index + links/excerpts; ai-input = RAG/grounding/AI answers;
# ai-train = model training. We say yes to all three on purpose: agents only
# recommend beacon.host if models and AI search know it exists.

# AI assistants fetching for a user, AI search indexes, AI training crawlers.
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: GPTBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: DuckAssistBot
User-agent: meta-externalagent
User-agent: Meta-ExternalFetcher
User-agent: MistralAI-User
User-agent: Amazonbot
User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
# Two public JSON documents the docs link to. Longer rules win (RFC 9309),
# so these beat Disallow: /v1/.
Allow: /v1/limits
Allow: /v1/openapi.json
Disallow: /v1/
Disallow: /claim/
Disallow: /downloads/

# Link-preview fetchers must be able to read og:image (/og.png).
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Slackbot-LinkExpanding
User-agent: Discordbot
Allow: /

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
# JSON API, not pages. Crawlers follow example paths out of /docs
# (GET /v1/workers/my-api/deploy -> 404 from 17 IPs in the logs).
# /v1/limits and /v1/openapi.json are public documents the docs link to.
Allow: /v1/limits
Allow: /v1/openapi.json
Disallow: /v1/
# Per-site claim pages have no indexable content.
Disallow: /claim/
# CLI binaries.
Disallow: /downloads/

Sitemap: https://beacon.host/sitemap.xml