# joinbuffet.com
#
# Crawling is allowed everywhere. Preview deployments are kept out of the index
# by an X-Robots-Tag: noindex header scoped to *.vercel.app hosts in vercel.json
# -- deliberately a header rather than a Disallow here, because Disallow stops
# crawlers reading the page at all, which would stop them seeing the noindex.

User-agent: *
Allow: /

# Answer engines are welcome, by name. Each of these is already covered by the
# wildcard above; naming them is so that a future rule for one of them can
# never be mistaken for a rule about all of them, and so anyone reading this
# file can see the decision was made rather than defaulted. Being quotable in
# an AI answer is the same distribution problem as ranking in search.
#
# /llms.txt is a plain summary of the site written for them, and
# /llms-full.txt is the full text of every page and article.

# OpenAI: training, search index, and the live browsing agent.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic: training, search index, and the live fetcher.
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Perplexity: index and the live fetcher.
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google's AI products beyond web search (Gemini, AI Overviews grounding).
User-agent: Google-Extended
Allow: /

# Apple Intelligence and Siri.
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Microsoft Bing, which also feeds Copilot and ChatGPT search.
User-agent: Bingbot
Allow: /

# Meta AI.
User-agent: Meta-ExternalAgent
Allow: /

# DuckDuckGo's answer features.
User-agent: DuckAssistBot
Allow: /

# Common Crawl, which most open models train on.
User-agent: CCBot
Allow: /

Sitemap: https://joinbuffet.com/sitemap-index.xml