# https://www.rfc-editor.org/rfc/rfc9309
# https://contentsignals.org/
# Besu documentation — allow indexing and AI crawlers.
# Content signals declare how automated systems may use the public docs:
# - search=yes allows search indexing and search results.
# - ai-input=yes allows use as model input for retrieval and grounding.
# - ai-train=yes allows use for training or fine-tuning AI models.

User-agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

Sitemap: https://docs.besu-eth.org/sitemap.xml