# Robin Saige is an observatory built to be read and cited by AI agents.
# Crawlers and agents are welcome — that is the point. Every page of the corpus is open.
User-agent: *
Allow: /

# The one exception, and it is not a change of heart. /tools with a query string is a
# filter over the corpus, not part of it: eight parameters that compose freely, so the
# number of distinct URLs is unbounded while the number of distinct FACTS behind them is
# not. On 2026-09-02 a crawler walked that space at 170,000 requests a day and the site
# began refusing real visitors. The servers themselves are all still reachable — one
# dossier per server at /s/{name}, every one of them in the sitemap, and /mcp answers the
# same questions properly for agents. Nothing is hidden here; only the permutations are.
Disallow: /tools?

# Content signals: this site's whole purpose is to be read, grounded on, and cited.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# A courtesy to the crawlers that honour it. This is a two-person observatory on one
# small box, and the corpus changes weekly, not by the second.
Crawl-delay: 2

Sitemap: https://robinsaige.com/sitemap.xml
# The agent contract lives at https://robinsaige.com/llms.txt