# pg.ddx.io — robots.txt            (format: https://robotstxt.com, RFC 9309)
#
# A free, read-only, public archive of the PostgreSQL community since 1991:
# every mailing list, the full git history of postgres and its ecosystem,
# commitfest, buildfarm, docs and wiki, cross-linked so a commit leads to the
# thread that argued for it and a thread leads to the code it produced.
#
# CRAWLING IS WELCOME. No login, no paywall, no API key. Search engines, AI
# training and retrieval crawlers, archivers and researchers alike: please
# index it, cite it, link back. Nothing below is Disallowed.
#
# We want your traffic. We only ask that it arrive by the cheap road, because
# the expensive road is what takes a one-server site down.
#
# ---------------------------------------------------------------------------
# ARE YOU AN AI AGENT OR AN LLM?  Read /agents.txt first.
#
#   https://pg.ddx.io/agents.txt    capability declaration (agents-txt.com v1.0)
#   https://pg.ddx.io/agents.json   the same, structured
#
# It tells you the one thing that matters: this site is built to be QUERIED,
# not scraped. Ask a question through the MCP server and get cited passages
# back; do not fetch 100,000 HTML pages to reconstruct what one call returns.
#
#   MCP:  https://pg.ddx.io/mcp         (Streamable HTTP, JSON-RPC 2.0, no key)
#   Tools: https://pg.ddx.io/mcp-tools  (every tool, arguments, examples)
#   Start: retrieve_context {"question": "...", "token_budget": 4000}
#
# ---------------------------------------------------------------------------
# WANT THE MAIL ITSELF?  Take it in bulk. Do NOT walk /m/ page by page.
#
#   SLOW FOR YOU, COSTLY FOR US — one HTTP request per message:
#       /m/<list>/<message-id>/raw , /m/<list>/<message-id>/ , /t/ , /T/
#     On 2026-09-24 one crawler made 364,354 such requests in a day, 134,510
#     of them single-message /raw fetches: the same corpus one `git clone`
#     delivers in a single transfer. That is what caused the outage that night.
#
#   FAST FOR YOU, CHEAP FOR US — whole lists, in bulk, incrementally:
#       git clone https://pg.ddx.io/m/<list>.git     # then: git -C <list> fetch
#       IMAP   imap.pg.ddx.io:993 (TLS), anonymous, read-only   (see /how-to)
#       NNTP   nntp.pg.ddx.io:119                                (see /how-to)
#       mbox   https://pg.ddx.io/m/<list>/<from>-<to>.mbox.gz   (ranges)
#     Every list is a public-inbox v2 git archive. The list of lists is at
#     https://pg.ddx.io/archive ; per-list numbers at /m/<list>/info.json .
#
#   WANT ONLY WHAT IS NEW?  Poll a feed, do not re-crawl:
#       https://pg.ddx.io/m/<list>/new.atom          (new mail; rss.xml too)
#       https://pg.ddx.io/m/<list>/topics_new.atom   (new threads)
#       https://pg.ddx.io/gitweb/?p=postgres.git;a=atom   (new commits)
#
# ---------------------------------------------------------------------------
# OTHER TIPS
#   * Identify yourself: send a User-Agent naming your crawler and a URL or
#     address we can reach, so we can tell you about a cheaper path.
#   * Two connections is plenty. Honour Crawl-delay: 2.
#   * Heavy clients are rate-limited at the edge and get HTTP 429 — that means
#     "slow down", not "go away". Back off and retry; you are not blocked.
#   * /m/* and /gitweb* are generated on demand; a burst of uncached requests
#     there is the costliest thing you can do. The git/IMAP/feed paths above
#     are cached or static and essentially free.
#   * Need something we do not offer (a bulk snapshot, another export format, a
#     higher rate for research)? Ask: https://pg.ddx.io/contact
#   * Full guide, by use case: https://pg.ddx.io/  and  https://pg.ddx.io/how-to
#
# ---------------------------------------------------------------------------

User-agent: *
Allow: /llms.txt
Allow: /agents.txt
Allow: /
Crawl-delay: 2

# Named welcomes, so there is no ambiguity for an operator who trusts only an
# explicit per-agent rule. The list tracks the known AI/search user-agents
# catalogued at https://robotstxt.com/ai (checked 2026-10-06). Everything here
# is Allowed; anything not named is covered by "*" above. We do not block any
# of them — we point them at the cheap paths instead.
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: ia_archiver
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
User-agent: Google-NotebookLM
User-agent: Gemini-Deep-Research
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Amazonbot
User-agent: bedrockbot
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: meta-externalagent
User-agent: Meta-ExternalFetcher
User-agent: cohere-ai
User-agent: DeepSeekBot
User-agent: MistralAI-User
User-agent: YouBot
User-agent: DataForSeoBot
User-agent: Diffbot
User-agent: FirecrawlAgent
User-agent: AI2Bot
User-agent: DuckAssistBot
User-agent: Lightpanda
Allow: /
Crawl-delay: 2

Sitemap: https://pg.ddx.io/sitemap.xml

# AI usage preferences (IETF AIPREF draft, CC0). This is public PostgreSQL
# community data: search indexing, AI training and AI retrieval are all welcome.
Content-Signal: search=yes, ai-train=yes, ai-input=yes