# RxMargin crawl policy (rxmargin.co.uk). Published 2026-08-04.
#
# Crawl the product. The dashboard, the drug pages and every static section are meant to be
# found, and nothing below closes any of them off.
#
# What is closed off is the machine data our own client fetches: the price-history shards, the
# code-axis shards, the per-practice volume files and the three big index artefacts. They carry
# no text a search engine can use, they are only worth anything as a complete set, and serving
# them to a crawler costs us bandwidth for nothing.
#
# This file states a boundary. It is not a control - any client can ignore every line of it.
# Automated bulk collection of the dataset as a whole is not permitted without our written
# permission; the terms are at https://rxmargin.co.uk/methodology/#terms. If you want the whole
# dataset, ask rather than crawl for it: a daily snapshot is available on request.
# Contact: edward@rxmargin.co.uk

User-agent: *
Allow: /

# Machine shards and bulk artefacts. Fetched by our own client, zero SEO value, no business
# being crawled by anyone.
Disallow: /live/history/
Disallow: /live/code-axis/
Disallow: /practice-data/
Disallow: /live/order-codes.json
Disallow: /live/supplier-name-index.json
Disallow: /live/catalogue-index.json

# Honoured by some crawlers and ignored by others - Googlebot ignores Crawl-delay entirely. It
# is stated here as a request for politeness, and is not relied on as a limit.
Crawl-delay: 10

# No Sitemap line: there is no sitemap.xml on this site today, and pointing at one that does not
# exist would be worse than pointing at none.