feat: robots.txt served file, AI bots and the non-production lock

- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the framework repo) - golem15_seo_robots singleton: rules and AI bot switches; eleven bots incl. Claude-User, all allowed by default - GET /robots.txt: rules, a Disallow group per switched-off bot and the golem15.seo.robots_sitemaps lines; outside production the lock with X-Robots-Tag; never a 5xx (last good body, else allow-all) - TestRobots on Google's matcher with the deciding line, and the noindex-blocked check (D-24)
2026-10-27 11:45:44 +01:00
parent 4cb0fbfebb
commit 3dd936202f
11 changed files with 643 additions and 5 deletions
--- a/classes/bots.go
+++ b/classes/bots.go
@@ -0,0 +1,55 @@
+package classes
+
+import "strings"
+
+// The kinds of AI bots (D-25): search and answer crawlers, fetchers that
+// load a page when a user asks for it, and crawlers that collect content
+// for training.
+const (
+	BotSearch   = "search"
+	BotUser     = "user"
+	BotTraining = "training"
+)
+
+// Bot is one AI crawler the robots.txt screen can switch off: its
+// user-agent token, its operator and its kind.
+type Bot struct {
+	Token    string
+	Operator string
+	Kind     string
+}
+
+// Bots is the AI bot list in the screen's order (UI-SPEC C10, D-25,
+// D-105): search bots, user-fetch bots, then training bots. Every bot is
+// allowed until an editor switches it off. The list is plugin data and
+// can grow without a layout change (Q-6).
+var Bots = []Bot{
+	{Token: "OAI-SearchBot", Operator: "OpenAI", Kind: BotSearch},
+	{Token: "Claude-SearchBot", Operator: "Anthropic", Kind: BotSearch},
+	{Token: "PerplexityBot", Operator: "Perplexity", Kind: BotSearch},
+	{Token: "ChatGPT-User", Operator: "OpenAI", Kind: BotUser},
+	{Token: "Claude-User", Operator: "Anthropic", Kind: BotUser},
+	{Token: "GPTBot", Operator: "OpenAI", Kind: BotTraining},
+	{Token: "ClaudeBot", Operator: "Anthropic", Kind: BotTraining},
+	{Token: "Google-Extended", Operator: "Google", Kind: BotTraining},
+	{Token: "Applebot-Extended", Operator: "Apple", Kind: BotTraining},
+	{Token: "CCBot", Operator: "Common Crawl", Kind: BotTraining},
+	{Token: "Meta-ExternalAgent", Operator: "Meta", Kind: BotTraining},
+}
+
+// BotField is the robots form's switch field of a bot: bot_ and its token
+// lowercased with - as _ (bot_oai_searchbot).
+func BotField(token string) string {
+	return "bot_" + strings.ReplaceAll(strings.ToLower(token), "-", "_")
+}
+
+// BotByToken returns the listed bot whose token equals token (letter case
+// ignored).
+func BotByToken(token string) (Bot, bool) {
+	for _, b := range Bots {
+		if strings.EqualFold(b.Token, token) {
+			return b, true
+		}
+	}
+	return Bot{}, false
+}
--- a/classes/robots.go
+++ b/classes/robots.go
@@ -0,0 +1,231 @@
+package classes
+
+import (
+	"context"
+	"sort"
+	"strings"
+
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"github.com/jimsmart/grobotstxt"
+	"gorm.io/gorm"
+)
+
+// RobotsSitemapsConfigKey lists the sitemap addresses robots.txt names,
+// one Sitemap line each, absolute on the site URL (P-23); default
+// ["/sitemap.xml"].
+const RobotsSitemapsConfigKey = "golem15.seo.robots_sitemaps"
+
+// MaxRobotsBytes is the largest robots.txt the editor stores: Google
+// ignores everything after 500 KiB.
+const MaxRobotsBytes = 500 << 10
+
+// The two fixed robots.txt bodies.
+const (
+	// NonProductionRobots is served outside production whatever is stored
+	// (D-22).
+	NonProductionRobots = "User-agent: *\nDisallow: /\n"
+	// FallbackRobots is served when the stored file cannot be read and no
+	// earlier body is cached: allow everything, never a 5xx.
+	FallbackRobots = "User-agent: *\nDisallow:\n"
+)
+
+// RobotsSitemaps returns the configured sitemap addresses, absolute on
+// the site URL.
+func (s *Service) RobotsSitemaps() []string {
+	var list []string
+	if cfg := s.Config(); cfg != nil {
+		if raw, ok := cfg.Lookup(RobotsSitemapsConfigKey); ok {
+			switch v := raw.(type) {
+			case []any:
+				for _, item := range v {
+					if str, ok := item.(string); ok {
+						list = append(list, str)
+					}
+				}
+			case []string:
+				list = append(list, v...)
+			case string:
+				list = strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' })
+			}
+		}
+	}
+	if list == nil {
+		list = []string{"/sitemap.xml"}
+	}
+	out := make([]string, 0, len(list))
+	for _, item := range list {
+		if item = strings.TrimSpace(item); item != "" {
+			out = append(out, AbsoluteURL(s.SiteURL(), item))
+		}
+	}
+	return out
+}
+
+// LoadRobots reads robots row 1; without one the defaults apply (no
+// rules, every bot allowed).
+func LoadRobots(db *gorm.DB) (models.Robots, error) {
+	var rows []models.Robots
+	if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil {
+		return models.Robots{}, err
+	}
+	if len(rows) == 0 {
+		return models.Robots{ID: 1, Bots: models.BotSwitches{}}, nil
+	}
+	return rows[0], nil
+}
+
+// ComposeRobots builds the production robots.txt (D-22, D-25, P-23): the
+// stored rules, one "User-agent: <token>" / "Disallow: /" group per
+// switched-off bot, then one Sitemap line per sitemap address.
+func ComposeRobots(content string, bots models.BotSwitches, sitemaps []string) string {
+	var b strings.Builder
+	content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t")
+	if content != "" {
+		b.WriteString(content)
+		b.WriteString("\n")
+	}
+	for _, bot := range Bots {
+		if bots.Allowed(bot.Token) {
+			continue
+		}
+		if b.Len() > 0 {
+			b.WriteString("\n")
+		}
+		b.WriteString("User-agent: " + bot.Token + "\nDisallow: /\n")
+	}
+	if len(sitemaps) > 0 && b.Len() > 0 {
+		b.WriteString("\n")
+	}
+	for _, sm := range sitemaps {
+		b.WriteString("Sitemap: " + sm + "\n")
+	}
+	return b.String()
+}
+
+// generatedStart is the 1-based line where the generated part of a
+// composed file starts: the lines after the stored rules.
+func generatedStart(content string) int {
+	content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t")
+	if content == "" {
+		return 1
+	}
+	return strings.Count(content, "\n") + 2
+}
+
+// ServedFile is the robots.txt crawlers get (D-22): outside production
+// always NonProductionRobots, else the composed file from the stored rules
+// and switches. It caches the last body it served, which LastServedRobots
+// answers when the store fails.
+func (s *Service) ServedFile(ctx context.Context, production bool) (string, error) {
+	if !production {
+		return NonProductionRobots, nil
+	}
+	db, err := s.DB(ctx)
+	if err != nil {
+		return "", err
+	}
+	row, err := LoadRobots(db)
+	if err != nil {
+		return "", err
+	}
+	body := ComposeRobots(row.Content, row.Bots, s.RobotsSitemaps())
+	if s != nil {
+		s.robots.Store(&body)
+	}
+	return body, nil
+}
+
+// LastServedRobots returns the last production body ServedFile answered,
+// "" when there is none.
+func (s *Service) LastServedRobots() string {
+	if s == nil {
+		return ""
+	}
+	if p := s.robots.Load(); p != nil {
+		return *p
+	}
+	return ""
+}
+
+// RobotsTest is the tester's answer (UI-SPEC E5): whether agent may fetch
+// the address, the deciding line (0 and "" when no rule matched) and
+// whether a bot switch decided it.
+type RobotsTest struct {
+	Allowed bool   `json:"allowed"`
+	Line    int    `json:"line"`
+	Rule    string `json:"rule"`
+	Bot     bool   `json:"bot"`
+	Agent   string `json:"agent"`
+	Staging bool   `json:"staging"`
+}
+
+// TestRobots answers whether agent may fetch path under the production
+// file composed from content and bots (unsaved values included), through
+// grobotstxt, Google's matcher (D-22, D-99): longest match, Allow wins a
+// tie, * and $.
+func (s *Service) TestRobots(content string, bots models.BotSwitches, agent, path string) RobotsTest {
+	file := ComposeRobots(content, bots, s.RobotsSitemaps())
+	uri := AbsoluteURL(s.SiteURL(), path)
+	if s.SiteURL() == "" {
+		uri = "http://localhost" + path
+	}
+	m := grobotstxt.NewRobotsMatcher()
+	out := RobotsTest{Agent: agent, Staging: s.Staging()}
+	out.Allowed = m.AgentAllowed(file, agent, uri)
+	if line := m.MatchingLine(); line > 0 {
+		out.Line = line
+		lines := strings.Split(file, "\n")
+		if line <= len(lines) {
+			out.Rule = strings.TrimSpace(lines[line-1])
+		}
+		out.Bot = !out.Allowed && line >= generatedStart(content)
+	}
+	return out
+}
+
+// noindexAddresses lists the addresses the site serves noindex that a
+// robots.txt could block (D-24): exact URL rules and records whose own
+// robots value says noindex.
+func noindexAddresses(db *gorm.DB) ([]string, error) {
+	seen := map[string]bool{}
+	var out []string
+	var rules []models.Rule
+	if err := db.Where("match = ? AND robots LIKE ?", models.MatchExact, "%noindex%").Find(&rules).Error; err != nil {
+		return nil, err
+	}
+	for _, r := range rules {
+		if !seen[r.URL] {
+			seen[r.URL] = true
+			out = append(out, r.URL)
+		}
+	}
+	var entries []models.Entry
+	if err := db.Where("robots LIKE ? AND url <> ''", "%noindex%").Find(&entries).Error; err != nil {
+		return nil, err
+	}
+	for _, e := range entries {
+		if !seen[e.URL] {
+			seen[e.URL] = true
+			out = append(out, e.URL)
+		}
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+// BlockedNoindex lists the noindex addresses Googlebot may not fetch under
+// the production file composed from content and bots (D-24): a crawler
+// that cannot fetch a page never sees its noindex.
+func (s *Service) BlockedNoindex(db *gorm.DB, content string, bots models.BotSwitches) ([]string, error) {
+	urls, err := noindexAddresses(db)
+	if err != nil {
+		return nil, err
+	}
+	var out []string
+	for _, u := range urls {
+		if !s.TestRobots(content, bots, "Googlebot", u).Allowed {
+			out = append(out, u)
+		}
+	}
+	return out, nil
+}
--- a/classes/service.go
+++ b/classes/service.go
@@ -5,6 +5,7 @@ package classes
 import (
 	"context"
 	"errors"
+	"sync/atomic"

 	"git.golem15.com/golem15/summercms/modules/backpack"
 	"git.golem15.com/golem15/summercms/modules/cabana"
@@ -22,6 +23,9 @@ var ErrNotBooted = errors.New("golem15.seo: app is not booted")
 // history.
 type Service struct {
 	app *backpack.App
+	// robots is the last production robots.txt served, answered when the
+	// store fails (D-22: never a 5xx).
+	robots atomic.Pointer[string]
 }

 // NewService returns the service of a booted application. The database
--- a/config/config.yaml
+++ b/config/config.yaml
@@ -19,3 +19,7 @@ notfound_token: ""
 # notfound_max_rows caps the 404 log; new addresses beyond it are dropped,
 # with one warning an hour.
 notfound_max_rows: 10000
+#
+# robots_sitemaps lists the sitemap addresses robots.txt names, one
+# "Sitemap:" line each, made absolute on the site URL.
+robots_sitemaps: ["/sitemap.xml"]
--- a/controllers/api/robots.go
+++ b/controllers/api/robots.go
@@ -0,0 +1,43 @@
+package api
+
+import (
+	"net/http"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+)
+
+// Robots serves GET /robots.txt (D-22, D-25, P-23) as text/plain. In
+// production it is the stored rules, a Disallow group for every
+// switched-off AI bot and the Sitemap lines; outside production it is
+// "User-agent: *" / "Disallow: /" with X-Robots-Tag: noindex, nofollow,
+// whatever is stored, and no admin setting can change that. It never
+// answers a 5xx: when the stored file cannot be read it serves the last
+// body it served, else an allow-all file, and logs the error.
+func Robots(service func() *classes.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		svc := service()
+		body := classes.FallbackRobots
+		staging := false
+		if svc == nil {
+			log.Error("golem15.seo: robots.txt served before boot")
+		} else {
+			staging = svc.Staging()
+			served, err := svc.ServedFile(r.Context(), !staging)
+			switch {
+			case err == nil:
+				body = served
+			case svc.LastServedRobots() != "":
+				log.Error("golem15.seo: robots.txt, serving the last good file", "error", err)
+				body = svc.LastServedRobots()
+			default:
+				log.Error("golem15.seo: robots.txt, serving the allow-all fallback", "error", err)
+			}
+		}
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		if staging {
+			w.Header().Set("X-Robots-Tag", "noindex, nofollow")
+		}
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(body))
+	}
+}
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/go-gormigrate/gormigrate/v2 v2.1.7
 	github.com/goccy/go-yaml v1.19.2
 	github.com/jackc/pgx/v5 v5.10.0
+	github.com/jimsmart/grobotstxt v1.0.3
 	github.com/testcontainers/testcontainers-go v0.44.0
 	github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0
 	gocloud.dev v0.46.0
--- a/go.sum
+++ b/go.sum
@@ -165,6 +165,8 @@ github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMD
 github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0=
 github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
 github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
 github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
 github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
@@ -195,8 +197,18 @@ github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
 github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
 github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
 github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo=
@@ -219,6 +231,7 @@ github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTs
 github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
 github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
 github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0=
@@ -231,6 +244,8 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
 github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jimsmart/grobotstxt v1.0.3 h1:DUP8ERo4MqVe0MZsudcz/ROKJcMhguGQLrVmA+8rDlM=
+github.com/jimsmart/grobotstxt v1.0.3/go.mod h1:WImegD7gBR7B9I1UOrcuoQHmeflNp267CIHkQmZOiYU=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -309,6 +324,17 @@ github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8=
 github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig=
 github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd17NIEQ=
 github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.6 h1:11TGpSHY7Esh/i/qnq02Jo5oVrI1Gue8Slbq0ujPZFQ=
+github.com/nxadm/tail v1.4.6/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M=
+github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.4 h1:NiTx7EEvBzu9sFOD1zORteLSt3o8gnlvZZwSE9TnY9U=
+github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
@@ -420,6 +446,8 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
 go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
 gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q=
 gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
 golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
@@ -427,27 +455,51 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQz
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
 golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
 golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
 golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
 golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
 golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
 golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
 golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
@@ -462,9 +514,22 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 h1:
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
 google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/postgres v1.6.3 h1:bAn6O2pUa8LtpWEvL5NFU4+52Tfx8Ut7IVaIacCLcI0=
 gorm.io/driver/postgres v1.6.3/go.mod h1:0c4fQA44XhOklXDkgtuKqysHCycTa5i9e3EIpDGCwXk=
--- a/models/robots.go
+++ b/models/robots.go
@@ -0,0 +1,82 @@
+package models
+
+import (
+	"database/sql/driver"
+	"encoding/json"
+	"errors"
+	"time"
+)
+
+// Robots is the robots.txt singleton (row id 1) on golem15_seo_robots
+// (D-22, D-25): the editor's rules and the AI bot switches. The served
+// file adds a Disallow group for every switched-off bot and the Sitemap
+// lines; outside production it is replaced by the non-production lock.
+type Robots struct {
+	ID        uint        `gorm:"column:id;primaryKey"`
+	Content   string      `gorm:"column:content"`
+	Bots      BotSwitches `gorm:"column:bots"`
+	CreatedAt *time.Time  `gorm:"column:created_at"`
+	UpdatedAt *time.Time  `gorm:"column:updated_at"`
+}
+
+func (Robots) TableName() string { return "golem15_seo_robots" }
+
+// MorphName names the robots row in history.
+func (Robots) MorphName() string { return `Golem15\SEO\Models\Robots` }
+
+// Fillable lists the form's columns: the rules. The bot switches are
+// written by the robots form's extension.
+func (Robots) Fillable() []string { return []string{"content"} }
+
+// Rules is empty: the robotstxt field type refuses an oversized file.
+func (Robots) Rules() map[string]string { return map[string]string{} }
+
+// BotSwitches maps a bot's user-agent token to whether it is allowed. A
+// bot that is not listed is allowed (D-25: every bot on by default).
+type BotSwitches map[string]bool
+
+// Allowed reports whether token is switched on.
+func (b BotSwitches) Allowed(token string) bool {
+	on, ok := b[token]
+	return !ok || on
+}
+
+// Value encodes the switches as a JSON object; nil is {}.
+func (b BotSwitches) Value() (driver.Value, error) {
+	if b == nil {
+		return "{}", nil
+	}
+	raw, err := json.Marshal(map[string]bool(b))
+	if err != nil {
+		return nil, err
+	}
+	return string(raw), nil
+}
+
+// Scan decodes a jsonb object; NULL reads as none.
+func (b *BotSwitches) Scan(src any) error {
+	var raw []byte
+	switch v := src.(type) {
+	case nil:
+		*b = BotSwitches{}
+		return nil
+	case []byte:
+		raw = v
+	case string:
+		raw = []byte(v)
+	default:
+		return errors.New("models: unsupported bots value")
+	}
+	out := map[string]bool{}
+	if len(raw) > 0 {
+		if err := json.Unmarshal(raw, &out); err != nil {
+			return err
+		}
+	}
+	*b = out
+	return nil
+}
+
+func init() {
+	Register(Robots{})
+}
--- a/robots_test.go
+++ b/robots_test.go
@@ -0,0 +1,115 @@
+package seo_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+)
+
+// robotsTxt fetches /robots.txt.
+func (e *testEnv) robotsTxt(t *testing.T) *httptest.ResponseRecorder {
+	t.Helper()
+	req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil)
+	req.RemoteAddr = "198.51.100.9:4000"
+	rec := httptest.NewRecorder()
+	e.h.ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("/robots.txt = %d %s, want 200", rec.Code, rec.Body.String())
+	}
+	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
+		t.Fatalf("/robots.txt content type = %q, want text/plain", ct)
+	}
+	return rec
+}
+
+// TestRobotsServed checks GET /robots.txt (D-22, D-25, P-23, T-32-19,
+// T-32-20): the stored rules, a Disallow group per switched-off AI bot and
+// the Sitemap line in production; the non-production lock whatever is
+// stored; and a 200 with the last good or the allow-all file when the
+// store cannot be read.
+func TestRobotsServed(t *testing.T) {
+	env := bootEnv(t)
+	if body := env.robotsTxt(t).Body.String(); body != "Sitemap: "+siteURL+"/sitemap.xml\n" {
+		t.Fatalf("fresh robots.txt = %q, want only the Sitemap line", body)
+	}
+	if err := env.db.Exec(`UPDATE golem15_seo_robots SET content = ?, bots = ? WHERE id = 1`,
+		"User-agent: *\nDisallow: /admin\n", `{"GPTBot": false, "CCBot": false, "ClaudeBot": true}`).Error; err != nil {
+		t.Fatal(err)
+	}
+	rec := env.robotsTxt(t)
+	want := "User-agent: *\nDisallow: /admin\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nSitemap: " + siteURL + "/sitemap.xml\n"
+	if body := rec.Body.String(); body != want {
+		t.Fatalf("robots.txt = %q, want %q", body, want)
+	}
+	if rec.Header().Get("X-Robots-Tag") != "" {
+		t.Fatalf("production robots.txt carries X-Robots-Tag %q", rec.Header().Get("X-Robots-Tag"))
+	}
+
+	// The store fails: the last good body, still 200.
+	if err := env.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil {
+		t.Fatal(err)
+	}
+	if body := env.robotsTxt(t).Body.String(); body != want {
+		t.Fatalf("robots.txt with the store gone = %q, want the last good body", body)
+	}
+	fresh := bootEnv(t)
+	if err := fresh.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil {
+		t.Fatal(err)
+	}
+	if body := fresh.robotsTxt(t).Body.String(); body != "User-agent: *\nDisallow:\n" {
+		t.Fatalf("robots.txt with the store gone and nothing cached = %q, want allow-all", body)
+	}
+
+	// Outside production: the lock, whatever is stored.
+	staging := bootEnv(t, "SUMMER_ENV=staging")
+	if err := staging.db.Exec(`UPDATE golem15_seo_robots SET content = ? WHERE id = 1`, "User-agent: *\nAllow: /\n").Error; err != nil {
+		t.Fatal(err)
+	}
+	rec = staging.robotsTxt(t)
+	if rec.Body.String() != "User-agent: *\nDisallow: /\n" || rec.Header().Get("X-Robots-Tag") != "noindex, nofollow" {
+		t.Fatalf("staging robots.txt = %q %v, want the lock with X-Robots-Tag", rec.Body.String(), rec.Header())
+	}
+}
+
+// TestRobotsTester checks the tester against Google's documented examples
+// (D-22, D-99): the longest match wins, Allow wins a tie, $ anchors the
+// end, the matching line is reported, and a switched-off bot is blocked by
+// its switch.
+func TestRobotsTester(t *testing.T) {
+	env := bootEnv(t)
+	svc := env.seoPlugin(t).Service()
+	none := models.BotSwitches{}
+	cases := []struct {
+		content, agent, path string
+		allowed              bool
+		line                 int
+		rule                 string
+	}{
+		{"User-agent: *\nAllow: /p\nDisallow: /\n", "Googlebot", "/page", true, 2, "Allow: /p"},
+		{"User-agent: *\nAllow: /folder\nDisallow: /folder\n", "Googlebot", "/folder/page", true, 2, "Allow: /folder"},
+		{"User-agent: *\nAllow: /page\nDisallow: /*.htm\n", "Googlebot", "/page.htm", false, 3, "Disallow: /*.htm"},
+		{"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/", true, 2, "Allow: /$"},
+		{"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/page.htm", false, 3, "Disallow: /"},
+		{"User-agent: *\nDisallow: /konto/\n", "Bingbot", "/blog", true, 0, ""},
+		{"User-agent: Googlebot\nDisallow: /private\n\nUser-agent: *\nDisallow: /\n", "Googlebot", "/public", true, 0, ""},
+	}
+	for _, c := range cases {
+		got := svc.TestRobots(c.content, none, c.agent, c.path)
+		if got.Allowed != c.allowed || got.Line != c.line || got.Rule != c.rule || got.Bot {
+			t.Fatalf("TestRobots(%q, %s, %s) = %+v, want allowed=%v line=%d rule=%q", c.content, c.agent, c.path, got, c.allowed, c.line, c.rule)
+		}
+	}
+	off := models.BotSwitches{"GPTBot": false}
+	if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "GPTBot", "/blog"); got.Allowed || !got.Bot || got.Rule != "Disallow: /" {
+		t.Fatalf("switched-off bot = %+v, want blocked by its switch", got)
+	}
+	if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "Googlebot", "/blog"); !got.Allowed || got.Bot {
+		t.Fatalf("Googlebot with GPTBot off = %+v, want allowed", got)
+	}
+	if got := svc.TestRobots("", off, "ClaudeBot", "/"); !got.Allowed {
+		t.Fatalf("a bot that is on = %+v, want allowed", got)
+	}
+}
--- a/routes.go
+++ b/routes.go
@@ -15,12 +15,16 @@ const (
 	BucketResolve = "seo-resolve"
 	// BucketNotFound is the 404 report endpoint's bucket.
 	BucketNotFound = "seo-notfound"
+	// BucketRobots is the /robots.txt bucket.
+	BucketRobots = "seo-robots"
 )

-// Routes mounts the /_seo/api/v1 endpoints. They carry no backend
-// middleware: the frontend server calls them anonymously (the 404 report
-// with its own token). Every answer carries X-Robots-Tag: noindex.
+// Routes mounts GET /robots.txt at the site root and the /_seo/api/v1
+// endpoints. None carries backend middleware: crawlers and the frontend
+// server call them anonymously (the 404 report with its own token). Every
+// /_seo/api/v1 answer carries X-Robots-Tag: noindex.
 func (p *Plugin) Routes(r pact.Router) error {
+	r.Get("/robots.txt", apicontrollers.Robots(p.Service), "throttle:"+BucketRobots)
 	r.Group("/_seo/api/v1", nil, func(g pact.Router) {
 		g.Get("/resolve", noindex(apicontrollers.Resolve(p.Service)), "throttle:"+BucketResolve)
 		g.Post("/notfound", noindex(apicontrollers.NotFound(p.Service)), "throttle:"+BucketNotFound)
@@ -38,8 +42,8 @@ func noindex(h http.HandlerFunc) http.HandlerFunc {

 // Buckets registers the per-client-IP buckets behind the configured
 // trusted proxies: seo-resolve 600 a minute (the frontend server resolves
-// every page it renders) and seo-notfound 600 a minute (one report per
-// 404 page).
+// every page it renders), seo-notfound 600 a minute (one report per 404
+// page) and seo-robots 120 a minute.
 func (p *Plugin) Buckets() map[string]surf.Bucket {
 	trusted := surf.TrustedProxies(nil)
 	if p.app != nil {
@@ -58,5 +62,6 @@ func (p *Plugin) Buckets() map[string]surf.Bucket {
 	return map[string]surf.Bucket{
 		BucketResolve:  bucket(BucketResolve, 600),
 		BucketNotFound: bucket(BucketNotFound, 600),
+		BucketRobots:   bucket(BucketRobots, 120),
 	}
 }
--- a/updates/202610270008_create_golem15_seo_robots.go
+++ b/updates/202610270008_create_golem15_seo_robots.go
@@ -0,0 +1,33 @@
+package updates
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// createRobots creates golem15_seo_robots, the robots.txt singleton (D-22,
+// D-25), with its row 1: no rules and every AI bot allowed.
+var createRobots = []*gormigrate.Migration{
+	{
+		ID: "202610270008_create_golem15_seo_robots",
+		Migrate: func(tx *gorm.DB) error {
+			if err := tx.Exec(`CREATE TABLE golem15_seo_robots (
+	id BIGSERIAL PRIMARY KEY,
+	content TEXT NOT NULL DEFAULT '',
+	bots JSONB NOT NULL DEFAULT '{}',
+	created_at TIMESTAMP(0) NULL,
+	updated_at TIMESTAMP(0) NULL
+)`).Error; err != nil {
+				return err
+			}
+			return tx.Exec(`INSERT INTO golem15_seo_robots (id, content, bots, created_at, updated_at) VALUES (1, '', '{}', NOW(), NOW())`).Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return tx.Exec(`DROP TABLE IF EXISTS golem15_seo_robots`).Error
+		},
+	},
+}
+
+func init() {
+	Register(createRobots...)
+}