From 3dd936202f7b26a2dab60c7d4e4e506d15b7bf84 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Tue, 27 Oct 2026 11:45:44 +0100 Subject: [PATCH] feat: robots.txt served file, AI bots and the non-production lock - github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the framework repo) - golem15_seo_robots singleton: rules and AI bot switches; eleven bots incl. Claude-User, all allowed by default - GET /robots.txt: rules, a Disallow group per switched-off bot and the golem15.seo.robots_sitemaps lines; outside production the lock with X-Robots-Tag; never a 5xx (last good body, else allow-all) - TestRobots on Google's matcher with the deciding line, and the noindex-blocked check (D-24) --- classes/bots.go | 55 +++++ classes/robots.go | 231 ++++++++++++++++++ classes/service.go | 4 + config/config.yaml | 4 + controllers/api/robots.go | 43 ++++ go.mod | 1 + go.sum | 65 +++++ models/robots.go | 82 +++++++ robots_test.go | 115 +++++++++ routes.go | 15 +- .../202610270008_create_golem15_seo_robots.go | 33 +++ 11 files changed, 643 insertions(+), 5 deletions(-) create mode 100644 classes/bots.go create mode 100644 classes/robots.go create mode 100644 controllers/api/robots.go create mode 100644 models/robots.go create mode 100644 robots_test.go create mode 100644 updates/202610270008_create_golem15_seo_robots.go diff --git a/classes/bots.go b/classes/bots.go new file mode 100644 index 0000000..4a89ee5 --- /dev/null +++ b/classes/bots.go @@ -0,0 +1,55 @@ +package classes + +import "strings" + +// The kinds of AI bots (D-25): search and answer crawlers, fetchers that +// load a page when a user asks for it, and crawlers that collect content +// for training. +const ( + BotSearch = "search" + BotUser = "user" + BotTraining = "training" +) + +// Bot is one AI crawler the robots.txt screen can switch off: its +// user-agent token, its operator and its kind. +type Bot struct { + Token string + Operator string + Kind string +} + +// Bots is the AI bot list in the screen's order (UI-SPEC C10, D-25, +// D-105): search bots, user-fetch bots, then training bots. Every bot is +// allowed until an editor switches it off. The list is plugin data and +// can grow without a layout change (Q-6). +var Bots = []Bot{ + {Token: "OAI-SearchBot", Operator: "OpenAI", Kind: BotSearch}, + {Token: "Claude-SearchBot", Operator: "Anthropic", Kind: BotSearch}, + {Token: "PerplexityBot", Operator: "Perplexity", Kind: BotSearch}, + {Token: "ChatGPT-User", Operator: "OpenAI", Kind: BotUser}, + {Token: "Claude-User", Operator: "Anthropic", Kind: BotUser}, + {Token: "GPTBot", Operator: "OpenAI", Kind: BotTraining}, + {Token: "ClaudeBot", Operator: "Anthropic", Kind: BotTraining}, + {Token: "Google-Extended", Operator: "Google", Kind: BotTraining}, + {Token: "Applebot-Extended", Operator: "Apple", Kind: BotTraining}, + {Token: "CCBot", Operator: "Common Crawl", Kind: BotTraining}, + {Token: "Meta-ExternalAgent", Operator: "Meta", Kind: BotTraining}, +} + +// BotField is the robots form's switch field of a bot: bot_ and its token +// lowercased with - as _ (bot_oai_searchbot). +func BotField(token string) string { + return "bot_" + strings.ReplaceAll(strings.ToLower(token), "-", "_") +} + +// BotByToken returns the listed bot whose token equals token (letter case +// ignored). +func BotByToken(token string) (Bot, bool) { + for _, b := range Bots { + if strings.EqualFold(b.Token, token) { + return b, true + } + } + return Bot{}, false +} diff --git a/classes/robots.go b/classes/robots.go new file mode 100644 index 0000000..0edf851 --- /dev/null +++ b/classes/robots.go @@ -0,0 +1,231 @@ +package classes + +import ( + "context" + "sort" + "strings" + + "git.golem15.com/golem15/sm-seo-plugin/models" + "github.com/jimsmart/grobotstxt" + "gorm.io/gorm" +) + +// RobotsSitemapsConfigKey lists the sitemap addresses robots.txt names, +// one Sitemap line each, absolute on the site URL (P-23); default +// ["/sitemap.xml"]. +const RobotsSitemapsConfigKey = "golem15.seo.robots_sitemaps" + +// MaxRobotsBytes is the largest robots.txt the editor stores: Google +// ignores everything after 500 KiB. +const MaxRobotsBytes = 500 << 10 + +// The two fixed robots.txt bodies. +const ( + // NonProductionRobots is served outside production whatever is stored + // (D-22). + NonProductionRobots = "User-agent: *\nDisallow: /\n" + // FallbackRobots is served when the stored file cannot be read and no + // earlier body is cached: allow everything, never a 5xx. + FallbackRobots = "User-agent: *\nDisallow:\n" +) + +// RobotsSitemaps returns the configured sitemap addresses, absolute on +// the site URL. +func (s *Service) RobotsSitemaps() []string { + var list []string + if cfg := s.Config(); cfg != nil { + if raw, ok := cfg.Lookup(RobotsSitemapsConfigKey); ok { + switch v := raw.(type) { + case []any: + for _, item := range v { + if str, ok := item.(string); ok { + list = append(list, str) + } + } + case []string: + list = append(list, v...) + case string: + list = strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' }) + } + } + } + if list == nil { + list = []string{"/sitemap.xml"} + } + out := make([]string, 0, len(list)) + for _, item := range list { + if item = strings.TrimSpace(item); item != "" { + out = append(out, AbsoluteURL(s.SiteURL(), item)) + } + } + return out +} + +// LoadRobots reads robots row 1; without one the defaults apply (no +// rules, every bot allowed). +func LoadRobots(db *gorm.DB) (models.Robots, error) { + var rows []models.Robots + if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil { + return models.Robots{}, err + } + if len(rows) == 0 { + return models.Robots{ID: 1, Bots: models.BotSwitches{}}, nil + } + return rows[0], nil +} + +// ComposeRobots builds the production robots.txt (D-22, D-25, P-23): the +// stored rules, one "User-agent: " / "Disallow: /" group per +// switched-off bot, then one Sitemap line per sitemap address. +func ComposeRobots(content string, bots models.BotSwitches, sitemaps []string) string { + var b strings.Builder + content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t") + if content != "" { + b.WriteString(content) + b.WriteString("\n") + } + for _, bot := range Bots { + if bots.Allowed(bot.Token) { + continue + } + if b.Len() > 0 { + b.WriteString("\n") + } + b.WriteString("User-agent: " + bot.Token + "\nDisallow: /\n") + } + if len(sitemaps) > 0 && b.Len() > 0 { + b.WriteString("\n") + } + for _, sm := range sitemaps { + b.WriteString("Sitemap: " + sm + "\n") + } + return b.String() +} + +// generatedStart is the 1-based line where the generated part of a +// composed file starts: the lines after the stored rules. +func generatedStart(content string) int { + content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t") + if content == "" { + return 1 + } + return strings.Count(content, "\n") + 2 +} + +// ServedFile is the robots.txt crawlers get (D-22): outside production +// always NonProductionRobots, else the composed file from the stored rules +// and switches. It caches the last body it served, which LastServedRobots +// answers when the store fails. +func (s *Service) ServedFile(ctx context.Context, production bool) (string, error) { + if !production { + return NonProductionRobots, nil + } + db, err := s.DB(ctx) + if err != nil { + return "", err + } + row, err := LoadRobots(db) + if err != nil { + return "", err + } + body := ComposeRobots(row.Content, row.Bots, s.RobotsSitemaps()) + if s != nil { + s.robots.Store(&body) + } + return body, nil +} + +// LastServedRobots returns the last production body ServedFile answered, +// "" when there is none. +func (s *Service) LastServedRobots() string { + if s == nil { + return "" + } + if p := s.robots.Load(); p != nil { + return *p + } + return "" +} + +// RobotsTest is the tester's answer (UI-SPEC E5): whether agent may fetch +// the address, the deciding line (0 and "" when no rule matched) and +// whether a bot switch decided it. +type RobotsTest struct { + Allowed bool `json:"allowed"` + Line int `json:"line"` + Rule string `json:"rule"` + Bot bool `json:"bot"` + Agent string `json:"agent"` + Staging bool `json:"staging"` +} + +// TestRobots answers whether agent may fetch path under the production +// file composed from content and bots (unsaved values included), through +// grobotstxt, Google's matcher (D-22, D-99): longest match, Allow wins a +// tie, * and $. +func (s *Service) TestRobots(content string, bots models.BotSwitches, agent, path string) RobotsTest { + file := ComposeRobots(content, bots, s.RobotsSitemaps()) + uri := AbsoluteURL(s.SiteURL(), path) + if s.SiteURL() == "" { + uri = "http://localhost" + path + } + m := grobotstxt.NewRobotsMatcher() + out := RobotsTest{Agent: agent, Staging: s.Staging()} + out.Allowed = m.AgentAllowed(file, agent, uri) + if line := m.MatchingLine(); line > 0 { + out.Line = line + lines := strings.Split(file, "\n") + if line <= len(lines) { + out.Rule = strings.TrimSpace(lines[line-1]) + } + out.Bot = !out.Allowed && line >= generatedStart(content) + } + return out +} + +// noindexAddresses lists the addresses the site serves noindex that a +// robots.txt could block (D-24): exact URL rules and records whose own +// robots value says noindex. +func noindexAddresses(db *gorm.DB) ([]string, error) { + seen := map[string]bool{} + var out []string + var rules []models.Rule + if err := db.Where("match = ? AND robots LIKE ?", models.MatchExact, "%noindex%").Find(&rules).Error; err != nil { + return nil, err + } + for _, r := range rules { + if !seen[r.URL] { + seen[r.URL] = true + out = append(out, r.URL) + } + } + var entries []models.Entry + if err := db.Where("robots LIKE ? AND url <> ''", "%noindex%").Find(&entries).Error; err != nil { + return nil, err + } + for _, e := range entries { + if !seen[e.URL] { + seen[e.URL] = true + out = append(out, e.URL) + } + } + sort.Strings(out) + return out, nil +} + +// BlockedNoindex lists the noindex addresses Googlebot may not fetch under +// the production file composed from content and bots (D-24): a crawler +// that cannot fetch a page never sees its noindex. +func (s *Service) BlockedNoindex(db *gorm.DB, content string, bots models.BotSwitches) ([]string, error) { + urls, err := noindexAddresses(db) + if err != nil { + return nil, err + } + var out []string + for _, u := range urls { + if !s.TestRobots(content, bots, "Googlebot", u).Allowed { + out = append(out, u) + } + } + return out, nil +} diff --git a/classes/service.go b/classes/service.go index 8034a68..a4af6ff 100644 --- a/classes/service.go +++ b/classes/service.go @@ -5,6 +5,7 @@ package classes import ( "context" "errors" + "sync/atomic" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/cabana" @@ -22,6 +23,9 @@ var ErrNotBooted = errors.New("golem15.seo: app is not booted") // history. type Service struct { app *backpack.App + // robots is the last production robots.txt served, answered when the + // store fails (D-22: never a 5xx). + robots atomic.Pointer[string] } // NewService returns the service of a booted application. The database diff --git a/config/config.yaml b/config/config.yaml index b0b703a..be46038 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -19,3 +19,7 @@ notfound_token: "" # notfound_max_rows caps the 404 log; new addresses beyond it are dropped, # with one warning an hour. notfound_max_rows: 10000 +# +# robots_sitemaps lists the sitemap addresses robots.txt names, one +# "Sitemap:" line each, made absolute on the site URL. +robots_sitemaps: ["/sitemap.xml"] diff --git a/controllers/api/robots.go b/controllers/api/robots.go new file mode 100644 index 0000000..157e26b --- /dev/null +++ b/controllers/api/robots.go @@ -0,0 +1,43 @@ +package api + +import ( + "net/http" + + "git.golem15.com/golem15/sm-seo-plugin/classes" +) + +// Robots serves GET /robots.txt (D-22, D-25, P-23) as text/plain. In +// production it is the stored rules, a Disallow group for every +// switched-off AI bot and the Sitemap lines; outside production it is +// "User-agent: *" / "Disallow: /" with X-Robots-Tag: noindex, nofollow, +// whatever is stored, and no admin setting can change that. It never +// answers a 5xx: when the stored file cannot be read it serves the last +// body it served, else an allow-all file, and logs the error. +func Robots(service func() *classes.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + svc := service() + body := classes.FallbackRobots + staging := false + if svc == nil { + log.Error("golem15.seo: robots.txt served before boot") + } else { + staging = svc.Staging() + served, err := svc.ServedFile(r.Context(), !staging) + switch { + case err == nil: + body = served + case svc.LastServedRobots() != "": + log.Error("golem15.seo: robots.txt, serving the last good file", "error", err) + body = svc.LastServedRobots() + default: + log.Error("golem15.seo: robots.txt, serving the allow-all fallback", "error", err) + } + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + if staging { + w.Header().Set("X-Robots-Tag", "noindex, nofollow") + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(body)) + } +} diff --git a/go.mod b/go.mod index 24fc64b..0d3ff9b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/go-gormigrate/gormigrate/v2 v2.1.7 github.com/goccy/go-yaml v1.19.2 github.com/jackc/pgx/v5 v5.10.0 + github.com/jimsmart/grobotstxt v1.0.3 github.com/testcontainers/testcontainers-go v0.44.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 gocloud.dev v0.46.0 diff --git a/go.sum b/go.sum index abcd5a0..e5ae083 100644 --- a/go.sum +++ b/go.sum @@ -165,6 +165,8 @@ github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMD github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI= @@ -195,8 +197,18 @@ github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo= @@ -219,6 +231,7 @@ github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTs github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0= @@ -231,6 +244,8 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jimsmart/grobotstxt v1.0.3 h1:DUP8ERo4MqVe0MZsudcz/ROKJcMhguGQLrVmA+8rDlM= +github.com/jimsmart/grobotstxt v1.0.3/go.mod h1:WImegD7gBR7B9I1UOrcuoQHmeflNp267CIHkQmZOiYU= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -309,6 +324,17 @@ github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd17NIEQ= github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.6 h1:11TGpSHY7Esh/i/qnq02Jo5oVrI1Gue8Slbq0ujPZFQ= +github.com/nxadm/tail v1.4.6/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= +github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.4 h1:NiTx7EEvBzu9sFOD1zORteLSt3o8gnlvZZwSE9TnY9U= +github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -420,6 +446,8 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= @@ -427,27 +455,51 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQz golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ= golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= @@ -462,9 +514,22 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/postgres v1.6.3 h1:bAn6O2pUa8LtpWEvL5NFU4+52Tfx8Ut7IVaIacCLcI0= gorm.io/driver/postgres v1.6.3/go.mod h1:0c4fQA44XhOklXDkgtuKqysHCycTa5i9e3EIpDGCwXk= diff --git a/models/robots.go b/models/robots.go new file mode 100644 index 0000000..44c1c02 --- /dev/null +++ b/models/robots.go @@ -0,0 +1,82 @@ +package models + +import ( + "database/sql/driver" + "encoding/json" + "errors" + "time" +) + +// Robots is the robots.txt singleton (row id 1) on golem15_seo_robots +// (D-22, D-25): the editor's rules and the AI bot switches. The served +// file adds a Disallow group for every switched-off bot and the Sitemap +// lines; outside production it is replaced by the non-production lock. +type Robots struct { + ID uint `gorm:"column:id;primaryKey"` + Content string `gorm:"column:content"` + Bots BotSwitches `gorm:"column:bots"` + CreatedAt *time.Time `gorm:"column:created_at"` + UpdatedAt *time.Time `gorm:"column:updated_at"` +} + +func (Robots) TableName() string { return "golem15_seo_robots" } + +// MorphName names the robots row in history. +func (Robots) MorphName() string { return `Golem15\SEO\Models\Robots` } + +// Fillable lists the form's columns: the rules. The bot switches are +// written by the robots form's extension. +func (Robots) Fillable() []string { return []string{"content"} } + +// Rules is empty: the robotstxt field type refuses an oversized file. +func (Robots) Rules() map[string]string { return map[string]string{} } + +// BotSwitches maps a bot's user-agent token to whether it is allowed. A +// bot that is not listed is allowed (D-25: every bot on by default). +type BotSwitches map[string]bool + +// Allowed reports whether token is switched on. +func (b BotSwitches) Allowed(token string) bool { + on, ok := b[token] + return !ok || on +} + +// Value encodes the switches as a JSON object; nil is {}. +func (b BotSwitches) Value() (driver.Value, error) { + if b == nil { + return "{}", nil + } + raw, err := json.Marshal(map[string]bool(b)) + if err != nil { + return nil, err + } + return string(raw), nil +} + +// Scan decodes a jsonb object; NULL reads as none. +func (b *BotSwitches) Scan(src any) error { + var raw []byte + switch v := src.(type) { + case nil: + *b = BotSwitches{} + return nil + case []byte: + raw = v + case string: + raw = []byte(v) + default: + return errors.New("models: unsupported bots value") + } + out := map[string]bool{} + if len(raw) > 0 { + if err := json.Unmarshal(raw, &out); err != nil { + return err + } + } + *b = out + return nil +} + +func init() { + Register(Robots{}) +} diff --git a/robots_test.go b/robots_test.go new file mode 100644 index 0000000..a837767 --- /dev/null +++ b/robots_test.go @@ -0,0 +1,115 @@ +package seo_test + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.golem15.com/golem15/sm-seo-plugin/models" +) + +// robotsTxt fetches /robots.txt. +func (e *testEnv) robotsTxt(t *testing.T) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil) + req.RemoteAddr = "198.51.100.9:4000" + rec := httptest.NewRecorder() + e.h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("/robots.txt = %d %s, want 200", rec.Code, rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") { + t.Fatalf("/robots.txt content type = %q, want text/plain", ct) + } + return rec +} + +// TestRobotsServed checks GET /robots.txt (D-22, D-25, P-23, T-32-19, +// T-32-20): the stored rules, a Disallow group per switched-off AI bot and +// the Sitemap line in production; the non-production lock whatever is +// stored; and a 200 with the last good or the allow-all file when the +// store cannot be read. +func TestRobotsServed(t *testing.T) { + env := bootEnv(t) + if body := env.robotsTxt(t).Body.String(); body != "Sitemap: "+siteURL+"/sitemap.xml\n" { + t.Fatalf("fresh robots.txt = %q, want only the Sitemap line", body) + } + if err := env.db.Exec(`UPDATE golem15_seo_robots SET content = ?, bots = ? WHERE id = 1`, + "User-agent: *\nDisallow: /admin\n", `{"GPTBot": false, "CCBot": false, "ClaudeBot": true}`).Error; err != nil { + t.Fatal(err) + } + rec := env.robotsTxt(t) + want := "User-agent: *\nDisallow: /admin\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nSitemap: " + siteURL + "/sitemap.xml\n" + if body := rec.Body.String(); body != want { + t.Fatalf("robots.txt = %q, want %q", body, want) + } + if rec.Header().Get("X-Robots-Tag") != "" { + t.Fatalf("production robots.txt carries X-Robots-Tag %q", rec.Header().Get("X-Robots-Tag")) + } + + // The store fails: the last good body, still 200. + if err := env.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil { + t.Fatal(err) + } + if body := env.robotsTxt(t).Body.String(); body != want { + t.Fatalf("robots.txt with the store gone = %q, want the last good body", body) + } + fresh := bootEnv(t) + if err := fresh.db.Exec(`DROP TABLE golem15_seo_robots`).Error; err != nil { + t.Fatal(err) + } + if body := fresh.robotsTxt(t).Body.String(); body != "User-agent: *\nDisallow:\n" { + t.Fatalf("robots.txt with the store gone and nothing cached = %q, want allow-all", body) + } + + // Outside production: the lock, whatever is stored. + staging := bootEnv(t, "SUMMER_ENV=staging") + if err := staging.db.Exec(`UPDATE golem15_seo_robots SET content = ? WHERE id = 1`, "User-agent: *\nAllow: /\n").Error; err != nil { + t.Fatal(err) + } + rec = staging.robotsTxt(t) + if rec.Body.String() != "User-agent: *\nDisallow: /\n" || rec.Header().Get("X-Robots-Tag") != "noindex, nofollow" { + t.Fatalf("staging robots.txt = %q %v, want the lock with X-Robots-Tag", rec.Body.String(), rec.Header()) + } +} + +// TestRobotsTester checks the tester against Google's documented examples +// (D-22, D-99): the longest match wins, Allow wins a tie, $ anchors the +// end, the matching line is reported, and a switched-off bot is blocked by +// its switch. +func TestRobotsTester(t *testing.T) { + env := bootEnv(t) + svc := env.seoPlugin(t).Service() + none := models.BotSwitches{} + cases := []struct { + content, agent, path string + allowed bool + line int + rule string + }{ + {"User-agent: *\nAllow: /p\nDisallow: /\n", "Googlebot", "/page", true, 2, "Allow: /p"}, + {"User-agent: *\nAllow: /folder\nDisallow: /folder\n", "Googlebot", "/folder/page", true, 2, "Allow: /folder"}, + {"User-agent: *\nAllow: /page\nDisallow: /*.htm\n", "Googlebot", "/page.htm", false, 3, "Disallow: /*.htm"}, + {"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/", true, 2, "Allow: /$"}, + {"User-agent: *\nAllow: /$\nDisallow: /\n", "Googlebot", "/page.htm", false, 3, "Disallow: /"}, + {"User-agent: *\nDisallow: /konto/\n", "Bingbot", "/blog", true, 0, ""}, + {"User-agent: Googlebot\nDisallow: /private\n\nUser-agent: *\nDisallow: /\n", "Googlebot", "/public", true, 0, ""}, + } + for _, c := range cases { + got := svc.TestRobots(c.content, none, c.agent, c.path) + if got.Allowed != c.allowed || got.Line != c.line || got.Rule != c.rule || got.Bot { + t.Fatalf("TestRobots(%q, %s, %s) = %+v, want allowed=%v line=%d rule=%q", c.content, c.agent, c.path, got, c.allowed, c.line, c.rule) + } + } + off := models.BotSwitches{"GPTBot": false} + if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "GPTBot", "/blog"); got.Allowed || !got.Bot || got.Rule != "Disallow: /" { + t.Fatalf("switched-off bot = %+v, want blocked by its switch", got) + } + if got := svc.TestRobots("User-agent: *\nAllow: /\n", off, "Googlebot", "/blog"); !got.Allowed || got.Bot { + t.Fatalf("Googlebot with GPTBot off = %+v, want allowed", got) + } + if got := svc.TestRobots("", off, "ClaudeBot", "/"); !got.Allowed { + t.Fatalf("a bot that is on = %+v, want allowed", got) + } +} diff --git a/routes.go b/routes.go index 66ba741..600370c 100644 --- a/routes.go +++ b/routes.go @@ -15,12 +15,16 @@ const ( BucketResolve = "seo-resolve" // BucketNotFound is the 404 report endpoint's bucket. BucketNotFound = "seo-notfound" + // BucketRobots is the /robots.txt bucket. + BucketRobots = "seo-robots" ) -// Routes mounts the /_seo/api/v1 endpoints. They carry no backend -// middleware: the frontend server calls them anonymously (the 404 report -// with its own token). Every answer carries X-Robots-Tag: noindex. +// Routes mounts GET /robots.txt at the site root and the /_seo/api/v1 +// endpoints. None carries backend middleware: crawlers and the frontend +// server call them anonymously (the 404 report with its own token). Every +// /_seo/api/v1 answer carries X-Robots-Tag: noindex. func (p *Plugin) Routes(r pact.Router) error { + r.Get("/robots.txt", apicontrollers.Robots(p.Service), "throttle:"+BucketRobots) r.Group("/_seo/api/v1", nil, func(g pact.Router) { g.Get("/resolve", noindex(apicontrollers.Resolve(p.Service)), "throttle:"+BucketResolve) g.Post("/notfound", noindex(apicontrollers.NotFound(p.Service)), "throttle:"+BucketNotFound) @@ -38,8 +42,8 @@ func noindex(h http.HandlerFunc) http.HandlerFunc { // Buckets registers the per-client-IP buckets behind the configured // trusted proxies: seo-resolve 600 a minute (the frontend server resolves -// every page it renders) and seo-notfound 600 a minute (one report per -// 404 page). +// every page it renders), seo-notfound 600 a minute (one report per 404 +// page) and seo-robots 120 a minute. func (p *Plugin) Buckets() map[string]surf.Bucket { trusted := surf.TrustedProxies(nil) if p.app != nil { @@ -58,5 +62,6 @@ func (p *Plugin) Buckets() map[string]surf.Bucket { return map[string]surf.Bucket{ BucketResolve: bucket(BucketResolve, 600), BucketNotFound: bucket(BucketNotFound, 600), + BucketRobots: bucket(BucketRobots, 120), } } diff --git a/updates/202610270008_create_golem15_seo_robots.go b/updates/202610270008_create_golem15_seo_robots.go new file mode 100644 index 0000000..98907dc --- /dev/null +++ b/updates/202610270008_create_golem15_seo_robots.go @@ -0,0 +1,33 @@ +package updates + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// createRobots creates golem15_seo_robots, the robots.txt singleton (D-22, +// D-25), with its row 1: no rules and every AI bot allowed. +var createRobots = []*gormigrate.Migration{ + { + ID: "202610270008_create_golem15_seo_robots", + Migrate: func(tx *gorm.DB) error { + if err := tx.Exec(`CREATE TABLE golem15_seo_robots ( + id BIGSERIAL PRIMARY KEY, + content TEXT NOT NULL DEFAULT '', + bots JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMP(0) NULL, + updated_at TIMESTAMP(0) NULL +)`).Error; err != nil { + return err + } + return tx.Exec(`INSERT INTO golem15_seo_robots (id, content, bots, created_at, updated_at) VALUES (1, '', '{}', NOW(), NOW())`).Error + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec(`DROP TABLE IF EXISTS golem15_seo_robots`).Error + }, + }, +} + +func init() { + Register(createRobots...) +}