feat: robots.txt served file, AI bots and the non-production lock

- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the framework repo) - golem15_seo_robots singleton: rules and AI bot switches; eleven bots incl. Claude-User, all allowed by default - GET /robots.txt: rules, a Disallow group per switched-off bot and the golem15.seo.robots_sitemaps lines; outside production the lock with X-Robots-Tag; never a 5xx (last good body, else allow-all) - TestRobots on Google's matcher with the deciding line, and the noindex-blocked check (D-24)
2026-10-27 11:45:44 +01:00
parent 4cb0fbfebb
commit 3dd936202f
11 changed files with 643 additions and 5 deletions
--- a/classes/robots.go
+++ b/classes/robots.go
@@ -0,0 +1,231 @@
+package classes
+
+import (
+	"context"
+	"sort"
+	"strings"
+
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"github.com/jimsmart/grobotstxt"
+	"gorm.io/gorm"
+)
+
+// RobotsSitemapsConfigKey lists the sitemap addresses robots.txt names,
+// one Sitemap line each, absolute on the site URL (P-23); default
+// ["/sitemap.xml"].
+const RobotsSitemapsConfigKey = "golem15.seo.robots_sitemaps"
+
+// MaxRobotsBytes is the largest robots.txt the editor stores: Google
+// ignores everything after 500 KiB.
+const MaxRobotsBytes = 500 << 10
+
+// The two fixed robots.txt bodies.
+const (
+	// NonProductionRobots is served outside production whatever is stored
+	// (D-22).
+	NonProductionRobots = "User-agent: *\nDisallow: /\n"
+	// FallbackRobots is served when the stored file cannot be read and no
+	// earlier body is cached: allow everything, never a 5xx.
+	FallbackRobots = "User-agent: *\nDisallow:\n"
+)
+
+// RobotsSitemaps returns the configured sitemap addresses, absolute on
+// the site URL.
+func (s *Service) RobotsSitemaps() []string {
+	var list []string
+	if cfg := s.Config(); cfg != nil {
+		if raw, ok := cfg.Lookup(RobotsSitemapsConfigKey); ok {
+			switch v := raw.(type) {
+			case []any:
+				for _, item := range v {
+					if str, ok := item.(string); ok {
+						list = append(list, str)
+					}
+				}
+			case []string:
+				list = append(list, v...)
+			case string:
+				list = strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' })
+			}
+		}
+	}
+	if list == nil {
+		list = []string{"/sitemap.xml"}
+	}
+	out := make([]string, 0, len(list))
+	for _, item := range list {
+		if item = strings.TrimSpace(item); item != "" {
+			out = append(out, AbsoluteURL(s.SiteURL(), item))
+		}
+	}
+	return out
+}
+
+// LoadRobots reads robots row 1; without one the defaults apply (no
+// rules, every bot allowed).
+func LoadRobots(db *gorm.DB) (models.Robots, error) {
+	var rows []models.Robots
+	if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil {
+		return models.Robots{}, err
+	}
+	if len(rows) == 0 {
+		return models.Robots{ID: 1, Bots: models.BotSwitches{}}, nil
+	}
+	return rows[0], nil
+}
+
+// ComposeRobots builds the production robots.txt (D-22, D-25, P-23): the
+// stored rules, one "User-agent: <token>" / "Disallow: /" group per
+// switched-off bot, then one Sitemap line per sitemap address.
+func ComposeRobots(content string, bots models.BotSwitches, sitemaps []string) string {
+	var b strings.Builder
+	content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t")
+	if content != "" {
+		b.WriteString(content)
+		b.WriteString("\n")
+	}
+	for _, bot := range Bots {
+		if bots.Allowed(bot.Token) {
+			continue
+		}
+		if b.Len() > 0 {
+			b.WriteString("\n")
+		}
+		b.WriteString("User-agent: " + bot.Token + "\nDisallow: /\n")
+	}
+	if len(sitemaps) > 0 && b.Len() > 0 {
+		b.WriteString("\n")
+	}
+	for _, sm := range sitemaps {
+		b.WriteString("Sitemap: " + sm + "\n")
+	}
+	return b.String()
+}
+
+// generatedStart is the 1-based line where the generated part of a
+// composed file starts: the lines after the stored rules.
+func generatedStart(content string) int {
+	content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t")
+	if content == "" {
+		return 1
+	}
+	return strings.Count(content, "\n") + 2
+}
+
+// ServedFile is the robots.txt crawlers get (D-22): outside production
+// always NonProductionRobots, else the composed file from the stored rules
+// and switches. It caches the last body it served, which LastServedRobots
+// answers when the store fails.
+func (s *Service) ServedFile(ctx context.Context, production bool) (string, error) {
+	if !production {
+		return NonProductionRobots, nil
+	}
+	db, err := s.DB(ctx)
+	if err != nil {
+		return "", err
+	}
+	row, err := LoadRobots(db)
+	if err != nil {
+		return "", err
+	}
+	body := ComposeRobots(row.Content, row.Bots, s.RobotsSitemaps())
+	if s != nil {
+		s.robots.Store(&body)
+	}
+	return body, nil
+}
+
+// LastServedRobots returns the last production body ServedFile answered,
+// "" when there is none.
+func (s *Service) LastServedRobots() string {
+	if s == nil {
+		return ""
+	}
+	if p := s.robots.Load(); p != nil {
+		return *p
+	}
+	return ""
+}
+
+// RobotsTest is the tester's answer (UI-SPEC E5): whether agent may fetch
+// the address, the deciding line (0 and "" when no rule matched) and
+// whether a bot switch decided it.
+type RobotsTest struct {
+	Allowed bool   `json:"allowed"`
+	Line    int    `json:"line"`
+	Rule    string `json:"rule"`
+	Bot     bool   `json:"bot"`
+	Agent   string `json:"agent"`
+	Staging bool   `json:"staging"`
+}
+
+// TestRobots answers whether agent may fetch path under the production
+// file composed from content and bots (unsaved values included), through
+// grobotstxt, Google's matcher (D-22, D-99): longest match, Allow wins a
+// tie, * and $.
+func (s *Service) TestRobots(content string, bots models.BotSwitches, agent, path string) RobotsTest {
+	file := ComposeRobots(content, bots, s.RobotsSitemaps())
+	uri := AbsoluteURL(s.SiteURL(), path)
+	if s.SiteURL() == "" {
+		uri = "http://localhost" + path
+	}
+	m := grobotstxt.NewRobotsMatcher()
+	out := RobotsTest{Agent: agent, Staging: s.Staging()}
+	out.Allowed = m.AgentAllowed(file, agent, uri)
+	if line := m.MatchingLine(); line > 0 {
+		out.Line = line
+		lines := strings.Split(file, "\n")
+		if line <= len(lines) {
+			out.Rule = strings.TrimSpace(lines[line-1])
+		}
+		out.Bot = !out.Allowed && line >= generatedStart(content)
+	}
+	return out
+}
+
+// noindexAddresses lists the addresses the site serves noindex that a
+// robots.txt could block (D-24): exact URL rules and records whose own
+// robots value says noindex.
+func noindexAddresses(db *gorm.DB) ([]string, error) {
+	seen := map[string]bool{}
+	var out []string
+	var rules []models.Rule
+	if err := db.Where("match = ? AND robots LIKE ?", models.MatchExact, "%noindex%").Find(&rules).Error; err != nil {
+		return nil, err
+	}
+	for _, r := range rules {
+		if !seen[r.URL] {
+			seen[r.URL] = true
+			out = append(out, r.URL)
+		}
+	}
+	var entries []models.Entry
+	if err := db.Where("robots LIKE ? AND url <> ''", "%noindex%").Find(&entries).Error; err != nil {
+		return nil, err
+	}
+	for _, e := range entries {
+		if !seen[e.URL] {
+			seen[e.URL] = true
+			out = append(out, e.URL)
+		}
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+// BlockedNoindex lists the noindex addresses Googlebot may not fetch under
+// the production file composed from content and bots (D-24): a crawler
+// that cannot fetch a page never sees its noindex.
+func (s *Service) BlockedNoindex(db *gorm.DB, content string, bots models.BotSwitches) ([]string, error) {
+	urls, err := noindexAddresses(db)
+	if err != nil {
+		return nil, err
+	}
+	var out []string
+	for _, u := range urls {
+		if !s.TestRobots(content, bots, "Googlebot", u).Allowed {
+			out = append(out, u)
+		}
+	}
+	return out, nil
+}