feat: robots.txt served file, AI bots and the non-production lock

- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the
  framework repo)
- golem15_seo_robots singleton: rules and AI bot switches; eleven bots
  incl. Claude-User, all allowed by default
- GET /robots.txt: rules, a Disallow group per switched-off bot and the
  golem15.seo.robots_sitemaps lines; outside production the lock with
  X-Robots-Tag; never a 5xx (last good body, else allow-all)
- TestRobots on Google's matcher with the deciding line, and the
  noindex-blocked check (D-24)

models/robots.go

@@ -0,0 +1,82 @@
+package models
+
+import (
+	"database/sql/driver"
+	"encoding/json"
+	"errors"
+	"time"
+)
+
+// Robots is the robots.txt singleton (row id 1) on golem15_seo_robots
+// (D-22, D-25): the editor's rules and the AI bot switches. The served
+// file adds a Disallow group for every switched-off bot and the Sitemap
+// lines; outside production it is replaced by the non-production lock.
+type Robots struct {
+	ID        uint        `gorm:"column:id;primaryKey"`
+	Content   string      `gorm:"column:content"`
+	Bots      BotSwitches `gorm:"column:bots"`
+	CreatedAt *time.Time  `gorm:"column:created_at"`
+	UpdatedAt *time.Time  `gorm:"column:updated_at"`
+}
+
+func (Robots) TableName() string { return "golem15_seo_robots" }
+
+// MorphName names the robots row in history.
+func (Robots) MorphName() string { return `Golem15\SEO\Models\Robots` }
+
+// Fillable lists the form's columns: the rules. The bot switches are
+// written by the robots form's extension.
+func (Robots) Fillable() []string { return []string{"content"} }
+
+// Rules is empty: the robotstxt field type refuses an oversized file.
+func (Robots) Rules() map[string]string { return map[string]string{} }
+
+// BotSwitches maps a bot's user-agent token to whether it is allowed. A
+// bot that is not listed is allowed (D-25: every bot on by default).
+type BotSwitches map[string]bool
+
+// Allowed reports whether token is switched on.
+func (b BotSwitches) Allowed(token string) bool {
+	on, ok := b[token]
+	return !ok || on
+}
+
+// Value encodes the switches as a JSON object; nil is {}.
+func (b BotSwitches) Value() (driver.Value, error) {
+	if b == nil {
+		return "{}", nil
+	}
+	raw, err := json.Marshal(map[string]bool(b))
+	if err != nil {
+		return nil, err
+	}
+	return string(raw), nil
+}
+
+// Scan decodes a jsonb object; NULL reads as none.
+func (b *BotSwitches) Scan(src any) error {
+	var raw []byte
+	switch v := src.(type) {
+	case nil:
+		*b = BotSwitches{}
+		return nil
+	case []byte:
+		raw = v
+	case string:
+		raw = []byte(v)
+	default:
+		return errors.New("models: unsupported bots value")
+	}
+	out := map[string]bool{}
+	if len(raw) > 0 {
+		if err := json.Unmarshal(raw, &out); err != nil {
+			return err
+		}
+	}
+	*b = out
+	return nil
+}
+
+func init() {
+	Register(Robots{})
+}