feat: robots.txt served file, AI bots and the non-production lock

- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the
  framework repo)
- golem15_seo_robots singleton: rules and AI bot switches; eleven bots
  incl. Claude-User, all allowed by default
- GET /robots.txt: rules, a Disallow group per switched-off bot and the
  golem15.seo.robots_sitemaps lines; outside production the lock with
  X-Robots-Tag; never a 5xx (last good body, else allow-all)
- TestRobots on Google's matcher with the deciding line, and the
  noindex-blocked check (D-24)

controllers/api/robots.go

@@ -0,0 +1,43 @@
+package api
+
+import (
+	"net/http"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+)
+
+// Robots serves GET /robots.txt (D-22, D-25, P-23) as text/plain. In
+// production it is the stored rules, a Disallow group for every
+// switched-off AI bot and the Sitemap lines; outside production it is
+// "User-agent: *" / "Disallow: /" with X-Robots-Tag: noindex, nofollow,
+// whatever is stored, and no admin setting can change that. It never
+// answers a 5xx: when the stored file cannot be read it serves the last
+// body it served, else an allow-all file, and logs the error.
+func Robots(service func() *classes.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		svc := service()
+		body := classes.FallbackRobots
+		staging := false
+		if svc == nil {
+			log.Error("golem15.seo: robots.txt served before boot")
+		} else {
+			staging = svc.Staging()
+			served, err := svc.ServedFile(r.Context(), !staging)
+			switch {
+			case err == nil:
+				body = served
+			case svc.LastServedRobots() != "":
+				log.Error("golem15.seo: robots.txt, serving the last good file", "error", err)
+				body = svc.LastServedRobots()
+			default:
+				log.Error("golem15.seo: robots.txt, serving the allow-all fallback", "error", err)
+			}
+		}
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		if staging {
+			w.Header().Set("X-Robots-Tag", "noindex, nofollow")
+		}
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(body))
+	}
+}