feat: robots.txt served file, AI bots and the non-production lock

- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the
  framework repo)
- golem15_seo_robots singleton: rules and AI bot switches; eleven bots
  incl. Claude-User, all allowed by default
- GET /robots.txt: rules, a Disallow group per switched-off bot and the
  golem15.seo.robots_sitemaps lines; outside production the lock with
  X-Robots-Tag; never a 5xx (last good body, else allow-all)
- TestRobots on Google's matcher with the deciding line, and the
  noindex-blocked check (D-24)

classes/bots.go

@@ -0,0 +1,55 @@
+package classes
+
+import "strings"
+
+// The kinds of AI bots (D-25): search and answer crawlers, fetchers that
+// load a page when a user asks for it, and crawlers that collect content
+// for training.
+const (
+	BotSearch   = "search"
+	BotUser     = "user"
+	BotTraining = "training"
+)
+
+// Bot is one AI crawler the robots.txt screen can switch off: its
+// user-agent token, its operator and its kind.
+type Bot struct {
+	Token    string
+	Operator string
+	Kind     string
+}
+
+// Bots is the AI bot list in the screen's order (UI-SPEC C10, D-25,
+// D-105): search bots, user-fetch bots, then training bots. Every bot is
+// allowed until an editor switches it off. The list is plugin data and
+// can grow without a layout change (Q-6).
+var Bots = []Bot{
+	{Token: "OAI-SearchBot", Operator: "OpenAI", Kind: BotSearch},
+	{Token: "Claude-SearchBot", Operator: "Anthropic", Kind: BotSearch},
+	{Token: "PerplexityBot", Operator: "Perplexity", Kind: BotSearch},
+	{Token: "ChatGPT-User", Operator: "OpenAI", Kind: BotUser},
+	{Token: "Claude-User", Operator: "Anthropic", Kind: BotUser},
+	{Token: "GPTBot", Operator: "OpenAI", Kind: BotTraining},
+	{Token: "ClaudeBot", Operator: "Anthropic", Kind: BotTraining},
+	{Token: "Google-Extended", Operator: "Google", Kind: BotTraining},
+	{Token: "Applebot-Extended", Operator: "Apple", Kind: BotTraining},
+	{Token: "CCBot", Operator: "Common Crawl", Kind: BotTraining},
+	{Token: "Meta-ExternalAgent", Operator: "Meta", Kind: BotTraining},
+}
+
+// BotField is the robots form's switch field of a bot: bot_ and its token
+// lowercased with - as _ (bot_oai_searchbot).
+func BotField(token string) string {
+	return "bot_" + strings.ReplaceAll(strings.ToLower(token), "-", "_")
+}
+
+// BotByToken returns the listed bot whose token equals token (letter case
+// ignored).
+func BotByToken(token string) (Bot, bool) {
+	for _, b := range Bots {
+		if strings.EqualFold(b.Token, token) {
+			return b, true
+		}
+	}
+	return Bot{}, false
+}