feat: robots.txt served file, AI bots and the non-production lock

- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the
  framework repo)
- golem15_seo_robots singleton: rules and AI bot switches; eleven bots
  incl. Claude-User, all allowed by default
- GET /robots.txt: rules, a Disallow group per switched-off bot and the
  golem15.seo.robots_sitemaps lines; outside production the lock with
  X-Robots-Tag; never a 5xx (last good body, else allow-all)
- TestRobots on Google's matcher with the deciding line, and the
  noindex-blocked check (D-24)
This commit is contained in:
2026-10-27 11:45:44 +01:00
parent 4cb0fbfebb
commit 3dd936202f
11 changed files with 643 additions and 5 deletions

View File

@@ -5,6 +5,7 @@ package classes
import (
"context"
"errors"
"sync/atomic"
"git.golem15.com/golem15/summercms/modules/backpack"
"git.golem15.com/golem15/summercms/modules/cabana"
@@ -22,6 +23,9 @@ var ErrNotBooted = errors.New("golem15.seo: app is not booted")
// history.
type Service struct {
app *backpack.App
// robots is the last production robots.txt served, answered when the
// store fails (D-22: never a 5xx).
robots atomic.Pointer[string]
}
// NewService returns the service of a booted application. The database