feat: robots.txt served file, AI bots and the non-production lock
- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the framework repo) - golem15_seo_robots singleton: rules and AI bot switches; eleven bots incl. Claude-User, all allowed by default - GET /robots.txt: rules, a Disallow group per switched-off bot and the golem15.seo.robots_sitemaps lines; outside production the lock with X-Robots-Tag; never a 5xx (last good body, else allow-all) - TestRobots on Google's matcher with the deciding line, and the noindex-blocked check (D-24)
This commit is contained in:
@@ -5,6 +5,7 @@ package classes
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/cabana"
|
||||
@@ -22,6 +23,9 @@ var ErrNotBooted = errors.New("golem15.seo: app is not booted")
|
||||
// history.
|
||||
type Service struct {
|
||||
app *backpack.App
|
||||
// robots is the last production robots.txt served, answered when the
|
||||
// store fails (D-22: never a 5xx).
|
||||
robots atomic.Pointer[string]
|
||||
}
|
||||
|
||||
// NewService returns the service of a booted application. The database
|
||||
|
||||
Reference in New Issue
Block a user