feat: robots.txt served file, AI bots and the non-production lock
- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the framework repo) - golem15_seo_robots singleton: rules and AI bot switches; eleven bots incl. Claude-User, all allowed by default - GET /robots.txt: rules, a Disallow group per switched-off bot and the golem15.seo.robots_sitemaps lines; outside production the lock with X-Robots-Tag; never a 5xx (last good body, else allow-all) - TestRobots on Google's matcher with the deciding line, and the noindex-blocked check (D-24)
This commit is contained in:
55
classes/bots.go
Normal file
55
classes/bots.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package classes
|
||||
|
||||
import "strings"
|
||||
|
||||
// The kinds of AI bots (D-25): search and answer crawlers, fetchers that
|
||||
// load a page when a user asks for it, and crawlers that collect content
|
||||
// for training.
|
||||
const (
|
||||
BotSearch = "search"
|
||||
BotUser = "user"
|
||||
BotTraining = "training"
|
||||
)
|
||||
|
||||
// Bot is one AI crawler the robots.txt screen can switch off: its
|
||||
// user-agent token, its operator and its kind.
|
||||
type Bot struct {
|
||||
Token string
|
||||
Operator string
|
||||
Kind string
|
||||
}
|
||||
|
||||
// Bots is the AI bot list in the screen's order (UI-SPEC C10, D-25,
|
||||
// D-105): search bots, user-fetch bots, then training bots. Every bot is
|
||||
// allowed until an editor switches it off. The list is plugin data and
|
||||
// can grow without a layout change (Q-6).
|
||||
var Bots = []Bot{
|
||||
{Token: "OAI-SearchBot", Operator: "OpenAI", Kind: BotSearch},
|
||||
{Token: "Claude-SearchBot", Operator: "Anthropic", Kind: BotSearch},
|
||||
{Token: "PerplexityBot", Operator: "Perplexity", Kind: BotSearch},
|
||||
{Token: "ChatGPT-User", Operator: "OpenAI", Kind: BotUser},
|
||||
{Token: "Claude-User", Operator: "Anthropic", Kind: BotUser},
|
||||
{Token: "GPTBot", Operator: "OpenAI", Kind: BotTraining},
|
||||
{Token: "ClaudeBot", Operator: "Anthropic", Kind: BotTraining},
|
||||
{Token: "Google-Extended", Operator: "Google", Kind: BotTraining},
|
||||
{Token: "Applebot-Extended", Operator: "Apple", Kind: BotTraining},
|
||||
{Token: "CCBot", Operator: "Common Crawl", Kind: BotTraining},
|
||||
{Token: "Meta-ExternalAgent", Operator: "Meta", Kind: BotTraining},
|
||||
}
|
||||
|
||||
// BotField is the robots form's switch field of a bot: bot_ and its token
|
||||
// lowercased with - as _ (bot_oai_searchbot).
|
||||
func BotField(token string) string {
|
||||
return "bot_" + strings.ReplaceAll(strings.ToLower(token), "-", "_")
|
||||
}
|
||||
|
||||
// BotByToken returns the listed bot whose token equals token (letter case
|
||||
// ignored).
|
||||
func BotByToken(token string) (Bot, bool) {
|
||||
for _, b := range Bots {
|
||||
if strings.EqualFold(b.Token, token) {
|
||||
return b, true
|
||||
}
|
||||
}
|
||||
return Bot{}, false
|
||||
}
|
||||
231
classes/robots.go
Normal file
231
classes/robots.go
Normal file
@@ -0,0 +1,231 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"github.com/jimsmart/grobotstxt"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// RobotsSitemapsConfigKey lists the sitemap addresses robots.txt names,
|
||||
// one Sitemap line each, absolute on the site URL (P-23); default
|
||||
// ["/sitemap.xml"].
|
||||
const RobotsSitemapsConfigKey = "golem15.seo.robots_sitemaps"
|
||||
|
||||
// MaxRobotsBytes is the largest robots.txt the editor stores: Google
|
||||
// ignores everything after 500 KiB.
|
||||
const MaxRobotsBytes = 500 << 10
|
||||
|
||||
// The two fixed robots.txt bodies.
|
||||
const (
|
||||
// NonProductionRobots is served outside production whatever is stored
|
||||
// (D-22).
|
||||
NonProductionRobots = "User-agent: *\nDisallow: /\n"
|
||||
// FallbackRobots is served when the stored file cannot be read and no
|
||||
// earlier body is cached: allow everything, never a 5xx.
|
||||
FallbackRobots = "User-agent: *\nDisallow:\n"
|
||||
)
|
||||
|
||||
// RobotsSitemaps returns the configured sitemap addresses, absolute on
|
||||
// the site URL.
|
||||
func (s *Service) RobotsSitemaps() []string {
|
||||
var list []string
|
||||
if cfg := s.Config(); cfg != nil {
|
||||
if raw, ok := cfg.Lookup(RobotsSitemapsConfigKey); ok {
|
||||
switch v := raw.(type) {
|
||||
case []any:
|
||||
for _, item := range v {
|
||||
if str, ok := item.(string); ok {
|
||||
list = append(list, str)
|
||||
}
|
||||
}
|
||||
case []string:
|
||||
list = append(list, v...)
|
||||
case string:
|
||||
list = strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' })
|
||||
}
|
||||
}
|
||||
}
|
||||
if list == nil {
|
||||
list = []string{"/sitemap.xml"}
|
||||
}
|
||||
out := make([]string, 0, len(list))
|
||||
for _, item := range list {
|
||||
if item = strings.TrimSpace(item); item != "" {
|
||||
out = append(out, AbsoluteURL(s.SiteURL(), item))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// LoadRobots reads robots row 1; without one the defaults apply (no
|
||||
// rules, every bot allowed).
|
||||
func LoadRobots(db *gorm.DB) (models.Robots, error) {
|
||||
var rows []models.Robots
|
||||
if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil {
|
||||
return models.Robots{}, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return models.Robots{ID: 1, Bots: models.BotSwitches{}}, nil
|
||||
}
|
||||
return rows[0], nil
|
||||
}
|
||||
|
||||
// ComposeRobots builds the production robots.txt (D-22, D-25, P-23): the
|
||||
// stored rules, one "User-agent: <token>" / "Disallow: /" group per
|
||||
// switched-off bot, then one Sitemap line per sitemap address.
|
||||
func ComposeRobots(content string, bots models.BotSwitches, sitemaps []string) string {
|
||||
var b strings.Builder
|
||||
content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t")
|
||||
if content != "" {
|
||||
b.WriteString(content)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
for _, bot := range Bots {
|
||||
if bots.Allowed(bot.Token) {
|
||||
continue
|
||||
}
|
||||
if b.Len() > 0 {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString("User-agent: " + bot.Token + "\nDisallow: /\n")
|
||||
}
|
||||
if len(sitemaps) > 0 && b.Len() > 0 {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
for _, sm := range sitemaps {
|
||||
b.WriteString("Sitemap: " + sm + "\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// generatedStart is the 1-based line where the generated part of a
|
||||
// composed file starts: the lines after the stored rules.
|
||||
func generatedStart(content string) int {
|
||||
content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t")
|
||||
if content == "" {
|
||||
return 1
|
||||
}
|
||||
return strings.Count(content, "\n") + 2
|
||||
}
|
||||
|
||||
// ServedFile is the robots.txt crawlers get (D-22): outside production
|
||||
// always NonProductionRobots, else the composed file from the stored rules
|
||||
// and switches. It caches the last body it served, which LastServedRobots
|
||||
// answers when the store fails.
|
||||
func (s *Service) ServedFile(ctx context.Context, production bool) (string, error) {
|
||||
if !production {
|
||||
return NonProductionRobots, nil
|
||||
}
|
||||
db, err := s.DB(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
row, err := LoadRobots(db)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
body := ComposeRobots(row.Content, row.Bots, s.RobotsSitemaps())
|
||||
if s != nil {
|
||||
s.robots.Store(&body)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// LastServedRobots returns the last production body ServedFile answered,
|
||||
// "" when there is none.
|
||||
func (s *Service) LastServedRobots() string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
if p := s.robots.Load(); p != nil {
|
||||
return *p
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// RobotsTest is the tester's answer (UI-SPEC E5): whether agent may fetch
|
||||
// the address, the deciding line (0 and "" when no rule matched) and
|
||||
// whether a bot switch decided it.
|
||||
type RobotsTest struct {
|
||||
Allowed bool `json:"allowed"`
|
||||
Line int `json:"line"`
|
||||
Rule string `json:"rule"`
|
||||
Bot bool `json:"bot"`
|
||||
Agent string `json:"agent"`
|
||||
Staging bool `json:"staging"`
|
||||
}
|
||||
|
||||
// TestRobots answers whether agent may fetch path under the production
|
||||
// file composed from content and bots (unsaved values included), through
|
||||
// grobotstxt, Google's matcher (D-22, D-99): longest match, Allow wins a
|
||||
// tie, * and $.
|
||||
func (s *Service) TestRobots(content string, bots models.BotSwitches, agent, path string) RobotsTest {
|
||||
file := ComposeRobots(content, bots, s.RobotsSitemaps())
|
||||
uri := AbsoluteURL(s.SiteURL(), path)
|
||||
if s.SiteURL() == "" {
|
||||
uri = "http://localhost" + path
|
||||
}
|
||||
m := grobotstxt.NewRobotsMatcher()
|
||||
out := RobotsTest{Agent: agent, Staging: s.Staging()}
|
||||
out.Allowed = m.AgentAllowed(file, agent, uri)
|
||||
if line := m.MatchingLine(); line > 0 {
|
||||
out.Line = line
|
||||
lines := strings.Split(file, "\n")
|
||||
if line <= len(lines) {
|
||||
out.Rule = strings.TrimSpace(lines[line-1])
|
||||
}
|
||||
out.Bot = !out.Allowed && line >= generatedStart(content)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// noindexAddresses lists the addresses the site serves noindex that a
|
||||
// robots.txt could block (D-24): exact URL rules and records whose own
|
||||
// robots value says noindex.
|
||||
func noindexAddresses(db *gorm.DB) ([]string, error) {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
var rules []models.Rule
|
||||
if err := db.Where("match = ? AND robots LIKE ?", models.MatchExact, "%noindex%").Find(&rules).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, r := range rules {
|
||||
if !seen[r.URL] {
|
||||
seen[r.URL] = true
|
||||
out = append(out, r.URL)
|
||||
}
|
||||
}
|
||||
var entries []models.Entry
|
||||
if err := db.Where("robots LIKE ? AND url <> ''", "%noindex%").Find(&entries).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !seen[e.URL] {
|
||||
seen[e.URL] = true
|
||||
out = append(out, e.URL)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// BlockedNoindex lists the noindex addresses Googlebot may not fetch under
|
||||
// the production file composed from content and bots (D-24): a crawler
|
||||
// that cannot fetch a page never sees its noindex.
|
||||
func (s *Service) BlockedNoindex(db *gorm.DB, content string, bots models.BotSwitches) ([]string, error) {
|
||||
urls, err := noindexAddresses(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, u := range urls {
|
||||
if !s.TestRobots(content, bots, "Googlebot", u).Allowed {
|
||||
out = append(out, u)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -5,6 +5,7 @@ package classes
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/cabana"
|
||||
@@ -22,6 +23,9 @@ var ErrNotBooted = errors.New("golem15.seo: app is not booted")
|
||||
// history.
|
||||
type Service struct {
|
||||
app *backpack.App
|
||||
// robots is the last production robots.txt served, answered when the
|
||||
// store fails (D-22: never a 5xx).
|
||||
robots atomic.Pointer[string]
|
||||
}
|
||||
|
||||
// NewService returns the service of a booted application. The database
|
||||
|
||||
Reference in New Issue
Block a user