Jakub Zych
3dd936202f
- github.com/jimsmart/grobotstxt v1.0.3 (D-99, decision note in the framework repo) - golem15_seo_robots singleton: rules and AI bot switches; eleven bots incl. Claude-User, all allowed by default - GET /robots.txt: rules, a Disallow group per switched-off bot and the golem15.seo.robots_sitemaps lines; outside production the lock with X-Robots-Tag; never a 5xx (last good body, else allow-all) - TestRobots on Google's matcher with the deciding line, and the noindex-blocked check (D-24)
83 lines
2.2 KiB
Go
83 lines
2.2 KiB
Go
package models
|
|
|
|
import (
|
|
"database/sql/driver"
|
|
"encoding/json"
|
|
"errors"
|
|
"time"
|
|
)
|
|
|
|
// Robots is the robots.txt singleton (row id 1) on golem15_seo_robots
|
|
// (D-22, D-25): the editor's rules and the AI bot switches. The served
|
|
// file adds a Disallow group for every switched-off bot and the Sitemap
|
|
// lines; outside production it is replaced by the non-production lock.
|
|
type Robots struct {
|
|
ID uint `gorm:"column:id;primaryKey"`
|
|
Content string `gorm:"column:content"`
|
|
Bots BotSwitches `gorm:"column:bots"`
|
|
CreatedAt *time.Time `gorm:"column:created_at"`
|
|
UpdatedAt *time.Time `gorm:"column:updated_at"`
|
|
}
|
|
|
|
func (Robots) TableName() string { return "golem15_seo_robots" }
|
|
|
|
// MorphName names the robots row in history.
|
|
func (Robots) MorphName() string { return `Golem15\SEO\Models\Robots` }
|
|
|
|
// Fillable lists the form's columns: the rules. The bot switches are
|
|
// written by the robots form's extension.
|
|
func (Robots) Fillable() []string { return []string{"content"} }
|
|
|
|
// Rules is empty: the robotstxt field type refuses an oversized file.
|
|
func (Robots) Rules() map[string]string { return map[string]string{} }
|
|
|
|
// BotSwitches maps a bot's user-agent token to whether it is allowed. A
|
|
// bot that is not listed is allowed (D-25: every bot on by default).
|
|
type BotSwitches map[string]bool
|
|
|
|
// Allowed reports whether token is switched on.
|
|
func (b BotSwitches) Allowed(token string) bool {
|
|
on, ok := b[token]
|
|
return !ok || on
|
|
}
|
|
|
|
// Value encodes the switches as a JSON object; nil is {}.
|
|
func (b BotSwitches) Value() (driver.Value, error) {
|
|
if b == nil {
|
|
return "{}", nil
|
|
}
|
|
raw, err := json.Marshal(map[string]bool(b))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return string(raw), nil
|
|
}
|
|
|
|
// Scan decodes a jsonb object; NULL reads as none.
|
|
func (b *BotSwitches) Scan(src any) error {
|
|
var raw []byte
|
|
switch v := src.(type) {
|
|
case nil:
|
|
*b = BotSwitches{}
|
|
return nil
|
|
case []byte:
|
|
raw = v
|
|
case string:
|
|
raw = []byte(v)
|
|
default:
|
|
return errors.New("models: unsupported bots value")
|
|
}
|
|
out := map[string]bool{}
|
|
if len(raw) > 0 {
|
|
if err := json.Unmarshal(raw, &out); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
*b = out
|
|
return nil
|
|
}
|
|
|
|
func init() {
|
|
Register(Robots{})
|
|
}
|