feat: robots.txt editor and tester screen
- golem15.seo.robots singleton (UI-SPEC C10): staging callout and the served file (P4); the plugin's own extension adds the editor, the tester and the AI bots tab, saved to the robots row with history - field type robotstxt (E4, live lint, size meter; Decode refuses over 500 KB) and widget golem15-seo-robotstester (E5) on robots_test, which tests the unsaved text and switches through grobotstxt - saving rules that block a noindex address answers robots.blocks_noindex
This commit is contained in:
252
extensions_robots.go
Normal file
252
extensions_robots.go
Normal file
@@ -0,0 +1,252 @@
|
||||
package seo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/controllers"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/cabana"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"github.com/goccy/go-yaml"
|
||||
)
|
||||
|
||||
// The robots.txt form's tab keys (UI-SPEC C10).
|
||||
const (
|
||||
TabRobotsFile = "golem15.seo::lang.robots.tab_file"
|
||||
TabRobotsBots = "golem15.seo::lang.robots.tab_bots"
|
||||
)
|
||||
|
||||
// RobotsTestAction is the tester's widget action (E5).
|
||||
const RobotsTestAction = "robots_test"
|
||||
|
||||
// botComments are the switch comments per bot kind.
|
||||
var botComments = map[string]string{
|
||||
classes.BotSearch: "golem15.seo::lang.bots.search_comment",
|
||||
classes.BotUser: "golem15.seo::lang.bots.user_comment",
|
||||
classes.BotTraining: "golem15.seo::lang.bots.training_comment",
|
||||
}
|
||||
|
||||
// robotsDocument builds the robots form's fields document: the editor
|
||||
// (E4) and the tester (E5, filled from the editor and every bot switch)
|
||||
// after the staging callout on the robots.txt tab, then the AI bots tab
|
||||
// with one switch per bot in classes.Bots order, labelled with its token
|
||||
// and operator.
|
||||
func robotsDocument() ([]byte, error) {
|
||||
type tabSpec struct {
|
||||
Name string `yaml:"name"`
|
||||
After string `yaml:"after"`
|
||||
}
|
||||
fill := []string{"content"}
|
||||
for _, b := range classes.Bots {
|
||||
fill = append(fill, classes.BotField(b.Token))
|
||||
}
|
||||
fields := yaml.MapSlice{
|
||||
{Key: "content", Value: yaml.MapSlice{
|
||||
{Key: "label", Value: "golem15.seo::lang.robots.content"},
|
||||
{Key: "comment", Value: "golem15.seo::lang.robots.content_comment"},
|
||||
{Key: "type", Value: "robotstxt"},
|
||||
{Key: "span", Value: "full"},
|
||||
{Key: "after", Value: "staging"},
|
||||
{Key: "tab", Value: TabRobotsFile},
|
||||
}},
|
||||
{Key: "tester", Value: yaml.MapSlice{
|
||||
{Key: "label", Value: "golem15.seo::lang.robots.tester"},
|
||||
{Key: "type", Value: "widget"},
|
||||
{Key: "widget", Value: classes.ElementRobotsTester},
|
||||
{Key: "action", Value: RobotsTestAction},
|
||||
{Key: "fill", Value: fill},
|
||||
{Key: "span", Value: "full"},
|
||||
{Key: "after", Value: "content"},
|
||||
{Key: "tab", Value: TabRobotsFile},
|
||||
}},
|
||||
}
|
||||
for i, b := range classes.Bots {
|
||||
span := "left"
|
||||
if i%2 == 1 {
|
||||
span = "right"
|
||||
}
|
||||
fields = append(fields, yaml.MapItem{Key: classes.BotField(b.Token), Value: yaml.MapSlice{
|
||||
{Key: "label", Value: b.Token + " (" + b.Operator + ")"},
|
||||
{Key: "comment", Value: botComments[b.Kind]},
|
||||
{Key: "type", Value: "switch"},
|
||||
{Key: "default", Value: true},
|
||||
{Key: "span", Value: span},
|
||||
{Key: "tab", Value: TabRobotsBots},
|
||||
}})
|
||||
}
|
||||
return yaml.Marshal(yaml.MapSlice{
|
||||
{Key: "tabs", Value: []tabSpec{{Name: TabRobotsBots, After: TabRobotsFile}}},
|
||||
{Key: "fields", Value: fields},
|
||||
})
|
||||
}
|
||||
|
||||
// robotsForm is the plugin's own extension on the robots.txt screen
|
||||
// (D-22, D-24, D-25): the editor, the tester and the AI bot switches,
|
||||
// loaded from and saved to the robots row inside the save transaction.
|
||||
// Save records history and answers robots.blocks_noindex on content when
|
||||
// the rules block a noindex address from Googlebot.
|
||||
func (p *Plugin) robotsForm() *pact.AdminFormExtension {
|
||||
doc, err := robotsDocument()
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("golem15.seo: robots form: %w", err))
|
||||
}
|
||||
return &pact.AdminFormExtension{
|
||||
FieldsYAML: doc,
|
||||
Permissions: []string{controllers.PermissionManage},
|
||||
Actions: []pact.AdminAction{p.robotsTestAction()},
|
||||
Load: func(_ context.Context, in pact.AdminFormLoad) (map[string]any, error) {
|
||||
row, ok := in.Record.(*models.Robots)
|
||||
if !ok || row == nil {
|
||||
return nil, nil
|
||||
}
|
||||
out := map[string]any{"content": row.Content}
|
||||
for _, b := range classes.Bots {
|
||||
out[classes.BotField(b.Token)] = row.Bots.Allowed(b.Token)
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Save: func(ctx context.Context, in pact.AdminFormSave) (pact.AdminFormSaveResult, error) {
|
||||
svc := p.Service()
|
||||
if svc == nil {
|
||||
return pact.AdminFormSaveResult{}, classes.ErrNotBooted
|
||||
}
|
||||
row, ok := in.Record.(*models.Robots)
|
||||
if !ok || row == nil {
|
||||
return pact.AdminFormSaveResult{}, nil
|
||||
}
|
||||
before := robotsValues(*row)
|
||||
if raw, ok := in.Values["content"]; ok {
|
||||
if text, ok := raw.(string); ok {
|
||||
row.Content = text
|
||||
}
|
||||
}
|
||||
bots := models.BotSwitches{}
|
||||
for token, on := range row.Bots {
|
||||
bots[token] = on
|
||||
}
|
||||
for _, b := range classes.Bots {
|
||||
if raw, ok := in.Values[classes.BotField(b.Token)]; ok {
|
||||
bots[b.Token] = truthy(raw)
|
||||
}
|
||||
}
|
||||
row.Bots = bots
|
||||
now := time.Now().UTC()
|
||||
row.UpdatedAt = &now
|
||||
if err := in.DB.Model(&models.Robots{}).Where("id = ?", row.ID).Updates(map[string]any{
|
||||
"content": row.Content, "bots": row.Bots, "updated_at": now,
|
||||
}).Error; err != nil {
|
||||
return pact.AdminFormSaveResult{}, err
|
||||
}
|
||||
if err := svc.Record(ctx, in.DB, classes.Change{
|
||||
Kind: models.SubjectRobots,
|
||||
SubjectID: strconv.FormatUint(uint64(row.ID), 10),
|
||||
What: svc.Phrase("golem15.seo::lang.history.what_robots", nil),
|
||||
Before: before,
|
||||
After: robotsValues(*row),
|
||||
}); err != nil {
|
||||
return pact.AdminFormSaveResult{}, err
|
||||
}
|
||||
blocked, err := svc.BlockedNoindex(in.DB, row.Content, row.Bots)
|
||||
if err != nil || len(blocked) == 0 {
|
||||
return pact.AdminFormSaveResult{}, err
|
||||
}
|
||||
list := blocked
|
||||
if len(list) > 10 {
|
||||
list = append(append([]string{}, list[:10]...), "…")
|
||||
}
|
||||
return pact.AdminFormSaveResult{Notices: map[string][]pact.FormFieldNotice{"content": {{
|
||||
Message: "golem15.seo::lang.robots.blocks_noindex",
|
||||
Params: map[string]string{"list": strings.Join(list, ", ")},
|
||||
Count: len(blocked),
|
||||
}}}}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// robotsValues are the robots row's values for history.
|
||||
func robotsValues(r models.Robots) map[string]any {
|
||||
bots := map[string]any{}
|
||||
for _, b := range classes.Bots {
|
||||
bots[b.Token] = r.Bots.Allowed(b.Token)
|
||||
}
|
||||
return map[string]any{"content": r.Content, "bots": bots}
|
||||
}
|
||||
|
||||
// robotsTestPayload is the tester's request: op context answers the
|
||||
// crawler list and the environment; op test answers whether agent may
|
||||
// fetch url under the editor's unsaved text and switches.
|
||||
type robotsTestPayload struct {
|
||||
Op string `json:"op"`
|
||||
URL string `json:"url"`
|
||||
Agent string `json:"agent"`
|
||||
}
|
||||
|
||||
// robotsTesterContext is the tester's context answer.
|
||||
type robotsTesterContext struct {
|
||||
Op string `json:"op"`
|
||||
Agents []string `json:"agents"`
|
||||
Bots []string `json:"bots"`
|
||||
Staging bool `json:"staging"`
|
||||
}
|
||||
|
||||
// robotsTestResult is the tester's test answer.
|
||||
type robotsTestResult struct {
|
||||
Op string `json:"op"`
|
||||
URL string `json:"url"`
|
||||
classes.RobotsTest
|
||||
}
|
||||
|
||||
// testerAgents are the crawlers E5 offers before the AI bots.
|
||||
var testerAgents = []string{"Googlebot", "Googlebot-Image", "Bingbot"}
|
||||
|
||||
// robotsTestAction is the tester's widget action (E5, D-22, D-99).
|
||||
func (p *Plugin) robotsTestAction() pact.AdminAction {
|
||||
return pact.AdminAction{
|
||||
Name: RobotsTestAction,
|
||||
Label: "golem15.seo::lang.robots.test",
|
||||
Permissions: []string{controllers.PermissionManage},
|
||||
Run: func(ctx context.Context, in pact.AdminActionInput) (pact.AdminActionResult, error) {
|
||||
svc := p.Service()
|
||||
if svc == nil {
|
||||
return pact.AdminActionResult{}, classes.ErrNotBooted
|
||||
}
|
||||
var payload robotsTestPayload
|
||||
if len(in.Payload) > 0 {
|
||||
if err := json.Unmarshal(in.Payload, &payload); err != nil {
|
||||
return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.test_invalid"}}}
|
||||
}
|
||||
}
|
||||
if payload.Op == "context" || payload.Op == "" {
|
||||
out := robotsTesterContext{Op: "context", Agents: append([]string{}, testerAgents...), Staging: svc.Staging()}
|
||||
for _, b := range classes.Bots {
|
||||
out.Agents = append(out.Agents, b.Token)
|
||||
out.Bots = append(out.Bots, b.Token)
|
||||
}
|
||||
out.Agents = append(out.Agents, "*")
|
||||
return pact.AdminActionResult{Data: out}, nil
|
||||
}
|
||||
path := strings.TrimSpace(payload.URL)
|
||||
if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") || len(path) > classes.MaxURLBytes {
|
||||
return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.url_required"}}}
|
||||
}
|
||||
agent := strings.TrimSpace(payload.Agent)
|
||||
if agent == "" {
|
||||
agent = "Googlebot"
|
||||
}
|
||||
content, _ := in.Values["content"].(string)
|
||||
bots := models.BotSwitches{}
|
||||
for _, b := range classes.Bots {
|
||||
if raw, ok := in.Values[classes.BotField(b.Token)]; ok {
|
||||
bots[b.Token] = truthy(raw)
|
||||
}
|
||||
}
|
||||
return pact.AdminActionResult{Data: robotsTestResult{Op: "test", URL: path, RobotsTest: svc.TestRobots(content, bots, agent, path)}}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user