Jakub Zych
51413fbda4
- golem15.seo.robots singleton (UI-SPEC C10): staging callout and the served file (P4); the plugin's own extension adds the editor, the tester and the AI bots tab, saved to the robots row with history - field type robotstxt (E4, live lint, size meter; Decode refuses over 500 KB) and widget golem15-seo-robotstester (E5) on robots_test, which tests the unsaved text and switches through grobotstxt - saving rules that block a noindex address answers robots.blocks_noindex
253 lines
8.5 KiB
Go
253 lines
8.5 KiB
Go
package seo
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
|
"git.golem15.com/golem15/sm-seo-plugin/controllers"
|
|
"git.golem15.com/golem15/sm-seo-plugin/models"
|
|
"git.golem15.com/golem15/summercms/modules/cabana"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"github.com/goccy/go-yaml"
|
|
)
|
|
|
|
// The robots.txt form's tab keys (UI-SPEC C10).
|
|
const (
|
|
TabRobotsFile = "golem15.seo::lang.robots.tab_file"
|
|
TabRobotsBots = "golem15.seo::lang.robots.tab_bots"
|
|
)
|
|
|
|
// RobotsTestAction is the tester's widget action (E5).
|
|
const RobotsTestAction = "robots_test"
|
|
|
|
// botComments are the switch comments per bot kind.
|
|
var botComments = map[string]string{
|
|
classes.BotSearch: "golem15.seo::lang.bots.search_comment",
|
|
classes.BotUser: "golem15.seo::lang.bots.user_comment",
|
|
classes.BotTraining: "golem15.seo::lang.bots.training_comment",
|
|
}
|
|
|
|
// robotsDocument builds the robots form's fields document: the editor
|
|
// (E4) and the tester (E5, filled from the editor and every bot switch)
|
|
// after the staging callout on the robots.txt tab, then the AI bots tab
|
|
// with one switch per bot in classes.Bots order, labelled with its token
|
|
// and operator.
|
|
func robotsDocument() ([]byte, error) {
|
|
type tabSpec struct {
|
|
Name string `yaml:"name"`
|
|
After string `yaml:"after"`
|
|
}
|
|
fill := []string{"content"}
|
|
for _, b := range classes.Bots {
|
|
fill = append(fill, classes.BotField(b.Token))
|
|
}
|
|
fields := yaml.MapSlice{
|
|
{Key: "content", Value: yaml.MapSlice{
|
|
{Key: "label", Value: "golem15.seo::lang.robots.content"},
|
|
{Key: "comment", Value: "golem15.seo::lang.robots.content_comment"},
|
|
{Key: "type", Value: "robotstxt"},
|
|
{Key: "span", Value: "full"},
|
|
{Key: "after", Value: "staging"},
|
|
{Key: "tab", Value: TabRobotsFile},
|
|
}},
|
|
{Key: "tester", Value: yaml.MapSlice{
|
|
{Key: "label", Value: "golem15.seo::lang.robots.tester"},
|
|
{Key: "type", Value: "widget"},
|
|
{Key: "widget", Value: classes.ElementRobotsTester},
|
|
{Key: "action", Value: RobotsTestAction},
|
|
{Key: "fill", Value: fill},
|
|
{Key: "span", Value: "full"},
|
|
{Key: "after", Value: "content"},
|
|
{Key: "tab", Value: TabRobotsFile},
|
|
}},
|
|
}
|
|
for i, b := range classes.Bots {
|
|
span := "left"
|
|
if i%2 == 1 {
|
|
span = "right"
|
|
}
|
|
fields = append(fields, yaml.MapItem{Key: classes.BotField(b.Token), Value: yaml.MapSlice{
|
|
{Key: "label", Value: b.Token + " (" + b.Operator + ")"},
|
|
{Key: "comment", Value: botComments[b.Kind]},
|
|
{Key: "type", Value: "switch"},
|
|
{Key: "default", Value: true},
|
|
{Key: "span", Value: span},
|
|
{Key: "tab", Value: TabRobotsBots},
|
|
}})
|
|
}
|
|
return yaml.Marshal(yaml.MapSlice{
|
|
{Key: "tabs", Value: []tabSpec{{Name: TabRobotsBots, After: TabRobotsFile}}},
|
|
{Key: "fields", Value: fields},
|
|
})
|
|
}
|
|
|
|
// robotsForm is the plugin's own extension on the robots.txt screen
|
|
// (D-22, D-24, D-25): the editor, the tester and the AI bot switches,
|
|
// loaded from and saved to the robots row inside the save transaction.
|
|
// Save records history and answers robots.blocks_noindex on content when
|
|
// the rules block a noindex address from Googlebot.
|
|
func (p *Plugin) robotsForm() *pact.AdminFormExtension {
|
|
doc, err := robotsDocument()
|
|
if err != nil {
|
|
panic(fmt.Errorf("golem15.seo: robots form: %w", err))
|
|
}
|
|
return &pact.AdminFormExtension{
|
|
FieldsYAML: doc,
|
|
Permissions: []string{controllers.PermissionManage},
|
|
Actions: []pact.AdminAction{p.robotsTestAction()},
|
|
Load: func(_ context.Context, in pact.AdminFormLoad) (map[string]any, error) {
|
|
row, ok := in.Record.(*models.Robots)
|
|
if !ok || row == nil {
|
|
return nil, nil
|
|
}
|
|
out := map[string]any{"content": row.Content}
|
|
for _, b := range classes.Bots {
|
|
out[classes.BotField(b.Token)] = row.Bots.Allowed(b.Token)
|
|
}
|
|
return out, nil
|
|
},
|
|
Save: func(ctx context.Context, in pact.AdminFormSave) (pact.AdminFormSaveResult, error) {
|
|
svc := p.Service()
|
|
if svc == nil {
|
|
return pact.AdminFormSaveResult{}, classes.ErrNotBooted
|
|
}
|
|
row, ok := in.Record.(*models.Robots)
|
|
if !ok || row == nil {
|
|
return pact.AdminFormSaveResult{}, nil
|
|
}
|
|
before := robotsValues(*row)
|
|
if raw, ok := in.Values["content"]; ok {
|
|
if text, ok := raw.(string); ok {
|
|
row.Content = text
|
|
}
|
|
}
|
|
bots := models.BotSwitches{}
|
|
for token, on := range row.Bots {
|
|
bots[token] = on
|
|
}
|
|
for _, b := range classes.Bots {
|
|
if raw, ok := in.Values[classes.BotField(b.Token)]; ok {
|
|
bots[b.Token] = truthy(raw)
|
|
}
|
|
}
|
|
row.Bots = bots
|
|
now := time.Now().UTC()
|
|
row.UpdatedAt = &now
|
|
if err := in.DB.Model(&models.Robots{}).Where("id = ?", row.ID).Updates(map[string]any{
|
|
"content": row.Content, "bots": row.Bots, "updated_at": now,
|
|
}).Error; err != nil {
|
|
return pact.AdminFormSaveResult{}, err
|
|
}
|
|
if err := svc.Record(ctx, in.DB, classes.Change{
|
|
Kind: models.SubjectRobots,
|
|
SubjectID: strconv.FormatUint(uint64(row.ID), 10),
|
|
What: svc.Phrase("golem15.seo::lang.history.what_robots", nil),
|
|
Before: before,
|
|
After: robotsValues(*row),
|
|
}); err != nil {
|
|
return pact.AdminFormSaveResult{}, err
|
|
}
|
|
blocked, err := svc.BlockedNoindex(in.DB, row.Content, row.Bots)
|
|
if err != nil || len(blocked) == 0 {
|
|
return pact.AdminFormSaveResult{}, err
|
|
}
|
|
list := blocked
|
|
if len(list) > 10 {
|
|
list = append(append([]string{}, list[:10]...), "…")
|
|
}
|
|
return pact.AdminFormSaveResult{Notices: map[string][]pact.FormFieldNotice{"content": {{
|
|
Message: "golem15.seo::lang.robots.blocks_noindex",
|
|
Params: map[string]string{"list": strings.Join(list, ", ")},
|
|
Count: len(blocked),
|
|
}}}}, nil
|
|
},
|
|
}
|
|
}
|
|
|
|
// robotsValues are the robots row's values for history.
|
|
func robotsValues(r models.Robots) map[string]any {
|
|
bots := map[string]any{}
|
|
for _, b := range classes.Bots {
|
|
bots[b.Token] = r.Bots.Allowed(b.Token)
|
|
}
|
|
return map[string]any{"content": r.Content, "bots": bots}
|
|
}
|
|
|
|
// robotsTestPayload is the tester's request: op context answers the
|
|
// crawler list and the environment; op test answers whether agent may
|
|
// fetch url under the editor's unsaved text and switches.
|
|
type robotsTestPayload struct {
|
|
Op string `json:"op"`
|
|
URL string `json:"url"`
|
|
Agent string `json:"agent"`
|
|
}
|
|
|
|
// robotsTesterContext is the tester's context answer.
|
|
type robotsTesterContext struct {
|
|
Op string `json:"op"`
|
|
Agents []string `json:"agents"`
|
|
Bots []string `json:"bots"`
|
|
Staging bool `json:"staging"`
|
|
}
|
|
|
|
// robotsTestResult is the tester's test answer.
|
|
type robotsTestResult struct {
|
|
Op string `json:"op"`
|
|
URL string `json:"url"`
|
|
classes.RobotsTest
|
|
}
|
|
|
|
// testerAgents are the crawlers E5 offers before the AI bots.
|
|
var testerAgents = []string{"Googlebot", "Googlebot-Image", "Bingbot"}
|
|
|
|
// robotsTestAction is the tester's widget action (E5, D-22, D-99).
|
|
func (p *Plugin) robotsTestAction() pact.AdminAction {
|
|
return pact.AdminAction{
|
|
Name: RobotsTestAction,
|
|
Label: "golem15.seo::lang.robots.test",
|
|
Permissions: []string{controllers.PermissionManage},
|
|
Run: func(ctx context.Context, in pact.AdminActionInput) (pact.AdminActionResult, error) {
|
|
svc := p.Service()
|
|
if svc == nil {
|
|
return pact.AdminActionResult{}, classes.ErrNotBooted
|
|
}
|
|
var payload robotsTestPayload
|
|
if len(in.Payload) > 0 {
|
|
if err := json.Unmarshal(in.Payload, &payload); err != nil {
|
|
return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.test_invalid"}}}
|
|
}
|
|
}
|
|
if payload.Op == "context" || payload.Op == "" {
|
|
out := robotsTesterContext{Op: "context", Agents: append([]string{}, testerAgents...), Staging: svc.Staging()}
|
|
for _, b := range classes.Bots {
|
|
out.Agents = append(out.Agents, b.Token)
|
|
out.Bots = append(out.Bots, b.Token)
|
|
}
|
|
out.Agents = append(out.Agents, "*")
|
|
return pact.AdminActionResult{Data: out}, nil
|
|
}
|
|
path := strings.TrimSpace(payload.URL)
|
|
if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") || len(path) > classes.MaxURLBytes {
|
|
return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.url_required"}}}
|
|
}
|
|
agent := strings.TrimSpace(payload.Agent)
|
|
if agent == "" {
|
|
agent = "Googlebot"
|
|
}
|
|
content, _ := in.Values["content"].(string)
|
|
bots := models.BotSwitches{}
|
|
for _, b := range classes.Bots {
|
|
if raw, ok := in.Values[classes.BotField(b.Token)]; ok {
|
|
bots[b.Token] = truthy(raw)
|
|
}
|
|
}
|
|
return pact.AdminActionResult{Data: robotsTestResult{Op: "test", URL: path, RobotsTest: svc.TestRobots(content, bots, agent, path)}}, nil
|
|
},
|
|
}
|
|
}
|