package seo import ( "context" "encoding/json" "fmt" "strconv" "strings" "time" "git.golem15.com/golem15/sm-seo-plugin/classes" "git.golem15.com/golem15/sm-seo-plugin/controllers" "git.golem15.com/golem15/sm-seo-plugin/models" "git.golem15.com/golem15/summercms/modules/cabana" "git.golem15.com/golem15/summercms/modules/pact" "github.com/goccy/go-yaml" ) // The robots.txt form's tab keys (UI-SPEC C10). const ( TabRobotsFile = "golem15.seo::lang.robots.tab_file" TabRobotsBots = "golem15.seo::lang.robots.tab_bots" ) // RobotsTestAction is the tester's widget action (E5). const RobotsTestAction = "robots_test" // botComments are the switch comments per bot kind. var botComments = map[string]string{ classes.BotSearch: "golem15.seo::lang.bots.search_comment", classes.BotUser: "golem15.seo::lang.bots.user_comment", classes.BotTraining: "golem15.seo::lang.bots.training_comment", } // robotsDocument builds the robots form's fields document: the editor // (E4) and the tester (E5, filled from the editor and every bot switch) // after the staging callout on the robots.txt tab, then the AI bots tab // with one switch per bot in classes.Bots order, labelled with its token // and operator. func robotsDocument() ([]byte, error) { type tabSpec struct { Name string `yaml:"name"` After string `yaml:"after"` } fill := []string{"content"} for _, b := range classes.Bots { fill = append(fill, classes.BotField(b.Token)) } fields := yaml.MapSlice{ {Key: "content", Value: yaml.MapSlice{ {Key: "label", Value: "golem15.seo::lang.robots.content"}, {Key: "comment", Value: "golem15.seo::lang.robots.content_comment"}, {Key: "type", Value: "robotstxt"}, {Key: "span", Value: "full"}, {Key: "after", Value: "staging"}, {Key: "tab", Value: TabRobotsFile}, }}, {Key: "tester", Value: yaml.MapSlice{ {Key: "label", Value: "golem15.seo::lang.robots.tester"}, {Key: "type", Value: "widget"}, {Key: "widget", Value: classes.ElementRobotsTester}, {Key: "action", Value: RobotsTestAction}, {Key: "fill", Value: fill}, {Key: "span", Value: "full"}, {Key: "after", Value: "content"}, {Key: "tab", Value: TabRobotsFile}, }}, } for i, b := range classes.Bots { span := "left" if i%2 == 1 { span = "right" } fields = append(fields, yaml.MapItem{Key: classes.BotField(b.Token), Value: yaml.MapSlice{ {Key: "label", Value: b.Token + " (" + b.Operator + ")"}, {Key: "comment", Value: botComments[b.Kind]}, {Key: "type", Value: "switch"}, {Key: "default", Value: true}, {Key: "span", Value: span}, {Key: "tab", Value: TabRobotsBots}, }}) } return yaml.Marshal(yaml.MapSlice{ {Key: "tabs", Value: []tabSpec{{Name: TabRobotsBots, After: TabRobotsFile}}}, {Key: "fields", Value: fields}, }) } // robotsForm is the plugin's own extension on the robots.txt screen // (D-22, D-24, D-25): the editor, the tester and the AI bot switches, // loaded from and saved to the robots row inside the save transaction. // Save records history and answers robots.blocks_noindex on content when // the rules block a noindex address from Googlebot. func (p *Plugin) robotsForm() *pact.AdminFormExtension { doc, err := robotsDocument() if err != nil { panic(fmt.Errorf("golem15.seo: robots form: %w", err)) } return &pact.AdminFormExtension{ FieldsYAML: doc, Permissions: []string{controllers.PermissionManage}, Actions: []pact.AdminAction{p.robotsTestAction()}, Load: func(_ context.Context, in pact.AdminFormLoad) (map[string]any, error) { row, ok := in.Record.(*models.Robots) if !ok || row == nil { return nil, nil } out := map[string]any{"content": row.Content} for _, b := range classes.Bots { out[classes.BotField(b.Token)] = row.Bots.Allowed(b.Token) } return out, nil }, Save: func(ctx context.Context, in pact.AdminFormSave) (pact.AdminFormSaveResult, error) { svc := p.Service() if svc == nil { return pact.AdminFormSaveResult{}, classes.ErrNotBooted } row, ok := in.Record.(*models.Robots) if !ok || row == nil { return pact.AdminFormSaveResult{}, nil } before := robotsValues(*row) if raw, ok := in.Values["content"]; ok { if text, ok := raw.(string); ok { row.Content = text } } bots := models.BotSwitches{} for token, on := range row.Bots { bots[token] = on } for _, b := range classes.Bots { if raw, ok := in.Values[classes.BotField(b.Token)]; ok { bots[b.Token] = truthy(raw) } } row.Bots = bots now := time.Now().UTC() row.UpdatedAt = &now if err := in.DB.Model(&models.Robots{}).Where("id = ?", row.ID).Updates(map[string]any{ "content": row.Content, "bots": row.Bots, "updated_at": now, }).Error; err != nil { return pact.AdminFormSaveResult{}, err } if err := svc.Record(ctx, in.DB, classes.Change{ Kind: models.SubjectRobots, SubjectID: strconv.FormatUint(uint64(row.ID), 10), What: svc.Phrase("golem15.seo::lang.history.what_robots", nil), Before: before, After: robotsValues(*row), }); err != nil { return pact.AdminFormSaveResult{}, err } blocked, err := svc.BlockedNoindex(in.DB, row.Content, row.Bots) if err != nil || len(blocked) == 0 { return pact.AdminFormSaveResult{}, err } list := blocked if len(list) > 10 { list = append(append([]string{}, list[:10]...), "…") } return pact.AdminFormSaveResult{Notices: map[string][]pact.FormFieldNotice{"content": {{ Message: "golem15.seo::lang.robots.blocks_noindex", Params: map[string]string{"list": strings.Join(list, ", ")}, Count: len(blocked), }}}}, nil }, } } // robotsValues are the robots row's values for history. func robotsValues(r models.Robots) map[string]any { bots := map[string]any{} for _, b := range classes.Bots { bots[b.Token] = r.Bots.Allowed(b.Token) } return map[string]any{"content": r.Content, "bots": bots} } // robotsTestPayload is the tester's request: op context answers the // crawler list and the environment; op test answers whether agent may // fetch url under the editor's unsaved text and switches. type robotsTestPayload struct { Op string `json:"op"` URL string `json:"url"` Agent string `json:"agent"` } // robotsTesterContext is the tester's context answer. type robotsTesterContext struct { Op string `json:"op"` Agents []string `json:"agents"` Bots []string `json:"bots"` Staging bool `json:"staging"` } // robotsTestResult is the tester's test answer. type robotsTestResult struct { Op string `json:"op"` URL string `json:"url"` classes.RobotsTest } // testerAgents are the crawlers E5 offers before the AI bots. var testerAgents = []string{"Googlebot", "Googlebot-Image", "Bingbot"} // robotsTestAction is the tester's widget action (E5, D-22, D-99). func (p *Plugin) robotsTestAction() pact.AdminAction { return pact.AdminAction{ Name: RobotsTestAction, Label: "golem15.seo::lang.robots.test", Permissions: []string{controllers.PermissionManage}, Run: func(ctx context.Context, in pact.AdminActionInput) (pact.AdminActionResult, error) { svc := p.Service() if svc == nil { return pact.AdminActionResult{}, classes.ErrNotBooted } var payload robotsTestPayload if len(in.Payload) > 0 { if err := json.Unmarshal(in.Payload, &payload); err != nil { return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.test_invalid"}}} } } if payload.Op == "context" || payload.Op == "" { out := robotsTesterContext{Op: "context", Agents: append([]string{}, testerAgents...), Staging: svc.Staging()} for _, b := range classes.Bots { out.Agents = append(out.Agents, b.Token) out.Bots = append(out.Bots, b.Token) } out.Agents = append(out.Agents, "*") return pact.AdminActionResult{Data: out}, nil } path := strings.TrimSpace(payload.URL) if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") || len(path) > classes.MaxURLBytes { return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.url_required"}}} } agent := strings.TrimSpace(payload.Agent) if agent == "" { agent = "Googlebot" } content, _ := in.Values["content"].(string) bots := models.BotSwitches{} for _, b := range classes.Bots { if raw, ok := in.Values[classes.BotField(b.Token)]; ok { bots[b.Token] = truthy(raw) } } return pact.AdminActionResult{Data: robotsTestResult{Op: "test", URL: path, RobotsTest: svc.TestRobots(content, bots, agent, path)}}, nil }, } }