package seo import ( "context" "encoding/json" "fmt" "strings" "git.golem15.com/golem15/sm-seo-plugin/classes" "git.golem15.com/golem15/sm-seo-plugin/controllers" "git.golem15.com/golem15/sm-seo-plugin/models" "git.golem15.com/golem15/summercms/modules/cabana" "git.golem15.com/golem15/summercms/modules/pact" "github.com/goccy/go-yaml" ) // The robots.txt form's tab keys (UI-SPEC C10). const ( TabRobotsFile = "golem15.seo::lang.robots.tab_file" TabRobotsBots = "golem15.seo::lang.robots.tab_bots" ) // RobotsTestAction is the tester's widget action (E5). const RobotsTestAction = "robots_test" // botComments are the switch comments per bot kind. var botComments = map[string]string{ classes.BotSearch: "golem15.seo::lang.bots.search_comment", classes.BotUser: "golem15.seo::lang.bots.user_comment", classes.BotTraining: "golem15.seo::lang.bots.training_comment", } // robotsDocument builds the robots form's fields document: the editor // (E4) and the tester (E5, filled from the editor and every bot switch) // after the staging callout on the robots.txt tab, then the AI bots tab // with one switch per bot in classes.Bots order, labelled with its token // and operator. func robotsDocument() ([]byte, error) { type tabSpec struct { Name string `yaml:"name"` After string `yaml:"after"` } fill := []string{"content"} for _, b := range classes.Bots { fill = append(fill, classes.BotField(b.Token)) } fields := yaml.MapSlice{ {Key: "content", Value: yaml.MapSlice{ {Key: "label", Value: "golem15.seo::lang.robots.content"}, {Key: "comment", Value: "golem15.seo::lang.robots.content_comment"}, {Key: "type", Value: "robotstxt"}, {Key: "span", Value: "full"}, {Key: "after", Value: "staging"}, {Key: "tab", Value: TabRobotsFile}, }}, {Key: "tester", Value: yaml.MapSlice{ {Key: "label", Value: "golem15.seo::lang.robots.tester"}, {Key: "type", Value: "widget"}, {Key: "widget", Value: classes.ElementRobotsTester}, {Key: "action", Value: RobotsTestAction}, {Key: "fill", Value: fill}, {Key: "span", Value: "full"}, {Key: "after", Value: "content"}, {Key: "tab", Value: TabRobotsFile}, }}, } for i, b := range classes.Bots { span := "left" if i%2 == 1 { span = "right" } fields = append(fields, yaml.MapItem{Key: classes.BotField(b.Token), Value: yaml.MapSlice{ {Key: "label", Value: b.Token + " (" + b.Operator + ")"}, {Key: "comment", Value: botComments[b.Kind]}, {Key: "type", Value: "switch"}, {Key: "default", Value: true}, {Key: "span", Value: span}, {Key: "tab", Value: TabRobotsBots}, }}) } // llms.txt (UI-SPEC C25) after the bot switches. fields = append(fields, yaml.MapItem{Key: "llms_enabled", Value: yaml.MapSlice{ {Key: "label", Value: "golem15.seo::lang.robots.llms_enabled"}, {Key: "comment", Value: "golem15.seo::lang.robots.llms_enabled_comment"}, {Key: "type", Value: "switch"}, {Key: "default", Value: false}, {Key: "span", Value: "full"}, {Key: "tab", Value: TabRobotsBots}, }}, yaml.MapItem{Key: "llms_intro", Value: yaml.MapSlice{ {Key: "label", Value: "golem15.seo::lang.robots.llms_intro"}, {Key: "comment", Value: "golem15.seo::lang.robots.llms_intro_comment"}, {Key: "type", Value: "textarea"}, {Key: "size", Value: "small"}, {Key: "span", Value: "full"}, {Key: "tab", Value: TabRobotsBots}, {Key: "trigger", Value: yaml.MapSlice{ {Key: "action", Value: "hide"}, {Key: "field", Value: "llms_enabled"}, {Key: "condition", Value: "unchecked"}, }}, }}, ) return yaml.Marshal(yaml.MapSlice{ {Key: "tabs", Value: []tabSpec{{Name: TabRobotsBots, After: TabRobotsFile}}}, {Key: "fields", Value: fields}, }) } // robotsForm is the plugin's own extension on the robots.txt screen // (D-22, D-24, D-25): the editor, the tester and the AI bot switches, // loaded from and saved to the robots row inside the save transaction. // Save records history and answers robots.blocks_noindex on content when // the rules block a noindex address from Googlebot. func (p *Plugin) robotsForm() *pact.AdminFormExtension { doc, err := robotsDocument() if err != nil { panic(fmt.Errorf("golem15.seo: robots form: %w", err)) } return &pact.AdminFormExtension{ FieldsYAML: doc, Permissions: []string{controllers.PermissionManage}, Actions: []pact.AdminAction{p.robotsTestAction()}, Load: func(_ context.Context, in pact.AdminFormLoad) (map[string]any, error) { row, ok := in.Record.(*models.Robots) if !ok || row == nil { return nil, nil } out := map[string]any{"content": row.Content, "llms_enabled": row.LlmsEnabled, "llms_intro": row.LlmsIntro} for _, b := range classes.Bots { out[classes.BotField(b.Token)] = row.Bots.Allowed(b.Token) } return out, nil }, Save: func(ctx context.Context, in pact.AdminFormSave) (pact.AdminFormSaveResult, error) { svc := p.Service() if svc == nil { return pact.AdminFormSaveResult{}, classes.ErrNotBooted } if row, ok := in.Record.(*models.Robots); !ok || row == nil { return pact.AdminFormSaveResult{}, nil } var w classes.RobotsWrite if raw, ok := in.Values["content"]; ok { if text, ok := raw.(string); ok { w.Content = &text } } for _, b := range classes.Bots { if raw, ok := in.Values[classes.BotField(b.Token)]; ok { if w.Bots == nil { w.Bots = map[string]bool{} } w.Bots[b.Token] = truthy(raw) } } if raw, ok := in.Values["llms_enabled"]; ok { on := truthy(raw) w.LlmsEnabled = &on } if raw, ok := in.Values["llms_intro"]; ok { text, _ := raw.(string) w.LlmsIntro = &text } _, blocked, err := svc.SaveRobots(ctx, in.DB, w) if err != nil || len(blocked) == 0 { return pact.AdminFormSaveResult{}, err } list := blocked if len(list) > 10 { list = append(append([]string{}, list[:10]...), "…") } return pact.AdminFormSaveResult{Notices: map[string][]pact.FormFieldNotice{"content": {{ Message: "golem15.seo::lang.robots.blocks_noindex", Params: map[string]string{"list": strings.Join(list, ", ")}, Count: len(blocked), }}}}, nil }, } } // robotsTestPayload is the tester's request: op context answers the // crawler list and the environment; op test answers whether agent may // fetch url under the editor's unsaved text and switches. type robotsTestPayload struct { Op string `json:"op"` URL string `json:"url"` Agent string `json:"agent"` } // robotsTesterContext is the tester's context answer. type robotsTesterContext struct { Op string `json:"op"` Agents []string `json:"agents"` Bots []string `json:"bots"` Staging bool `json:"staging"` } // robotsTestResult is the tester's test answer. type robotsTestResult struct { Op string `json:"op"` URL string `json:"url"` classes.RobotsTest } // testerAgents are the crawlers E5 offers before the AI bots. var testerAgents = []string{"Googlebot", "Googlebot-Image", "Bingbot"} // robotsTestAction is the tester's widget action (E5, D-22, D-99). func (p *Plugin) robotsTestAction() pact.AdminAction { return pact.AdminAction{ Name: RobotsTestAction, Label: "golem15.seo::lang.robots.test", Permissions: []string{controllers.PermissionManage}, Run: func(ctx context.Context, in pact.AdminActionInput) (pact.AdminActionResult, error) { svc := p.Service() if svc == nil { return pact.AdminActionResult{}, classes.ErrNotBooted } var payload robotsTestPayload if len(in.Payload) > 0 { if err := json.Unmarshal(in.Payload, &payload); err != nil { return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.test_invalid"}}} } } if payload.Op == "context" || payload.Op == "" { out := robotsTesterContext{Op: "context", Agents: append([]string{}, testerAgents...), Staging: svc.Staging()} for _, b := range classes.Bots { out.Agents = append(out.Agents, b.Token) out.Bots = append(out.Bots, b.Token) } out.Agents = append(out.Agents, "*") return pact.AdminActionResult{Data: out}, nil } path := strings.TrimSpace(payload.URL) if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") || len(path) > classes.MaxURLBytes { return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.url_required"}}} } agent := strings.TrimSpace(payload.Agent) if agent == "" { agent = "Googlebot" } content, _ := in.Values["content"].(string) bots := models.BotSwitches{} for _, b := range classes.Bots { if raw, ok := in.Values[classes.BotField(b.Token)]; ok { bots[b.Token] = truthy(raw) } } return pact.AdminActionResult{Data: robotsTestResult{Op: "test", URL: path, RobotsTest: svc.TestRobots(content, bots, agent, path)}}, nil }, } }