742a5b930b
- GET /_seo/api/v1/site (public, noindex): site_url, default_locale, locales, unprefixed_locale, links.new_tab, html_sitemap (from the sitemap plugin's Listing), routes of every content type, root_files (verification files and the IndexNow key file) and llms - GET /llms.txt: with llms_enabled (robots.txt screen, AI bots tab, off by default) and in production, the indexable pages from the sitemaps per group under the site name and llms_intro (else the default description); otherwise 404; seo_update_robots takes both fields - robots.txt names the sitemap plugin's index unless golem15.seo.robots_sitemaps is set; the config default is now a comment
255 lines
8.7 KiB
Go
255 lines
8.7 KiB
Go
package seo
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
|
"git.golem15.com/golem15/sm-seo-plugin/controllers"
|
|
"git.golem15.com/golem15/sm-seo-plugin/models"
|
|
"git.golem15.com/golem15/summercms/modules/cabana"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"github.com/goccy/go-yaml"
|
|
)
|
|
|
|
// The robots.txt form's tab keys (UI-SPEC C10).
|
|
const (
|
|
TabRobotsFile = "golem15.seo::lang.robots.tab_file"
|
|
TabRobotsBots = "golem15.seo::lang.robots.tab_bots"
|
|
)
|
|
|
|
// RobotsTestAction is the tester's widget action (E5).
|
|
const RobotsTestAction = "robots_test"
|
|
|
|
// botComments are the switch comments per bot kind.
|
|
var botComments = map[string]string{
|
|
classes.BotSearch: "golem15.seo::lang.bots.search_comment",
|
|
classes.BotUser: "golem15.seo::lang.bots.user_comment",
|
|
classes.BotTraining: "golem15.seo::lang.bots.training_comment",
|
|
}
|
|
|
|
// robotsDocument builds the robots form's fields document: the editor
|
|
// (E4) and the tester (E5, filled from the editor and every bot switch)
|
|
// after the staging callout on the robots.txt tab, then the AI bots tab
|
|
// with one switch per bot in classes.Bots order, labelled with its token
|
|
// and operator.
|
|
func robotsDocument() ([]byte, error) {
|
|
type tabSpec struct {
|
|
Name string `yaml:"name"`
|
|
After string `yaml:"after"`
|
|
}
|
|
fill := []string{"content"}
|
|
for _, b := range classes.Bots {
|
|
fill = append(fill, classes.BotField(b.Token))
|
|
}
|
|
fields := yaml.MapSlice{
|
|
{Key: "content", Value: yaml.MapSlice{
|
|
{Key: "label", Value: "golem15.seo::lang.robots.content"},
|
|
{Key: "comment", Value: "golem15.seo::lang.robots.content_comment"},
|
|
{Key: "type", Value: "robotstxt"},
|
|
{Key: "span", Value: "full"},
|
|
{Key: "after", Value: "staging"},
|
|
{Key: "tab", Value: TabRobotsFile},
|
|
}},
|
|
{Key: "tester", Value: yaml.MapSlice{
|
|
{Key: "label", Value: "golem15.seo::lang.robots.tester"},
|
|
{Key: "type", Value: "widget"},
|
|
{Key: "widget", Value: classes.ElementRobotsTester},
|
|
{Key: "action", Value: RobotsTestAction},
|
|
{Key: "fill", Value: fill},
|
|
{Key: "span", Value: "full"},
|
|
{Key: "after", Value: "content"},
|
|
{Key: "tab", Value: TabRobotsFile},
|
|
}},
|
|
}
|
|
for i, b := range classes.Bots {
|
|
span := "left"
|
|
if i%2 == 1 {
|
|
span = "right"
|
|
}
|
|
fields = append(fields, yaml.MapItem{Key: classes.BotField(b.Token), Value: yaml.MapSlice{
|
|
{Key: "label", Value: b.Token + " (" + b.Operator + ")"},
|
|
{Key: "comment", Value: botComments[b.Kind]},
|
|
{Key: "type", Value: "switch"},
|
|
{Key: "default", Value: true},
|
|
{Key: "span", Value: span},
|
|
{Key: "tab", Value: TabRobotsBots},
|
|
}})
|
|
}
|
|
// llms.txt (UI-SPEC C25) after the bot switches.
|
|
fields = append(fields,
|
|
yaml.MapItem{Key: "llms_enabled", Value: yaml.MapSlice{
|
|
{Key: "label", Value: "golem15.seo::lang.robots.llms_enabled"},
|
|
{Key: "comment", Value: "golem15.seo::lang.robots.llms_enabled_comment"},
|
|
{Key: "type", Value: "switch"},
|
|
{Key: "default", Value: false},
|
|
{Key: "span", Value: "full"},
|
|
{Key: "tab", Value: TabRobotsBots},
|
|
}},
|
|
yaml.MapItem{Key: "llms_intro", Value: yaml.MapSlice{
|
|
{Key: "label", Value: "golem15.seo::lang.robots.llms_intro"},
|
|
{Key: "comment", Value: "golem15.seo::lang.robots.llms_intro_comment"},
|
|
{Key: "type", Value: "textarea"},
|
|
{Key: "size", Value: "small"},
|
|
{Key: "span", Value: "full"},
|
|
{Key: "tab", Value: TabRobotsBots},
|
|
{Key: "trigger", Value: yaml.MapSlice{
|
|
{Key: "action", Value: "hide"},
|
|
{Key: "field", Value: "llms_enabled"},
|
|
{Key: "condition", Value: "unchecked"},
|
|
}},
|
|
}},
|
|
)
|
|
return yaml.Marshal(yaml.MapSlice{
|
|
{Key: "tabs", Value: []tabSpec{{Name: TabRobotsBots, After: TabRobotsFile}}},
|
|
{Key: "fields", Value: fields},
|
|
})
|
|
}
|
|
|
|
// robotsForm is the plugin's own extension on the robots.txt screen
|
|
// (D-22, D-24, D-25): the editor, the tester and the AI bot switches,
|
|
// loaded from and saved to the robots row inside the save transaction.
|
|
// Save records history and answers robots.blocks_noindex on content when
|
|
// the rules block a noindex address from Googlebot.
|
|
func (p *Plugin) robotsForm() *pact.AdminFormExtension {
|
|
doc, err := robotsDocument()
|
|
if err != nil {
|
|
panic(fmt.Errorf("golem15.seo: robots form: %w", err))
|
|
}
|
|
return &pact.AdminFormExtension{
|
|
FieldsYAML: doc,
|
|
Permissions: []string{controllers.PermissionManage},
|
|
Actions: []pact.AdminAction{p.robotsTestAction()},
|
|
Load: func(_ context.Context, in pact.AdminFormLoad) (map[string]any, error) {
|
|
row, ok := in.Record.(*models.Robots)
|
|
if !ok || row == nil {
|
|
return nil, nil
|
|
}
|
|
out := map[string]any{"content": row.Content, "llms_enabled": row.LlmsEnabled, "llms_intro": row.LlmsIntro}
|
|
for _, b := range classes.Bots {
|
|
out[classes.BotField(b.Token)] = row.Bots.Allowed(b.Token)
|
|
}
|
|
return out, nil
|
|
},
|
|
Save: func(ctx context.Context, in pact.AdminFormSave) (pact.AdminFormSaveResult, error) {
|
|
svc := p.Service()
|
|
if svc == nil {
|
|
return pact.AdminFormSaveResult{}, classes.ErrNotBooted
|
|
}
|
|
if row, ok := in.Record.(*models.Robots); !ok || row == nil {
|
|
return pact.AdminFormSaveResult{}, nil
|
|
}
|
|
var w classes.RobotsWrite
|
|
if raw, ok := in.Values["content"]; ok {
|
|
if text, ok := raw.(string); ok {
|
|
w.Content = &text
|
|
}
|
|
}
|
|
for _, b := range classes.Bots {
|
|
if raw, ok := in.Values[classes.BotField(b.Token)]; ok {
|
|
if w.Bots == nil {
|
|
w.Bots = map[string]bool{}
|
|
}
|
|
w.Bots[b.Token] = truthy(raw)
|
|
}
|
|
}
|
|
if raw, ok := in.Values["llms_enabled"]; ok {
|
|
on := truthy(raw)
|
|
w.LlmsEnabled = &on
|
|
}
|
|
if raw, ok := in.Values["llms_intro"]; ok {
|
|
text, _ := raw.(string)
|
|
w.LlmsIntro = &text
|
|
}
|
|
_, blocked, err := svc.SaveRobots(ctx, in.DB, w)
|
|
if err != nil || len(blocked) == 0 {
|
|
return pact.AdminFormSaveResult{}, err
|
|
}
|
|
list := blocked
|
|
if len(list) > 10 {
|
|
list = append(append([]string{}, list[:10]...), "…")
|
|
}
|
|
return pact.AdminFormSaveResult{Notices: map[string][]pact.FormFieldNotice{"content": {{
|
|
Message: "golem15.seo::lang.robots.blocks_noindex",
|
|
Params: map[string]string{"list": strings.Join(list, ", ")},
|
|
Count: len(blocked),
|
|
}}}}, nil
|
|
},
|
|
}
|
|
}
|
|
|
|
// robotsTestPayload is the tester's request: op context answers the
|
|
// crawler list and the environment; op test answers whether agent may
|
|
// fetch url under the editor's unsaved text and switches.
|
|
type robotsTestPayload struct {
|
|
Op string `json:"op"`
|
|
URL string `json:"url"`
|
|
Agent string `json:"agent"`
|
|
}
|
|
|
|
// robotsTesterContext is the tester's context answer.
|
|
type robotsTesterContext struct {
|
|
Op string `json:"op"`
|
|
Agents []string `json:"agents"`
|
|
Bots []string `json:"bots"`
|
|
Staging bool `json:"staging"`
|
|
}
|
|
|
|
// robotsTestResult is the tester's test answer.
|
|
type robotsTestResult struct {
|
|
Op string `json:"op"`
|
|
URL string `json:"url"`
|
|
classes.RobotsTest
|
|
}
|
|
|
|
// testerAgents are the crawlers E5 offers before the AI bots.
|
|
var testerAgents = []string{"Googlebot", "Googlebot-Image", "Bingbot"}
|
|
|
|
// robotsTestAction is the tester's widget action (E5, D-22, D-99).
|
|
func (p *Plugin) robotsTestAction() pact.AdminAction {
|
|
return pact.AdminAction{
|
|
Name: RobotsTestAction,
|
|
Label: "golem15.seo::lang.robots.test",
|
|
Permissions: []string{controllers.PermissionManage},
|
|
Run: func(ctx context.Context, in pact.AdminActionInput) (pact.AdminActionResult, error) {
|
|
svc := p.Service()
|
|
if svc == nil {
|
|
return pact.AdminActionResult{}, classes.ErrNotBooted
|
|
}
|
|
var payload robotsTestPayload
|
|
if len(in.Payload) > 0 {
|
|
if err := json.Unmarshal(in.Payload, &payload); err != nil {
|
|
return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.test_invalid"}}}
|
|
}
|
|
}
|
|
if payload.Op == "context" || payload.Op == "" {
|
|
out := robotsTesterContext{Op: "context", Agents: append([]string{}, testerAgents...), Staging: svc.Staging()}
|
|
for _, b := range classes.Bots {
|
|
out.Agents = append(out.Agents, b.Token)
|
|
out.Bots = append(out.Bots, b.Token)
|
|
}
|
|
out.Agents = append(out.Agents, "*")
|
|
return pact.AdminActionResult{Data: out}, nil
|
|
}
|
|
path := strings.TrimSpace(payload.URL)
|
|
if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") || len(path) > classes.MaxURLBytes {
|
|
return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.url_required"}}}
|
|
}
|
|
agent := strings.TrimSpace(payload.Agent)
|
|
if agent == "" {
|
|
agent = "Googlebot"
|
|
}
|
|
content, _ := in.Values["content"].(string)
|
|
bots := models.BotSwitches{}
|
|
for _, b := range classes.Bots {
|
|
if raw, ok := in.Values[classes.BotField(b.Token)]; ok {
|
|
bots[b.Token] = truthy(raw)
|
|
}
|
|
}
|
|
return pact.AdminActionResult{Data: robotsTestResult{Op: "test", URL: path, RobotsTest: svc.TestRobots(content, bots, agent, path)}}, nil
|
|
},
|
|
}
|
|
}
|