sm-seo-plugin/extensions_robots.go

package seo

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"

	"git.golem15.com/golem15/sm-seo-plugin/classes"
	"git.golem15.com/golem15/sm-seo-plugin/controllers"
	"git.golem15.com/golem15/sm-seo-plugin/models"
	"git.golem15.com/golem15/summercms/modules/cabana"
	"git.golem15.com/golem15/summercms/modules/pact"
	"github.com/goccy/go-yaml"
)

// The robots.txt form's tab keys (UI-SPEC C10).
const (
	TabRobotsFile = "golem15.seo::lang.robots.tab_file"
	TabRobotsBots = "golem15.seo::lang.robots.tab_bots"
)

// RobotsTestAction is the tester's widget action (E5).
const RobotsTestAction = "robots_test"

// botComments are the switch comments per bot kind.
var botComments = map[string]string{
	classes.BotSearch:   "golem15.seo::lang.bots.search_comment",
	classes.BotUser:     "golem15.seo::lang.bots.user_comment",
	classes.BotTraining: "golem15.seo::lang.bots.training_comment",
}

// robotsDocument builds the robots form's fields document: the editor
// (E4) and the tester (E5, filled from the editor and every bot switch)
// after the staging callout on the robots.txt tab, then the AI bots tab
// with one switch per bot in classes.Bots order, labelled with its token
// and operator.
func robotsDocument() ([]byte, error) {
	type tabSpec struct {
		Name  string `yaml:"name"`
		After string `yaml:"after"`
	}
	fill := []string{"content"}
	for _, b := range classes.Bots {
		fill = append(fill, classes.BotField(b.Token))
	}
	fields := yaml.MapSlice{
		{Key: "content", Value: yaml.MapSlice{
			{Key: "label", Value: "golem15.seo::lang.robots.content"},
			{Key: "comment", Value: "golem15.seo::lang.robots.content_comment"},
			{Key: "type", Value: "robotstxt"},
			{Key: "span", Value: "full"},
			{Key: "after", Value: "staging"},
			{Key: "tab", Value: TabRobotsFile},
		}},
		{Key: "tester", Value: yaml.MapSlice{
			{Key: "label", Value: "golem15.seo::lang.robots.tester"},
			{Key: "type", Value: "widget"},
			{Key: "widget", Value: classes.ElementRobotsTester},
			{Key: "action", Value: RobotsTestAction},
			{Key: "fill", Value: fill},
			{Key: "span", Value: "full"},
			{Key: "after", Value: "content"},
			{Key: "tab", Value: TabRobotsFile},
		}},
	}
	for i, b := range classes.Bots {
		span := "left"
		if i%2 == 1 {
			span = "right"
		}
		fields = append(fields, yaml.MapItem{Key: classes.BotField(b.Token), Value: yaml.MapSlice{
			{Key: "label", Value: b.Token + " (" + b.Operator + ")"},
			{Key: "comment", Value: botComments[b.Kind]},
			{Key: "type", Value: "switch"},
			{Key: "default", Value: true},
			{Key: "span", Value: span},
			{Key: "tab", Value: TabRobotsBots},
		}})
	}
	return yaml.Marshal(yaml.MapSlice{
		{Key: "tabs", Value: []tabSpec{{Name: TabRobotsBots, After: TabRobotsFile}}},
		{Key: "fields", Value: fields},
	})
}

// robotsForm is the plugin's own extension on the robots.txt screen
// (D-22, D-24, D-25): the editor, the tester and the AI bot switches,
// loaded from and saved to the robots row inside the save transaction.
// Save records history and answers robots.blocks_noindex on content when
// the rules block a noindex address from Googlebot.
func (p *Plugin) robotsForm() *pact.AdminFormExtension {
	doc, err := robotsDocument()
	if err != nil {
		panic(fmt.Errorf("golem15.seo: robots form: %w", err))
	}
	return &pact.AdminFormExtension{
		FieldsYAML:  doc,
		Permissions: []string{controllers.PermissionManage},
		Actions:     []pact.AdminAction{p.robotsTestAction()},
		Load: func(_ context.Context, in pact.AdminFormLoad) (map[string]any, error) {
			row, ok := in.Record.(*models.Robots)
			if !ok || row == nil {
				return nil, nil
			}
			out := map[string]any{"content": row.Content}
			for _, b := range classes.Bots {
				out[classes.BotField(b.Token)] = row.Bots.Allowed(b.Token)
			}
			return out, nil
		},
		Save: func(ctx context.Context, in pact.AdminFormSave) (pact.AdminFormSaveResult, error) {
			svc := p.Service()
			if svc == nil {
				return pact.AdminFormSaveResult{}, classes.ErrNotBooted
			}
			if row, ok := in.Record.(*models.Robots); !ok || row == nil {
				return pact.AdminFormSaveResult{}, nil
			}
			var w classes.RobotsWrite
			if raw, ok := in.Values["content"]; ok {
				if text, ok := raw.(string); ok {
					w.Content = &text
				}
			}
			for _, b := range classes.Bots {
				if raw, ok := in.Values[classes.BotField(b.Token)]; ok {
					if w.Bots == nil {
						w.Bots = map[string]bool{}
					}
					w.Bots[b.Token] = truthy(raw)
				}
			}
			_, blocked, err := svc.SaveRobots(ctx, in.DB, w)
			if err != nil || len(blocked) == 0 {
				return pact.AdminFormSaveResult{}, err
			}
			list := blocked
			if len(list) > 10 {
				list = append(append([]string{}, list[:10]...), "…")
			}
			return pact.AdminFormSaveResult{Notices: map[string][]pact.FormFieldNotice{"content": {{
				Message: "golem15.seo::lang.robots.blocks_noindex",
				Params:  map[string]string{"list": strings.Join(list, ", ")},
				Count:   len(blocked),
			}}}}, nil
		},
	}
}

// robotsTestPayload is the tester's request: op context answers the
// crawler list and the environment; op test answers whether agent may
// fetch url under the editor's unsaved text and switches.
type robotsTestPayload struct {
	Op    string `json:"op"`
	URL   string `json:"url"`
	Agent string `json:"agent"`
}

// robotsTesterContext is the tester's context answer.
type robotsTesterContext struct {
	Op      string   `json:"op"`
	Agents  []string `json:"agents"`
	Bots    []string `json:"bots"`
	Staging bool     `json:"staging"`
}

// robotsTestResult is the tester's test answer.
type robotsTestResult struct {
	Op  string `json:"op"`
	URL string `json:"url"`
	classes.RobotsTest
}

// testerAgents are the crawlers E5 offers before the AI bots.
var testerAgents = []string{"Googlebot", "Googlebot-Image", "Bingbot"}

// robotsTestAction is the tester's widget action (E5, D-22, D-99).
func (p *Plugin) robotsTestAction() pact.AdminAction {
	return pact.AdminAction{
		Name:        RobotsTestAction,
		Label:       "golem15.seo::lang.robots.test",
		Permissions: []string{controllers.PermissionManage},
		Run: func(ctx context.Context, in pact.AdminActionInput) (pact.AdminActionResult, error) {
			svc := p.Service()
			if svc == nil {
				return pact.AdminActionResult{}, classes.ErrNotBooted
			}
			var payload robotsTestPayload
			if len(in.Payload) > 0 {
				if err := json.Unmarshal(in.Payload, &payload); err != nil {
					return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.test_invalid"}}}
				}
			}
			if payload.Op == "context" || payload.Op == "" {
				out := robotsTesterContext{Op: "context", Agents: append([]string{}, testerAgents...), Staging: svc.Staging()}
				for _, b := range classes.Bots {
					out.Agents = append(out.Agents, b.Token)
					out.Bots = append(out.Bots, b.Token)
				}
				out.Agents = append(out.Agents, "*")
				return pact.AdminActionResult{Data: out}, nil
			}
			path := strings.TrimSpace(payload.URL)
			if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") || len(path) > classes.MaxURLBytes {
				return pact.AdminActionResult{}, &cabana.ValidationError{Details: map[string]any{in.Field: []string{"golem15.seo::lang.robots.url_required"}}}
			}
			agent := strings.TrimSpace(payload.Agent)
			if agent == "" {
				agent = "Googlebot"
			}
			content, _ := in.Values["content"].(string)
			bots := models.BotSwitches{}
			for _, b := range classes.Bots {
				if raw, ok := in.Values[classes.BotField(b.Token)]; ok {
					bots[b.Token] = truthy(raw)
				}
			}
			return pact.AdminActionResult{Data: robotsTestResult{Op: "test", URL: path, RobotsTest: svc.TestRobots(content, bots, agent, path)}}, nil
		},
	}
}