sm-seo-plugin/controllers/api/robots.go

package api

import (
	"context"
	"net/http"
	"strings"

	"git.golem15.com/golem15/sm-seo-plugin/classes"
	"git.golem15.com/golem15/sm-seo-plugin/models"
	"gorm.io/gorm"
)

// Robots serves GET /robots.txt (D-22, D-25, P-23) as text/plain. In
// production it is the stored rules, a Disallow group for every
// switched-off AI bot and the Sitemap lines; outside production it is
// "User-agent: *" / "Disallow: /" with X-Robots-Tag: noindex, nofollow,
// whatever is stored, and no admin setting can change that. It never
// answers a 5xx: when the stored file cannot be read it serves the last
// body it served, else an allow-all file, and logs the error.
func Robots(service func() *classes.Service) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		svc := service()
		body := classes.FallbackRobots
		staging := false
		if svc == nil {
			log.Error("golem15.seo: robots.txt served before boot")
		} else {
			staging = svc.Staging()
			served, err := svc.ServedFile(r.Context(), !staging)
			switch {
			case err == nil:
				body = served
			case svc.LastServedRobots() != "":
				log.Error("golem15.seo: robots.txt, serving the last good file", "error", err)
				body = svc.LastServedRobots()
			default:
				log.Error("golem15.seo: robots.txt, serving the allow-all fallback", "error", err)
			}
		}
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		if staging {
			w.Header().Set("X-Robots-Tag", "noindex, nofollow")
		}
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write([]byte(body))
	}
}

// UpdateRobotsInput is the body of PUT /_seo/api/v1/robots.
type UpdateRobotsInput struct {
	Content *string         `json:"content,omitempty" jsonschema:"the robots.txt rules; Sitemap lines and the AI bot rules are added when the file is served"`
	Bots    map[string]bool `json:"bots,omitempty" jsonschema:"AI bot switches by user-agent token, for example {\"GPTBot\": false}; a bot switched off is disallowed"`
	// LlmsEnabled and LlmsIntro are the llms.txt switch and introduction.
	LlmsEnabled *bool   `json:"llms_enabled,omitempty" jsonschema:"serve /llms.txt: the site's indexable pages from the sitemaps, for AI tools (off by default)"`
	LlmsIntro   *string `json:"llms_intro,omitempty" jsonschema:"a short description of the site at the top of llms.txt; empty uses the default meta description"`
}

// TestRobotsInput is the query of GET /_seo/api/v1/robots/test.
type TestRobotsInput struct {
	URL   string `json:"url" query:"url" jsonschema:"the address to test, a path starting with /"`
	Agent string `json:"agent,omitempty" query:"agent" jsonschema:"the crawler's user agent token (Googlebot by default)"`
}

// robotsJSON is the robots.txt screen in an answer: the rules, every AI
// bot with its switch, the file crawlers get in production and whether
// this site runs outside production (then crawlers get the lock).
func robotsJSON(ctx context.Context, svc *classes.Service, row models.Robots) (map[string]any, error) {
	bots := make([]map[string]any, 0, len(classes.Bots))
	for _, b := range classes.Bots {
		bots = append(bots, map[string]any{"token": b.Token, "operator": b.Operator, "kind": b.Kind, "allowed": row.Bots.Allowed(b.Token)})
	}
	production := classes.ComposeRobots(row.Content, row.Bots, svc.SitemapLines(ctx))
	return map[string]any{
		"content":      row.Content,
		"bots":         bots,
		"llms_enabled": row.LlmsEnabled,
		"llms_intro":   row.LlmsIntro,
		"production":   production,
		"staging":      svc.Staging(),
	}, nil
}

// ShowRobots serves GET /_seo/api/v1/robots.
func ShowRobots(service func() *classes.Service) http.HandlerFunc {
	return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
		db, err := read(ctx, svc)
		if err != nil {
			return 0, nil, err
		}
		row, err := classes.LoadRobots(db)
		if err != nil {
			return 0, nil, err
		}
		body, err := robotsJSON(ctx, svc, row)
		if err != nil {
			return 0, nil, err
		}
		return http.StatusOK, map[string]any{"data": body}, nil
	})
}

// UpdateRobots serves PUT /_seo/api/v1/robots: the rules and the bot
// switches sent change (D-22, D-25). The answer lists the noindex
// addresses the new rules keep Googlebot from (robots.blocks_noindex).
func UpdateRobots(service func() *classes.Service) http.HandlerFunc {
	return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
		var in UpdateRobotsInput
		if err := decode(r, classes.MaxRobotsBytes+(64<<10), &in); err != nil {
			return 0, nil, err
		}
		var row models.Robots
		var blocked []string
		err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
			var err error
			row, blocked, err = svc.SaveRobots(ctx, tx, classes.RobotsWrite{Content: in.Content, Bots: in.Bots, LlmsEnabled: in.LlmsEnabled, LlmsIntro: in.LlmsIntro})
			return err
		})
		if err != nil {
			return 0, nil, err
		}
		body, err := robotsJSON(ctx, svc, row)
		if err != nil {
			return 0, nil, err
		}
		out := map[string]any{"data": body}
		if len(blocked) > 0 {
			out["warnings"] = []map[string]any{{"kind": "blocks_noindex", "urls": blocked}}
		}
		return http.StatusOK, out, nil
	})
}

// TestRobotsURL serves GET /_seo/api/v1/robots/test?url=&agent=: whether
// the crawler may fetch the address under the saved production file,
// through Google's matcher, with the deciding line (D-99).
func TestRobotsURL(service func() *classes.Service) http.HandlerFunc {
	return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
		path := strings.TrimSpace(r.URL.Query().Get("url"))
		if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") || len(path) > classes.MaxURLBytes {
			return 0, nil, &apiError{Status: http.StatusUnprocessableEntity, Body: map[string]any{
				"error": "validation_failed", "errors": map[string][]string{"url": {svc.PhraseCtx(ctx, "golem15.seo::lang.robots.url_required", nil)}},
			}}
		}
		agent := strings.TrimSpace(r.URL.Query().Get("agent"))
		if agent == "" {
			agent = "Googlebot"
		}
		db, err := read(ctx, svc)
		if err != nil {
			return 0, nil, err
		}
		row, err := classes.LoadRobots(db)
		if err != nil {
			return 0, nil, err
		}
		return http.StatusOK, map[string]any{"url": path, "result": svc.TestRobots(row.Content, row.Bots, agent, path)}, nil
	})
}