feat: site endpoint, llms.txt and the sitemap index in robots.txt

- GET /_seo/api/v1/site (public, noindex): site_url, default_locale,
  locales, unprefixed_locale, links.new_tab, html_sitemap (from the
  sitemap plugin's Listing), routes of every content type, root_files
  (verification files and the IndexNow key file) and llms
- GET /llms.txt: with llms_enabled (robots.txt screen, AI bots tab, off by
  default) and in production, the indexable pages from the sitemaps per
  group under the site name and llms_intro (else the default
  description); otherwise 404; seo_update_robots takes both fields
- robots.txt names the sitemap plugin's index unless
  golem15.seo.robots_sitemaps is set; the config default is now a comment

controllers/api/llms.go

@@ -0,0 +1,34 @@
+package api
+
+import (
+	"net/http"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+)
+
+// LLMS serves GET /llms.txt at the site root (D-25, UI-SPEC C25): while
+// llms.txt is on (the robots.txt screen's AI bots tab) and in production,
+// the site's indexable pages from the sitemaps as text/plain; otherwise
+// 404.
+func LLMS(service func() *classes.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		svc := service()
+		if svc == nil {
+			http.Error(w, "SEO is not available", http.StatusServiceUnavailable)
+			return
+		}
+		body, ok, err := svc.LLMSText(r.Context())
+		if err != nil {
+			log.Error("golem15.seo: llms.txt", "error", err)
+			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+			return
+		}
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		_, _ = w.Write([]byte(body))
+	}
+}

controllers/api/robots.go

@@ -50,6 +50,9 @@ func Robots(service func() *classes.Service) http.HandlerFunc {
 type UpdateRobotsInput struct {
 	Content *string         `json:"content,omitempty" jsonschema:"the robots.txt rules; Sitemap lines and the AI bot rules are added when the file is served"`
 	Bots    map[string]bool `json:"bots,omitempty" jsonschema:"AI bot switches by user-agent token, for example {\"GPTBot\": false}; a bot switched off is disallowed"`
+	// LlmsEnabled and LlmsIntro are the llms.txt switch and introduction.
+	LlmsEnabled *bool   `json:"llms_enabled,omitempty" jsonschema:"serve /llms.txt: the site's indexable pages from the sitemaps, for AI tools (off by default)"`
+	LlmsIntro   *string `json:"llms_intro,omitempty" jsonschema:"a short description of the site at the top of llms.txt; empty uses the default meta description"`
 }
 
 // TestRobotsInput is the query of GET /_seo/api/v1/robots/test.
@@ -66,12 +69,14 @@ func robotsJSON(ctx context.Context, svc *classes.Service, row models.Robots) (m
 	for _, b := range classes.Bots {
 		bots = append(bots, map[string]any{"token": b.Token, "operator": b.Operator, "kind": b.Kind, "allowed": row.Bots.Allowed(b.Token)})
 	}
-	production := classes.ComposeRobots(row.Content, row.Bots, svc.RobotsSitemaps())
+	production := classes.ComposeRobots(row.Content, row.Bots, svc.SitemapLines(ctx))
 	return map[string]any{
-		"content":    row.Content,
-		"bots":       bots,
-		"production": production,
-		"staging":    svc.Staging(),
+		"content":      row.Content,
+		"bots":         bots,
+		"llms_enabled": row.LlmsEnabled,
+		"llms_intro":   row.LlmsIntro,
+		"production":   production,
+		"staging":      svc.Staging(),
 	}, nil
 }
 
@@ -107,7 +112,7 @@ func UpdateRobots(service func() *classes.Service) http.HandlerFunc {
 		var blocked []string
 		err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
 			var err error
-			row, blocked, err = svc.SaveRobots(ctx, tx, classes.RobotsWrite{Content: in.Content, Bots: in.Bots})
+			row, blocked, err = svc.SaveRobots(ctx, tx, classes.RobotsWrite{Content: in.Content, Bots: in.Bots, LlmsEnabled: in.LlmsEnabled, LlmsIntro: in.LlmsIntro})
 			return err
 		})
 		if err != nil {

controllers/api/site.go

@@ -0,0 +1,29 @@
+package api
+
+import (
+	"net/http"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+	"git.golem15.com/golem15/summercms/modules/wire"
+)
+
+// Site serves GET /_seo/api/v1/site (P33-17): the site facts the frontend
+// layer needs (site URL, locales, the new-tab switch, the HTML sitemap
+// settings, the content types' URL patterns, the root file names and
+// whether llms.txt is served). Every value is public.
+func Site(service func() *classes.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		svc := service()
+		if svc == nil {
+			wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "SEO is not available"})
+			return
+		}
+		out, err := svc.Site(r.Context())
+		if err != nil {
+			log.Error("golem15.seo: site answer", "error", err)
+			wire.WriteJSON(w, http.StatusInternalServerError, map[string]string{"error": errInternal})
+			return
+		}
+		wire.WriteJSON(w, http.StatusOK, out)
+	}
+}