feat: site endpoint, llms.txt and the sitemap index in robots.txt

- GET /_seo/api/v1/site (public, noindex): site_url, default_locale,
  locales, unprefixed_locale, links.new_tab, html_sitemap (from the
  sitemap plugin's Listing), routes of every content type, root_files
  (verification files and the IndexNow key file) and llms
- GET /llms.txt: with llms_enabled (robots.txt screen, AI bots tab, off by
  default) and in production, the indexable pages from the sitemaps per
  group under the site name and llms_intro (else the default
  description); otherwise 404; seo_update_robots takes both fields
- robots.txt names the sitemap plugin's index unless
  golem15.seo.robots_sitemaps is set; the config default is now a comment

classes/llms.go

@@ -0,0 +1,177 @@
+package classes
+
+import (
+	"context"
+	"sort"
+	"strings"
+
+	"git.golem15.com/golem15/sm-sitemap-plugin/contract"
+)
+
+// MaxLLMSEntries caps the links llms.txt lists.
+const MaxLLMSEntries = 10000
+
+// llmsGroup is one section of llms.txt.
+type llmsGroup struct {
+	title   string
+	order   int
+	key     string
+	entries []contract.Entry
+}
+
+// LLMSText builds /llms.txt (D-25, UI-SPEC C25): "# <site name>", the
+// introduction (llms_intro, else the default meta description) as a quote,
+// then one "## <group title>" section per sitemap group with listed
+// addresses in the default locale, each a "- [title](absolute url)" line.
+// It reads the sitemaps through contract.Listing, so an address the
+// sitemaps leave out (noindex, unpublished, canonicalised) never appears
+// (T-33-20). The groups follow the content types' order, as in the HTML
+// sitemap. ok is false while llms.txt is off and outside production.
+func (s *Service) LLMSText(ctx context.Context) (string, bool, error) {
+	if s.Staging() {
+		return "", false, nil
+	}
+	db, err := s.DB(ctx)
+	if err != nil {
+		return "", false, err
+	}
+	robots, err := LoadRobots(db)
+	if err != nil {
+		return "", false, err
+	}
+	if !robots.LlmsEnabled {
+		return "", false, nil
+	}
+	settings, err := LoadSettings(db)
+	if err != nil {
+		return "", false, err
+	}
+	locale := s.DefaultLocale(ctx, db)
+	var b strings.Builder
+	name := s.SiteName(ctx, settings, locale)
+	if name == "" {
+		name = s.SiteURL()
+	}
+	b.WriteString("# " + llmsLine(name) + "\n")
+	intro := strings.TrimSpace(robots.LlmsIntro)
+	if intro == "" {
+		intro = strings.TrimSpace(s.translatedSetting(ctx, db, &settings, "default_description", settings.DefaultDescription, locale))
+	}
+	if intro != "" {
+		b.WriteString("\n> " + llmsLine(intro) + "\n")
+	}
+	groups, err := s.llmsGroups(ctx, locale)
+	if err != nil {
+		return "", false, err
+	}
+	for _, g := range groups {
+		b.WriteString("\n## " + llmsLine(g.title) + "\n\n")
+		for _, e := range g.entries {
+			title := strings.TrimSpace(e.Title)
+			if title == "" {
+				title = e.URL
+			}
+			b.WriteString("- [" + llmsText(title) + "](" + llmsURL(e.URL) + ")\n")
+		}
+	}
+	return b.String(), true, nil
+}
+
+// llmsGroups reads the listed addresses of every sitemap group in locale,
+// ordered by the content type of their first address (registration order),
+// then by key.
+func (s *Service) llmsGroups(ctx context.Context, locale string) ([]llmsGroup, error) {
+	l := s.listing()
+	if l == nil {
+		return nil, nil
+	}
+	st, err := l.Settings(ctx)
+	if err != nil {
+		return nil, err
+	}
+	order := map[string]int{}
+	if reg := s.Registry(); reg != nil {
+		for i, typ := range reg.ContentTypes() {
+			order[typ.Key] = i
+		}
+	}
+	total := 0
+	var out []llmsGroup
+	for _, g := range st.Groups {
+		if g.Count == 0 || total >= MaxLLMSEntries {
+			continue
+		}
+		grp := llmsGroup{key: g.Key, order: len(order) + 1, title: groupTitle(g, locale, st.DefaultLocale)}
+		after := ""
+		for total < MaxLLMSEntries {
+			page, err := l.Entries(ctx, contract.EntriesQuery{Group: g.Key, Locale: locale, After: after, Limit: 1000})
+			if err != nil {
+				return nil, err
+			}
+			for _, e := range page.Entries {
+				if total >= MaxLLMSEntries {
+					break
+				}
+				if len(grp.entries) == 0 {
+					if i, ok := order[e.TypeKey]; ok {
+						grp.order = i
+					}
+				}
+				grp.entries = append(grp.entries, e)
+				total++
+			}
+			if page.Next == "" || page.Next == after {
+				break
+			}
+			after = page.Next
+		}
+		if len(grp.entries) > 0 {
+			out = append(out, grp)
+		}
+	}
+	sort.SliceStable(out, func(i, j int) bool {
+		if out[i].order != out[j].order {
+			return out[i].order < out[j].order
+		}
+		return out[i].key < out[j].key
+	})
+	return out, nil
+}
+
+// groupTitle is a group's heading in locale: its title there, else in the
+// sitemap's default locale, else any, else its key.
+func groupTitle(g contract.GroupInfo, locale, def string) string {
+	for _, l := range []string{locale, def} {
+		if t := strings.TrimSpace(g.Titles[l]); t != "" {
+			return t
+		}
+	}
+	keys := make([]string, 0, len(g.Titles))
+	for k := range g.Titles {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		if t := strings.TrimSpace(g.Titles[k]); t != "" {
+			return t
+		}
+	}
+	return g.Key
+}
+
+// llmsLine collapses text to one line.
+func llmsLine(text string) string {
+	return strings.Join(strings.Fields(text), " ")
+}
+
+// llmsText is a link text: one line with the Markdown brackets and
+// backslashes escaped.
+func llmsText(text string) string {
+	return strings.NewReplacer(`\`, `\\`, `[`, `\[`, `]`, `\]`).Replace(llmsLine(text))
+}
+
+// llmsURL is a link target: spaces and parentheses percent-encoded, so
+// the Markdown link stays whole.
+func llmsURL(u string) string {
+	return strings.NewReplacer(" ", "%20", "(", "%28", ")", "%29", "<", "%3C", ">", "%3E").Replace(strings.TrimSpace(u))
+}

classes/robots.go

@@ -11,8 +11,8 @@ import (
 )
 
 // RobotsSitemapsConfigKey lists the sitemap addresses robots.txt names,
-// one Sitemap line each, absolute on the site URL (P-23); default
-// ["/sitemap.xml"].
+// one Sitemap line each, absolute on the site URL (P-23). Unset, robots.txt
+// names the sitemap plugin's index (D-22, SitemapLines).
 const RobotsSitemapsConfigKey = "golem15.seo.robots_sitemaps"
 
 // MaxRobotsBytes is the largest robots.txt the editor stores: Google
@@ -29,8 +29,49 @@ const (
 	FallbackRobots = "User-agent: *\nDisallow:\n"
 )
 
+// SitemapLines are the addresses of robots.txt's Sitemap lines (D-22):
+// golem15.seo.robots_sitemaps when it is set, else the sitemap index of
+// the sitemap plugin's contract.Listing (its IndexURL), else /sitemap.xml
+// on the site URL. The per-type files are never listed: the index names
+// them.
+func (s *Service) SitemapLines(ctx context.Context) []string {
+	if s.robotsSitemapsSet() {
+		return s.RobotsSitemaps()
+	}
+	if l := s.listing(); l != nil {
+		if st, err := l.Settings(ctx); err == nil && strings.TrimSpace(st.IndexURL) != "" {
+			return []string{strings.TrimSpace(st.IndexURL)}
+		} else if err != nil {
+			log.Warn("golem15.seo: sitemap index address", "error", err)
+		}
+	}
+	return s.RobotsSitemaps()
+}
+
+// robotsSitemapsSet reports whether golem15.seo.robots_sitemaps is set to
+// a non-empty list.
+func (s *Service) robotsSitemapsSet() bool {
+	cfg := s.Config()
+	if cfg == nil {
+		return false
+	}
+	raw, ok := cfg.Lookup(RobotsSitemapsConfigKey)
+	if !ok || raw == nil {
+		return false
+	}
+	switch v := raw.(type) {
+	case []any:
+		return len(v) > 0
+	case []string:
+		return len(v) > 0
+	case string:
+		return strings.TrimSpace(v) != ""
+	}
+	return false
+}
+
 // RobotsSitemaps returns the configured sitemap addresses, absolute on
-// the site URL.
+// the site URL; /sitemap.xml when none is configured.
 func (s *Service) RobotsSitemaps() []string {
 	var list []string
 	if cfg := s.Config(); cfg != nil {
@@ -128,7 +169,7 @@ func (s *Service) ServedFile(ctx context.Context, production bool) (string, erro
 	if err != nil {
 		return "", err
 	}
-	body := ComposeRobots(row.Content, row.Bots, s.RobotsSitemaps())
+	body := ComposeRobots(row.Content, row.Bots, s.SitemapLines(ctx))
 	if s != nil {
 		s.robots.Store(&body)
 	}
@@ -164,7 +205,7 @@ type RobotsTest struct {
 // grobotstxt, Google's matcher (D-22, D-99): longest match, Allow wins a
 // tie, * and $.
 func (s *Service) TestRobots(content string, bots models.BotSwitches, agent, path string) RobotsTest {
-	file := ComposeRobots(content, bots, s.RobotsSitemaps())
+	file := ComposeRobots(content, bots, s.SitemapLines(context.Background()))
 	uri := AbsoluteURL(s.SiteURL(), path)
 	if s.SiteURL() == "" {
 		uri = "http://localhost" + path

classes/robots_write.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"strconv"
+	"strings"
 	"time"
 
 	"git.golem15.com/golem15/sm-seo-plugin/models"
@@ -42,14 +43,16 @@ func RobotsValues(r models.Robots) map[string]any {
 	for _, b := range Bots {
 		bots[b.Token] = r.Bots.Allowed(b.Token)
 	}
-	return map[string]any{"content": r.Content, "bots": bots}
+	return map[string]any{"content": r.Content, "bots": bots, "llms_enabled": r.LlmsEnabled, "llms_intro": r.LlmsIntro}
 }
 
 // RobotsWrite is one change of robots.txt: the editor's text when Content
 // is set, and the switches of the bots listed in Bots (by token).
 type RobotsWrite struct {
-	Content *string
-	Bots    map[string]bool
+	Content     *string
+	Bots        map[string]bool
+	LlmsEnabled *bool
+	LlmsIntro   *string
 }
 
 // SaveRobots writes robots.txt inside tx and records it in history
@@ -87,10 +90,16 @@ func (s *Service) SaveRobots(ctx context.Context, tx *gorm.DB, in RobotsWrite) (
 		bots[b.Token] = on
 	}
 	row.Bots = bots
+	if in.LlmsEnabled != nil {
+		row.LlmsEnabled = *in.LlmsEnabled
+	}
+	if in.LlmsIntro != nil {
+		row.LlmsIntro = strings.TrimSpace(*in.LlmsIntro)
+	}
 	now := time.Now().UTC()
 	row.UpdatedAt = &now
 	if err := tx.Model(&models.Robots{}).Where("id = ?", row.ID).Updates(map[string]any{
-		"content": row.Content, "bots": row.Bots, "updated_at": now,
+		"content": row.Content, "bots": row.Bots, "llms_enabled": row.LlmsEnabled, "llms_intro": row.LlmsIntro, "updated_at": now,
 	}).Error; err != nil {
 		return models.Robots{}, nil, err
 	}

classes/site.go

@@ -0,0 +1,139 @@
+package classes
+
+import (
+	"context"
+	"strings"
+
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/sm-sitemap-plugin/contract"
+)
+
+// SiteAnswer is the answer of GET /_seo/api/v1/site (P33-17): the facts
+// about the whole site the frontend layer needs, all public. SiteURL,
+// DefaultLocale, Locales and UnprefixedLocale are the resolver's; Links
+// the settings' new-tab switch; HTMLSitemap the sitemap plugin's HTML
+// sitemap settings; Routes each content type's layered URL pattern for
+// the layer's drift check; RootFiles the file names the layer serves at
+// the site root (verification files, the IndexNow key file); LLMS whether
+// /llms.txt is served.
+type SiteAnswer struct {
+	SiteURL          string          `json:"site_url"`
+	DefaultLocale    string          `json:"default_locale"`
+	Locales          []string        `json:"locales"`
+	UnprefixedLocale string          `json:"unprefixed_locale"`
+	Links            SiteLinks       `json:"links"`
+	HTMLSitemap      SiteHTMLSitemap `json:"html_sitemap"`
+	Routes           []SiteRoute     `json:"routes"`
+	RootFiles        []string        `json:"root_files"`
+	LLMS             bool            `json:"llms"`
+}
+
+// SiteLinks is the site answer's links block.
+type SiteLinks struct {
+	NewTab bool `json:"new_tab"`
+}
+
+// SiteHTMLSitemap is the site answer's HTML sitemap block: whether the
+// sitemap plugin serves it, its path per locale and its page size.
+type SiteHTMLSitemap struct {
+	Enabled  bool              `json:"enabled"`
+	Paths    map[string]string `json:"paths"`
+	PageSize int               `json:"page_size"`
+}
+
+// SiteRoute is one content type's URL pattern.
+type SiteRoute struct {
+	Type    string `json:"type"`
+	Pattern string `json:"pattern"`
+}
+
+// listing returns the sitemap plugin's published contract.Listing, nil
+// when none is published.
+func (s *Service) listing() contract.Listing {
+	if s == nil || s.app == nil {
+		return nil
+	}
+	l, ok := s.app.Lookup[contract.Listing]()
+	if !ok {
+		return nil
+	}
+	return l
+}
+
+// Site builds the site answer. It names only public facts: no token, no
+// unpublished data (T-33-19).
+func (s *Service) Site(ctx context.Context) (SiteAnswer, error) {
+	out := SiteAnswer{
+		SiteURL:          s.SiteURL(),
+		UnprefixedLocale: s.UnprefixedLocale(),
+		HTMLSitemap:      SiteHTMLSitemap{Paths: map[string]string{}},
+		Routes:           []SiteRoute{},
+		RootFiles:        []string{},
+	}
+	db, err := s.DB(ctx)
+	if err != nil {
+		return out, err
+	}
+	settings, err := LoadSettings(db)
+	if err != nil {
+		return out, err
+	}
+	robots, err := LoadRobots(db)
+	if err != nil {
+		return out, err
+	}
+	out.DefaultLocale = s.DefaultLocale(ctx, db)
+	out.Locales = s.Locales(ctx, db)
+	out.Links.NewTab = settings.LinksNewTab
+	out.LLMS = robots.LlmsEnabled
+	if reg := s.Registry(); reg != nil {
+		for _, typ := range reg.ContentTypes() {
+			if typ.Key == PageTypeKey {
+				continue
+			}
+			out.Routes = append(out.Routes, SiteRoute{Type: typ.Key, Pattern: reg.Pattern(typ.Key, "")})
+		}
+	}
+	seen := map[string]bool{}
+	addFile := func(name string) {
+		name = strings.TrimSpace(name)
+		if name != "" && verificationName.MatchString(name) && !seen[name] {
+			seen[name] = true
+			out.RootFiles = append(out.RootFiles, name)
+		}
+	}
+	for _, f := range settings.Files() {
+		addFile(f.Filename)
+	}
+	if l := s.listing(); l != nil {
+		st, err := l.Settings(ctx)
+		if err != nil {
+			return out, err
+		}
+		out.HTMLSitemap.Enabled = st.HTMLEnabled
+		for locale, path := range st.HTMLPaths {
+			if strings.TrimSpace(path) != "" {
+				out.HTMLSitemap.Paths[locale] = path
+			}
+		}
+		out.HTMLSitemap.PageSize = st.HTMLPageSize
+		addFile(st.KeyFile)
+	}
+	return out, nil
+}
+
+// SiteName is the site's name in locale: the settings' site name, else
+// the application name.
+func (s *Service) SiteName(ctx context.Context, settings models.Settings, locale string) string {
+	db, err := s.DB(ctx)
+	name := settings.SiteName
+	if err == nil {
+		name = s.translatedSetting(ctx, db, &settings, "site_name", settings.SiteName, locale)
+	}
+	if strings.TrimSpace(name) == "" {
+		if cfg := s.Config(); cfg != nil {
+			name = cfg.String("app.name")
+		}
+	}
+	return strings.TrimSpace(name)
+}

classes/undo.go

@@ -471,6 +471,13 @@ func (s *Service) restore(ctx context.Context, tx *gorm.DB, rev models.Revision,
 				w.Bots[token] = b
 			}
 		}
+		if v, ok := rev.Before["llms_enabled"].(bool); ok {
+			w.LlmsEnabled = &v
+		}
+		if v, ok := rev.Before["llms_intro"]; ok {
+			intro := text(v)
+			w.LlmsIntro = &intro
+		}
 		_, _, err := s.SaveRobots(ctx, tx, w)
 		return err
 	case models.SubjectSettings:

config/config.yaml

@@ -21,8 +21,11 @@ notfound_token: ""
 notfound_max_rows: 10000
 #
 # robots_sitemaps lists the sitemap addresses robots.txt names, one
-# "Sitemap:" line each, made absolute on the site URL.
-robots_sitemaps: ["/sitemap.xml"]
+# "Sitemap:" line each, made absolute on the site URL. Unset, robots.txt
+# names the sitemap plugin's index (its site address plus /sitemap.xml);
+# the per-type files are never listed, the index names them. Set it only
+# to name other sitemaps:
+# robots_sitemaps: ["/sitemap.xml"]
 #
 # route_report_token is the shared secret the frontend server sends as
 # "Authorization: Bearer <token>" with its route report to

controllers/api/llms.go

@@ -0,0 +1,34 @@
+package api
+
+import (
+	"net/http"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+)
+
+// LLMS serves GET /llms.txt at the site root (D-25, UI-SPEC C25): while
+// llms.txt is on (the robots.txt screen's AI bots tab) and in production,
+// the site's indexable pages from the sitemaps as text/plain; otherwise
+// 404.
+func LLMS(service func() *classes.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		svc := service()
+		if svc == nil {
+			http.Error(w, "SEO is not available", http.StatusServiceUnavailable)
+			return
+		}
+		body, ok, err := svc.LLMSText(r.Context())
+		if err != nil {
+			log.Error("golem15.seo: llms.txt", "error", err)
+			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+			return
+		}
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		_, _ = w.Write([]byte(body))
+	}
+}

controllers/api/robots.go

@@ -50,6 +50,9 @@ func Robots(service func() *classes.Service) http.HandlerFunc {
 type UpdateRobotsInput struct {
 	Content *string         `json:"content,omitempty" jsonschema:"the robots.txt rules; Sitemap lines and the AI bot rules are added when the file is served"`
 	Bots    map[string]bool `json:"bots,omitempty" jsonschema:"AI bot switches by user-agent token, for example {\"GPTBot\": false}; a bot switched off is disallowed"`
+	// LlmsEnabled and LlmsIntro are the llms.txt switch and introduction.
+	LlmsEnabled *bool   `json:"llms_enabled,omitempty" jsonschema:"serve /llms.txt: the site's indexable pages from the sitemaps, for AI tools (off by default)"`
+	LlmsIntro   *string `json:"llms_intro,omitempty" jsonschema:"a short description of the site at the top of llms.txt; empty uses the default meta description"`
 }
 
 // TestRobotsInput is the query of GET /_seo/api/v1/robots/test.
@@ -66,12 +69,14 @@ func robotsJSON(ctx context.Context, svc *classes.Service, row models.Robots) (m
 	for _, b := range classes.Bots {
 		bots = append(bots, map[string]any{"token": b.Token, "operator": b.Operator, "kind": b.Kind, "allowed": row.Bots.Allowed(b.Token)})
 	}
-	production := classes.ComposeRobots(row.Content, row.Bots, svc.RobotsSitemaps())
+	production := classes.ComposeRobots(row.Content, row.Bots, svc.SitemapLines(ctx))
 	return map[string]any{
-		"content":    row.Content,
-		"bots":       bots,
-		"production": production,
-		"staging":    svc.Staging(),
+		"content":      row.Content,
+		"bots":         bots,
+		"llms_enabled": row.LlmsEnabled,
+		"llms_intro":   row.LlmsIntro,
+		"production":   production,
+		"staging":      svc.Staging(),
 	}, nil
 }
 
@@ -107,7 +112,7 @@ func UpdateRobots(service func() *classes.Service) http.HandlerFunc {
 		var blocked []string
 		err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
 			var err error
-			row, blocked, err = svc.SaveRobots(ctx, tx, classes.RobotsWrite{Content: in.Content, Bots: in.Bots})
+			row, blocked, err = svc.SaveRobots(ctx, tx, classes.RobotsWrite{Content: in.Content, Bots: in.Bots, LlmsEnabled: in.LlmsEnabled, LlmsIntro: in.LlmsIntro})
 			return err
 		})
 		if err != nil {

controllers/api/site.go

@@ -0,0 +1,29 @@
+package api
+
+import (
+	"net/http"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+	"git.golem15.com/golem15/summercms/modules/wire"
+)
+
+// Site serves GET /_seo/api/v1/site (P33-17): the site facts the frontend
+// layer needs (site URL, locales, the new-tab switch, the HTML sitemap
+// settings, the content types' URL patterns, the root file names and
+// whether llms.txt is served). Every value is public.
+func Site(service func() *classes.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		svc := service()
+		if svc == nil {
+			wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "SEO is not available"})
+			return
+		}
+		out, err := svc.Site(r.Context())
+		if err != nil {
+			log.Error("golem15.seo: site answer", "error", err)
+			wire.WriteJSON(w, http.StatusInternalServerError, map[string]string{"error": errInternal})
+			return
+		}
+		wire.WriteJSON(w, http.StatusOK, out)
+	}
+}

extensions_robots.go

@@ -78,6 +78,30 @@ func robotsDocument() ([]byte, error) {
 			{Key: "tab", Value: TabRobotsBots},
 		}})
 	}
+	// llms.txt (UI-SPEC C25) after the bot switches.
+	fields = append(fields,
+		yaml.MapItem{Key: "llms_enabled", Value: yaml.MapSlice{
+			{Key: "label", Value: "golem15.seo::lang.robots.llms_enabled"},
+			{Key: "comment", Value: "golem15.seo::lang.robots.llms_enabled_comment"},
+			{Key: "type", Value: "switch"},
+			{Key: "default", Value: false},
+			{Key: "span", Value: "full"},
+			{Key: "tab", Value: TabRobotsBots},
+		}},
+		yaml.MapItem{Key: "llms_intro", Value: yaml.MapSlice{
+			{Key: "label", Value: "golem15.seo::lang.robots.llms_intro"},
+			{Key: "comment", Value: "golem15.seo::lang.robots.llms_intro_comment"},
+			{Key: "type", Value: "textarea"},
+			{Key: "size", Value: "small"},
+			{Key: "span", Value: "full"},
+			{Key: "tab", Value: TabRobotsBots},
+			{Key: "trigger", Value: yaml.MapSlice{
+				{Key: "action", Value: "hide"},
+				{Key: "field", Value: "llms_enabled"},
+				{Key: "condition", Value: "unchecked"},
+			}},
+		}},
+	)
 	return yaml.Marshal(yaml.MapSlice{
 		{Key: "tabs", Value: []tabSpec{{Name: TabRobotsBots, After: TabRobotsFile}}},
 		{Key: "fields", Value: fields},
@@ -103,7 +127,7 @@ func (p *Plugin) robotsForm() *pact.AdminFormExtension {
 			if !ok || row == nil {
 				return nil, nil
 			}
-			out := map[string]any{"content": row.Content}
+			out := map[string]any{"content": row.Content, "llms_enabled": row.LlmsEnabled, "llms_intro": row.LlmsIntro}
 			for _, b := range classes.Bots {
 				out[classes.BotField(b.Token)] = row.Bots.Allowed(b.Token)
 			}
@@ -131,6 +155,14 @@ func (p *Plugin) robotsForm() *pact.AdminFormExtension {
 					w.Bots[b.Token] = truthy(raw)
 				}
 			}
+			if raw, ok := in.Values["llms_enabled"]; ok {
+				on := truthy(raw)
+				w.LlmsEnabled = &on
+			}
+			if raw, ok := in.Values["llms_intro"]; ok {
+				text, _ := raw.(string)
+				w.LlmsIntro = &text
+			}
 			_, blocked, err := svc.SaveRobots(ctx, in.DB, w)
 			if err != nil || len(blocked) == 0 {
 				return pact.AdminFormSaveResult{}, err

lang/en/lang.yaml

@@ -37,6 +37,10 @@ field:
     og_description: Social description
     og_image: Social image
 robots:
+    llms_enabled: Serve llms.txt
+    llms_enabled_comment: Lists the site's indexable pages from the sitemaps at /llms.txt, for AI tools that read it.
+    llms_intro: Introduction
+    llms_intro_comment: "A short description of the site at the top of llms.txt. Empty: the default meta description."
     invalid: The robots value is not valid. Use index or noindex, follow or nofollow and the advanced directives, separated by commas.
     inherit: Inherit
     site_default: Site default

lang/pl/lang.yaml

@@ -37,6 +37,10 @@ field:
     og_description: Opis w mediach społecznościowych
     og_image: Obraz w mediach społecznościowych
 robots:
+    llms_enabled: Udostępniaj llms.txt
+    llms_enabled_comment: Wypisuje indeksowalne strony witryny z map witryny pod /llms.txt, dla narzędzi AI, które go czytają.
+    llms_intro: Wstęp
+    llms_intro_comment: "Krótki opis witryny na początku llms.txt. Puste: domyślny opis meta."
     invalid: Wartość robots jest nieprawidłowa. Użyj index lub noindex, follow lub nofollow oraz dyrektyw zaawansowanych, rozdzielonych przecinkami.
     inherit: Dziedzicz
     site_default: Domyślne

models/robots.go

@@ -12,11 +12,15 @@ import (
 // file adds a Disallow group for every switched-off bot and the Sitemap
 // lines; outside production it is replaced by the non-production lock.
 type Robots struct {
-	ID        uint        `gorm:"column:id;primaryKey"`
-	Content   string      `gorm:"column:content"`
-	Bots      BotSwitches `gorm:"column:bots"`
-	CreatedAt *time.Time  `gorm:"column:created_at"`
-	UpdatedAt *time.Time  `gorm:"column:updated_at"`
+	ID      uint        `gorm:"column:id;primaryKey"`
+	Content string      `gorm:"column:content"`
+	Bots    BotSwitches `gorm:"column:bots"`
+	// LlmsEnabled serves /llms.txt (D-25, off by default); LlmsIntro is
+	// its introduction, the default meta description when empty.
+	LlmsEnabled bool       `gorm:"column:llms_enabled"`
+	LlmsIntro   string     `gorm:"column:llms_intro"`
+	CreatedAt   *time.Time `gorm:"column:created_at"`
+	UpdatedAt   *time.Time `gorm:"column:updated_at"`
 }
 
 func (Robots) TableName() string { return "golem15_seo_robots" }

registry_test.go

@@ -384,3 +384,125 @@ func TestSettingsLinks(t *testing.T) {
 		t.Fatalf("outbound = %s", raw)
 	}
 }
+
+// TestSiteEndpoint covers GET /_seo/api/v1/site (P33-17, T-33-19): every
+// key, the verification and IndexNow root files, each content type's
+// pattern (not the registry's), the HTML sitemap settings, noindex.
+func TestSiteEndpoint(t *testing.T) {
+	env := bootEnv(t, "SUMMER_GOLEM15__SEO__UNPREFIXED_LOCALE=en")
+	token := env.admin(t, true)
+	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{
+		"verify_files":  []map[string]any{{"filename": "google0123.html", "body": "google-site-verification: google0123.html"}},
+		"links_new_tab": false,
+	})
+	key := "0123456789abcdef0123456789abcdef"
+	if err := env.db.Exec(`UPDATE golem15_sitemap_settings SET indexnow_enabled = TRUE, indexnow_key = ?, html_enabled = TRUE WHERE id = 1`, key).Error; err != nil {
+		t.Fatal(err)
+	}
+
+	rec := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/site", "", nil)
+	if rec.Header().Get("X-Robots-Tag") != "noindex" {
+		t.Fatalf("site headers = %v, want X-Robots-Tag noindex", rec.Header())
+	}
+	var site map[string]any
+	if err := json.Unmarshal(rec.Body.Bytes(), &site); err != nil {
+		t.Fatal(err)
+	}
+	for _, k := range []string{"site_url", "default_locale", "locales", "unprefixed_locale", "links", "html_sitemap", "routes", "root_files", "llms"} {
+		if _, ok := site[k]; !ok {
+			t.Fatalf("site answer lacks %s: %s", k, rec.Body.String())
+		}
+	}
+	if site["site_url"] != siteURL || site["default_locale"] != "en" || site["unprefixed_locale"] != "en" || site["llms"] != false {
+		t.Fatalf("site answer = %s", rec.Body.String())
+	}
+	body := rec.Body.String()
+	for _, want := range []string{
+		`"locales":["en","pl"]`,
+		`"links":{"new_tab":false}`,
+		`"root_files":["google0123.html","` + key + `.txt"]`,
+		`{"type":"acme-page","pattern":"/pages/:slug"}`,
+		`{"type":"acme-tag","pattern":"/pages/tag/:slug"}`,
+		`"html_sitemap":{"enabled":true,"paths":{`,
+	} {
+		if !jsonContains(body, want) {
+			t.Fatalf("site answer lacks %s: %s", want, body)
+		}
+	}
+	if strings.Contains(body, "seo-page") || strings.Contains(body, "token") {
+		t.Fatalf("site answer names the registry type or a token: %s", body)
+	}
+}
+
+// TestLLMSTxt covers /llms.txt (D-25, UI-SPEC C25, T-33-20): off by default
+// (404), on in production it lists the indexable pages from the sitemaps
+// per group, headed by the site name and the introduction; excluded
+// addresses never appear; staging answers 404.
+func TestLLMSTxt(t *testing.T) {
+	env := bootEnv(t)
+	token := env.admin(t, true)
+	env.expect(t, http.StatusNotFound, http.MethodGet, "/llms.txt", "", nil)
+
+	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"site_name": "Acme", "default_description": "Acme makes things."})
+	env.createPage(t, token, map[string]any{"title": "About [us]", "slug": "about", "published": true})
+	hidden := env.createPage(t, token, map[string]any{"title": "Hidden", "slug": "hidden", "published": true})
+	env.createPage(t, token, map[string]any{"title": "Draft", "slug": "draft", "published": false})
+	env.expect(t, http.StatusOK, http.MethodPut, adminAPI("/acme/pages/pages/"+hidden), token, map[string]any{
+		"title": "Hidden", "slug": "hidden", "published": true, "seo_robots": map[string]any{"en": "noindex"},
+	})
+	env.createRegistryPage(t, token, map[string]any{"url": map[string]any{"en": "/kontakt"}, "title": map[string]any{"en": "Contact"}})
+	env.rebuildSitemap(t)
+
+	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/robots", token, map[string]any{"llms_enabled": true})
+	rec := env.expect(t, http.StatusOK, http.MethodGet, "/llms.txt", "", nil)
+	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
+		t.Fatalf("content type = %q", ct)
+	}
+	body := rec.Body.String()
+	for _, want := range []string{
+		"# Acme\n\n> Acme makes things.\n",
+		"- [About \\[us\\]](https://app.example.test/pages/about)\n",
+		"## Page\n\n",
+		"- [Contact](https://app.example.test/kontakt)\n",
+	} {
+		if !strings.Contains(body, want) {
+			t.Fatalf("llms.txt lacks %q:\n%s", want, body)
+		}
+	}
+	if strings.Contains(body, "/pages/hidden") || strings.Contains(body, "/pages/draft") || strings.Contains(body, "/pl/") {
+		t.Fatalf("llms.txt lists a left-out or other-locale address:\n%s", body)
+	}
+
+	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/robots", token, map[string]any{"llms_intro": "Things, made well."})
+	if body := env.expect(t, http.StatusOK, http.MethodGet, "/llms.txt", "", nil).Body.String(); !strings.Contains(body, "\n> Things, made well.\n") {
+		t.Fatalf("llms.txt intro:\n%s", body)
+	}
+	show := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/robots", token, nil).Body.String()
+	if !jsonContains(show, `"llms_enabled":true`) || !jsonContains(show, `"llms_intro":"Things, made well."`) {
+		t.Fatalf("robots answer = %s", show)
+	}
+	if site := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/site", "", nil).Body.String(); !jsonContains(site, `"llms":true`) {
+		t.Fatalf("site answer = %s", site)
+	}
+
+	staging := bootEnv(t, "SUMMER_ENV=staging")
+	stoken := staging.admin(t, true)
+	staging.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/robots", stoken, map[string]any{"llms_enabled": true})
+	staging.expect(t, http.StatusNotFound, http.MethodGet, "/llms.txt", "", nil)
+}
+
+// TestRobotsSitemapLine covers robots.txt's Sitemap line (D-22): without
+// golem15.seo.robots_sitemaps it names the sitemap plugin's index only;
+// with it, the configured lines.
+func TestRobotsSitemapLine(t *testing.T) {
+	env := bootEnv(t)
+	body := env.expect(t, http.StatusOK, http.MethodGet, "/robots.txt", "", nil).Body.String()
+	if strings.Count(body, "Sitemap:") != 1 || !strings.Contains(body, "Sitemap: https://app.example.test/sitemap.xml\n") {
+		t.Fatalf("robots.txt without robots_sitemaps:\n%s", body)
+	}
+	custom := bootEnv(t, "SUMMER_GOLEM15__SEO__ROBOTS_SITEMAPS=/a.xml,/b.xml")
+	body = custom.expect(t, http.StatusOK, http.MethodGet, "/robots.txt", "", nil).Body.String()
+	if !strings.Contains(body, "Sitemap: "+siteURL+"/a.xml\nSitemap: "+siteURL+"/b.xml\n") || strings.Contains(body, "app.example.test") {
+		t.Fatalf("robots.txt with robots_sitemaps:\n%s", body)
+	}
+}

robots_test.go

@@ -25,6 +25,10 @@ func (e *testEnv) robotsTxt(t *testing.T) *httptest.ResponseRecorder {
 	return rec
 }
 
+// sitemapIndexURL is the sitemap plugin's index robots.txt names: the
+// fixture sets no sitemap site address, so it is on app.url.
+const sitemapIndexURL = "https://app.example.test/sitemap.xml"
+
 // TestRobotsServed checks GET /robots.txt (D-22, D-25, P-23, T-32-19,
 // T-32-20): the stored rules, a Disallow group per switched-off AI bot and
 // the Sitemap line in production; the non-production lock whatever is
@@ -32,7 +36,7 @@ func (e *testEnv) robotsTxt(t *testing.T) *httptest.ResponseRecorder {
 // store cannot be read.
 func TestRobotsServed(t *testing.T) {
 	env := bootEnv(t)
-	if body := env.robotsTxt(t).Body.String(); body != "Sitemap: "+siteURL+"/sitemap.xml\n" {
+	if body := env.robotsTxt(t).Body.String(); body != "Sitemap: "+sitemapIndexURL+"\n" {
 		t.Fatalf("fresh robots.txt = %q, want only the Sitemap line", body)
 	}
 	if err := env.db.Exec(`UPDATE golem15_seo_robots SET content = ?, bots = ? WHERE id = 1`,
@@ -40,7 +44,7 @@ func TestRobotsServed(t *testing.T) {
 		t.Fatal(err)
 	}
 	rec := env.robotsTxt(t)
-	want := "User-agent: *\nDisallow: /admin\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nSitemap: " + siteURL + "/sitemap.xml\n"
+	want := "User-agent: *\nDisallow: /admin\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nSitemap: " + sitemapIndexURL + "\n"
 	if body := rec.Body.String(); body != want {
 		t.Fatalf("robots.txt = %q, want %q", body, want)
 	}
@@ -128,8 +132,8 @@ func TestRobotsScreen(t *testing.T) {
 	for _, f := range fields {
 		names = append(names, f.Name)
 	}
-	if strings.Join(names[:4], ",") != "staging,content,tester,served" || len(names) != 4+11 || names[4] != "bot_oai_searchbot" || names[8] != "bot_claude_user" {
-		t.Fatalf("robots form fields = %v, want staging, content, tester, served, then eleven bot switches", names)
+	if strings.Join(names[:4], ",") != "staging,content,tester,served" || len(names) != 4+11+2 || names[4] != "bot_oai_searchbot" || names[8] != "bot_claude_user" || strings.Join(names[15:], ",") != "llms_enabled,llms_intro" {
+		t.Fatalf("robots form fields = %v, want staging, content, tester, served, eleven bot switches, then the llms.txt switch and introduction", names)
 	}
 
 	env.expect(t, http.StatusCreated, http.MethodPost, adminAPI("/golem15/seo/rules"), token, map[string]any{"url": "/private/x", "match": "exact", "robots": "noindex,follow"})
@@ -163,7 +167,7 @@ func TestRobotsScreen(t *testing.T) {
 		t.Fatalf("show = %s, want the switches", show)
 	}
 	served := env.expect(t, http.StatusOK, http.MethodGet, api+"/partials/robots_served?id=1", token, nil).Body.String()
-	if !strings.Contains(served, "User-agent: GPTBot") || !strings.Contains(served, "Sitemap: "+siteURL+"/sitemap.xml") {
+	if !strings.Contains(served, "User-agent: GPTBot") || !strings.Contains(served, "Sitemap: "+sitemapIndexURL) {
 		t.Fatalf("served partial = %s, want the composed file", served)
 	}
 

routes.go

@@ -22,7 +22,7 @@ const (
 	BucketAPI = "seo-api"
 )
 
-// Routes mounts GET /robots.txt at the site root, the public /_seo/api/v1
+// Routes mounts GET /robots.txt and GET /llms.txt at the site root, the public /_seo/api/v1
 // endpoints and the management API. The public ones carry no backend
 // middleware: crawlers and the frontend server call them anonymously (the
 // 404 report with its own token). The management routes (bucket seo-api)
@@ -31,8 +31,10 @@ const (
 // tool (mcp.go). Every /_seo/api/v1 answer carries X-Robots-Tag: noindex.
 func (p *Plugin) Routes(r pact.Router) error {
 	r.Get("/robots.txt", apicontrollers.Robots(p.Service), "throttle:"+BucketRobots)
+	r.Get("/llms.txt", apicontrollers.LLMS(p.Service), "throttle:"+BucketRobots)
 	r.Group("/_seo/api/v1", nil, func(g pact.Router) {
 		g.Get("/resolve", noindex(apicontrollers.Resolve(p.Service)), "throttle:"+BucketResolve)
+		g.Get("/site", noindex(apicontrollers.Site(p.Service)), "throttle:"+BucketResolve)
 		g.Post("/notfound", noindex(apicontrollers.NotFound(p.Service)), "throttle:"+BucketNotFound)
 		// The frontend server's route report (D-33), token-authenticated
 		// like the 404 report.