feat: site endpoint, llms.txt and the sitemap index in robots.txt

- GET /_seo/api/v1/site (public, noindex): site_url, default_locale,
  locales, unprefixed_locale, links.new_tab, html_sitemap (from the
  sitemap plugin's Listing), routes of every content type, root_files
  (verification files and the IndexNow key file) and llms
- GET /llms.txt: with llms_enabled (robots.txt screen, AI bots tab, off by
  default) and in production, the indexable pages from the sitemaps per
  group under the site name and llms_intro (else the default
  description); otherwise 404; seo_update_robots takes both fields
- robots.txt names the sitemap plugin's index unless
  golem15.seo.robots_sitemaps is set; the config default is now a comment

classes/llms.go

@@ -0,0 +1,177 @@
+package classes
+
+import (
+	"context"
+	"sort"
+	"strings"
+
+	"git.golem15.com/golem15/sm-sitemap-plugin/contract"
+)
+
+// MaxLLMSEntries caps the links llms.txt lists.
+const MaxLLMSEntries = 10000
+
+// llmsGroup is one section of llms.txt.
+type llmsGroup struct {
+	title   string
+	order   int
+	key     string
+	entries []contract.Entry
+}
+
+// LLMSText builds /llms.txt (D-25, UI-SPEC C25): "# <site name>", the
+// introduction (llms_intro, else the default meta description) as a quote,
+// then one "## <group title>" section per sitemap group with listed
+// addresses in the default locale, each a "- [title](absolute url)" line.
+// It reads the sitemaps through contract.Listing, so an address the
+// sitemaps leave out (noindex, unpublished, canonicalised) never appears
+// (T-33-20). The groups follow the content types' order, as in the HTML
+// sitemap. ok is false while llms.txt is off and outside production.
+func (s *Service) LLMSText(ctx context.Context) (string, bool, error) {
+	if s.Staging() {
+		return "", false, nil
+	}
+	db, err := s.DB(ctx)
+	if err != nil {
+		return "", false, err
+	}
+	robots, err := LoadRobots(db)
+	if err != nil {
+		return "", false, err
+	}
+	if !robots.LlmsEnabled {
+		return "", false, nil
+	}
+	settings, err := LoadSettings(db)
+	if err != nil {
+		return "", false, err
+	}
+	locale := s.DefaultLocale(ctx, db)
+	var b strings.Builder
+	name := s.SiteName(ctx, settings, locale)
+	if name == "" {
+		name = s.SiteURL()
+	}
+	b.WriteString("# " + llmsLine(name) + "\n")
+	intro := strings.TrimSpace(robots.LlmsIntro)
+	if intro == "" {
+		intro = strings.TrimSpace(s.translatedSetting(ctx, db, &settings, "default_description", settings.DefaultDescription, locale))
+	}
+	if intro != "" {
+		b.WriteString("\n> " + llmsLine(intro) + "\n")
+	}
+	groups, err := s.llmsGroups(ctx, locale)
+	if err != nil {
+		return "", false, err
+	}
+	for _, g := range groups {
+		b.WriteString("\n## " + llmsLine(g.title) + "\n\n")
+		for _, e := range g.entries {
+			title := strings.TrimSpace(e.Title)
+			if title == "" {
+				title = e.URL
+			}
+			b.WriteString("- [" + llmsText(title) + "](" + llmsURL(e.URL) + ")\n")
+		}
+	}
+	return b.String(), true, nil
+}
+
+// llmsGroups reads the listed addresses of every sitemap group in locale,
+// ordered by the content type of their first address (registration order),
+// then by key.
+func (s *Service) llmsGroups(ctx context.Context, locale string) ([]llmsGroup, error) {
+	l := s.listing()
+	if l == nil {
+		return nil, nil
+	}
+	st, err := l.Settings(ctx)
+	if err != nil {
+		return nil, err
+	}
+	order := map[string]int{}
+	if reg := s.Registry(); reg != nil {
+		for i, typ := range reg.ContentTypes() {
+			order[typ.Key] = i
+		}
+	}
+	total := 0
+	var out []llmsGroup
+	for _, g := range st.Groups {
+		if g.Count == 0 || total >= MaxLLMSEntries {
+			continue
+		}
+		grp := llmsGroup{key: g.Key, order: len(order) + 1, title: groupTitle(g, locale, st.DefaultLocale)}
+		after := ""
+		for total < MaxLLMSEntries {
+			page, err := l.Entries(ctx, contract.EntriesQuery{Group: g.Key, Locale: locale, After: after, Limit: 1000})
+			if err != nil {
+				return nil, err
+			}
+			for _, e := range page.Entries {
+				if total >= MaxLLMSEntries {
+					break
+				}
+				if len(grp.entries) == 0 {
+					if i, ok := order[e.TypeKey]; ok {
+						grp.order = i
+					}
+				}
+				grp.entries = append(grp.entries, e)
+				total++
+			}
+			if page.Next == "" || page.Next == after {
+				break
+			}
+			after = page.Next
+		}
+		if len(grp.entries) > 0 {
+			out = append(out, grp)
+		}
+	}
+	sort.SliceStable(out, func(i, j int) bool {
+		if out[i].order != out[j].order {
+			return out[i].order < out[j].order
+		}
+		return out[i].key < out[j].key
+	})
+	return out, nil
+}
+
+// groupTitle is a group's heading in locale: its title there, else in the
+// sitemap's default locale, else any, else its key.
+func groupTitle(g contract.GroupInfo, locale, def string) string {
+	for _, l := range []string{locale, def} {
+		if t := strings.TrimSpace(g.Titles[l]); t != "" {
+			return t
+		}
+	}
+	keys := make([]string, 0, len(g.Titles))
+	for k := range g.Titles {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		if t := strings.TrimSpace(g.Titles[k]); t != "" {
+			return t
+		}
+	}
+	return g.Key
+}
+
+// llmsLine collapses text to one line.
+func llmsLine(text string) string {
+	return strings.Join(strings.Fields(text), " ")
+}
+
+// llmsText is a link text: one line with the Markdown brackets and
+// backslashes escaped.
+func llmsText(text string) string {
+	return strings.NewReplacer(`\`, `\\`, `[`, `\[`, `]`, `\]`).Replace(llmsLine(text))
+}
+
+// llmsURL is a link target: spaces and parentheses percent-encoded, so
+// the Markdown link stays whole.
+func llmsURL(u string) string {
+	return strings.NewReplacer(" ", "%20", "(", "%28", ")", "%29", "<", "%3C", ">", "%3E").Replace(strings.TrimSpace(u))
+}

classes/robots.go

@@ -11,8 +11,8 @@ import (
 )
 
 // RobotsSitemapsConfigKey lists the sitemap addresses robots.txt names,
-// one Sitemap line each, absolute on the site URL (P-23); default
-// ["/sitemap.xml"].
+// one Sitemap line each, absolute on the site URL (P-23). Unset, robots.txt
+// names the sitemap plugin's index (D-22, SitemapLines).
 const RobotsSitemapsConfigKey = "golem15.seo.robots_sitemaps"
 
 // MaxRobotsBytes is the largest robots.txt the editor stores: Google
@@ -29,8 +29,49 @@ const (
 	FallbackRobots = "User-agent: *\nDisallow:\n"
 )
 
+// SitemapLines are the addresses of robots.txt's Sitemap lines (D-22):
+// golem15.seo.robots_sitemaps when it is set, else the sitemap index of
+// the sitemap plugin's contract.Listing (its IndexURL), else /sitemap.xml
+// on the site URL. The per-type files are never listed: the index names
+// them.
+func (s *Service) SitemapLines(ctx context.Context) []string {
+	if s.robotsSitemapsSet() {
+		return s.RobotsSitemaps()
+	}
+	if l := s.listing(); l != nil {
+		if st, err := l.Settings(ctx); err == nil && strings.TrimSpace(st.IndexURL) != "" {
+			return []string{strings.TrimSpace(st.IndexURL)}
+		} else if err != nil {
+			log.Warn("golem15.seo: sitemap index address", "error", err)
+		}
+	}
+	return s.RobotsSitemaps()
+}
+
+// robotsSitemapsSet reports whether golem15.seo.robots_sitemaps is set to
+// a non-empty list.
+func (s *Service) robotsSitemapsSet() bool {
+	cfg := s.Config()
+	if cfg == nil {
+		return false
+	}
+	raw, ok := cfg.Lookup(RobotsSitemapsConfigKey)
+	if !ok || raw == nil {
+		return false
+	}
+	switch v := raw.(type) {
+	case []any:
+		return len(v) > 0
+	case []string:
+		return len(v) > 0
+	case string:
+		return strings.TrimSpace(v) != ""
+	}
+	return false
+}
+
 // RobotsSitemaps returns the configured sitemap addresses, absolute on
-// the site URL.
+// the site URL; /sitemap.xml when none is configured.
 func (s *Service) RobotsSitemaps() []string {
 	var list []string
 	if cfg := s.Config(); cfg != nil {
@@ -128,7 +169,7 @@ func (s *Service) ServedFile(ctx context.Context, production bool) (string, erro
 	if err != nil {
 		return "", err
 	}
-	body := ComposeRobots(row.Content, row.Bots, s.RobotsSitemaps())
+	body := ComposeRobots(row.Content, row.Bots, s.SitemapLines(ctx))
 	if s != nil {
 		s.robots.Store(&body)
 	}
@@ -164,7 +205,7 @@ type RobotsTest struct {
 // grobotstxt, Google's matcher (D-22, D-99): longest match, Allow wins a
 // tie, * and $.
 func (s *Service) TestRobots(content string, bots models.BotSwitches, agent, path string) RobotsTest {
-	file := ComposeRobots(content, bots, s.RobotsSitemaps())
+	file := ComposeRobots(content, bots, s.SitemapLines(context.Background()))
 	uri := AbsoluteURL(s.SiteURL(), path)
 	if s.SiteURL() == "" {
 		uri = "http://localhost" + path

classes/robots_write.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"strconv"
+	"strings"
 	"time"
 
 	"git.golem15.com/golem15/sm-seo-plugin/models"
@@ -42,14 +43,16 @@ func RobotsValues(r models.Robots) map[string]any {
 	for _, b := range Bots {
 		bots[b.Token] = r.Bots.Allowed(b.Token)
 	}
-	return map[string]any{"content": r.Content, "bots": bots}
+	return map[string]any{"content": r.Content, "bots": bots, "llms_enabled": r.LlmsEnabled, "llms_intro": r.LlmsIntro}
 }
 
 // RobotsWrite is one change of robots.txt: the editor's text when Content
 // is set, and the switches of the bots listed in Bots (by token).
 type RobotsWrite struct {
-	Content *string
-	Bots    map[string]bool
+	Content     *string
+	Bots        map[string]bool
+	LlmsEnabled *bool
+	LlmsIntro   *string
 }
 
 // SaveRobots writes robots.txt inside tx and records it in history
@@ -87,10 +90,16 @@ func (s *Service) SaveRobots(ctx context.Context, tx *gorm.DB, in RobotsWrite) (
 		bots[b.Token] = on
 	}
 	row.Bots = bots
+	if in.LlmsEnabled != nil {
+		row.LlmsEnabled = *in.LlmsEnabled
+	}
+	if in.LlmsIntro != nil {
+		row.LlmsIntro = strings.TrimSpace(*in.LlmsIntro)
+	}
 	now := time.Now().UTC()
 	row.UpdatedAt = &now
 	if err := tx.Model(&models.Robots{}).Where("id = ?", row.ID).Updates(map[string]any{
-		"content": row.Content, "bots": row.Bots, "updated_at": now,
+		"content": row.Content, "bots": row.Bots, "llms_enabled": row.LlmsEnabled, "llms_intro": row.LlmsIntro, "updated_at": now,
 	}).Error; err != nil {
 		return models.Robots{}, nil, err
 	}

classes/site.go

@@ -0,0 +1,139 @@
+package classes
+
+import (
+	"context"
+	"strings"
+
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/sm-sitemap-plugin/contract"
+)
+
+// SiteAnswer is the answer of GET /_seo/api/v1/site (P33-17): the facts
+// about the whole site the frontend layer needs, all public. SiteURL,
+// DefaultLocale, Locales and UnprefixedLocale are the resolver's; Links
+// the settings' new-tab switch; HTMLSitemap the sitemap plugin's HTML
+// sitemap settings; Routes each content type's layered URL pattern for
+// the layer's drift check; RootFiles the file names the layer serves at
+// the site root (verification files, the IndexNow key file); LLMS whether
+// /llms.txt is served.
+type SiteAnswer struct {
+	SiteURL          string          `json:"site_url"`
+	DefaultLocale    string          `json:"default_locale"`
+	Locales          []string        `json:"locales"`
+	UnprefixedLocale string          `json:"unprefixed_locale"`
+	Links            SiteLinks       `json:"links"`
+	HTMLSitemap      SiteHTMLSitemap `json:"html_sitemap"`
+	Routes           []SiteRoute     `json:"routes"`
+	RootFiles        []string        `json:"root_files"`
+	LLMS             bool            `json:"llms"`
+}
+
+// SiteLinks is the site answer's links block.
+type SiteLinks struct {
+	NewTab bool `json:"new_tab"`
+}
+
+// SiteHTMLSitemap is the site answer's HTML sitemap block: whether the
+// sitemap plugin serves it, its path per locale and its page size.
+type SiteHTMLSitemap struct {
+	Enabled  bool              `json:"enabled"`
+	Paths    map[string]string `json:"paths"`
+	PageSize int               `json:"page_size"`
+}
+
+// SiteRoute is one content type's URL pattern.
+type SiteRoute struct {
+	Type    string `json:"type"`
+	Pattern string `json:"pattern"`
+}
+
+// listing returns the sitemap plugin's published contract.Listing, nil
+// when none is published.
+func (s *Service) listing() contract.Listing {
+	if s == nil || s.app == nil {
+		return nil
+	}
+	l, ok := s.app.Lookup[contract.Listing]()
+	if !ok {
+		return nil
+	}
+	return l
+}
+
+// Site builds the site answer. It names only public facts: no token, no
+// unpublished data (T-33-19).
+func (s *Service) Site(ctx context.Context) (SiteAnswer, error) {
+	out := SiteAnswer{
+		SiteURL:          s.SiteURL(),
+		UnprefixedLocale: s.UnprefixedLocale(),
+		HTMLSitemap:      SiteHTMLSitemap{Paths: map[string]string{}},
+		Routes:           []SiteRoute{},
+		RootFiles:        []string{},
+	}
+	db, err := s.DB(ctx)
+	if err != nil {
+		return out, err
+	}
+	settings, err := LoadSettings(db)
+	if err != nil {
+		return out, err
+	}
+	robots, err := LoadRobots(db)
+	if err != nil {
+		return out, err
+	}
+	out.DefaultLocale = s.DefaultLocale(ctx, db)
+	out.Locales = s.Locales(ctx, db)
+	out.Links.NewTab = settings.LinksNewTab
+	out.LLMS = robots.LlmsEnabled
+	if reg := s.Registry(); reg != nil {
+		for _, typ := range reg.ContentTypes() {
+			if typ.Key == PageTypeKey {
+				continue
+			}
+			out.Routes = append(out.Routes, SiteRoute{Type: typ.Key, Pattern: reg.Pattern(typ.Key, "")})
+		}
+	}
+	seen := map[string]bool{}
+	addFile := func(name string) {
+		name = strings.TrimSpace(name)
+		if name != "" && verificationName.MatchString(name) && !seen[name] {
+			seen[name] = true
+			out.RootFiles = append(out.RootFiles, name)
+		}
+	}
+	for _, f := range settings.Files() {
+		addFile(f.Filename)
+	}
+	if l := s.listing(); l != nil {
+		st, err := l.Settings(ctx)
+		if err != nil {
+			return out, err
+		}
+		out.HTMLSitemap.Enabled = st.HTMLEnabled
+		for locale, path := range st.HTMLPaths {
+			if strings.TrimSpace(path) != "" {
+				out.HTMLSitemap.Paths[locale] = path
+			}
+		}
+		out.HTMLSitemap.PageSize = st.HTMLPageSize
+		addFile(st.KeyFile)
+	}
+	return out, nil
+}
+
+// SiteName is the site's name in locale: the settings' site name, else
+// the application name.
+func (s *Service) SiteName(ctx context.Context, settings models.Settings, locale string) string {
+	db, err := s.DB(ctx)
+	name := settings.SiteName
+	if err == nil {
+		name = s.translatedSetting(ctx, db, &settings, "site_name", settings.SiteName, locale)
+	}
+	if strings.TrimSpace(name) == "" {
+		if cfg := s.Config(); cfg != nil {
+			name = cfg.String("app.name")
+		}
+	}
+	return strings.TrimSpace(name)
+}

classes/undo.go

@@ -471,6 +471,13 @@ func (s *Service) restore(ctx context.Context, tx *gorm.DB, rev models.Revision,
 				w.Bots[token] = b
 			}
 		}
+		if v, ok := rev.Before["llms_enabled"].(bool); ok {
+			w.LlmsEnabled = &v
+		}
+		if v, ok := rev.Before["llms_intro"]; ok {
+			intro := text(v)
+			w.LlmsIntro = &intro
+		}
 		_, _, err := s.SaveRobots(ctx, tx, w)
 		return err
 	case models.SubjectSettings: