sm-seo-plugin/classes/llms.go

package classes

import (
	"context"
	"sort"
	"strings"

	"git.golem15.com/golem15/sm-sitemap-plugin/contract"
)

// MaxLLMSEntries caps the links llms.txt lists.
const MaxLLMSEntries = 10000

// llmsGroup is one section of llms.txt.
type llmsGroup struct {
	title   string
	order   int
	key     string
	entries []contract.Entry
}

// LLMSText builds /llms.txt (D-25, UI-SPEC C25): "# <site name>", the
// introduction (llms_intro, else the default meta description) as a quote,
// then one "## <group title>" section per sitemap group with listed
// addresses in the default locale, each a "- [title](absolute url)" line.
// It reads the sitemaps through contract.Listing, so an address the
// sitemaps leave out (noindex, unpublished, canonicalised) never appears
// (T-33-20). The groups follow the content types' order, as in the HTML
// sitemap. ok is false while llms.txt is off and outside production.
func (s *Service) LLMSText(ctx context.Context) (string, bool, error) {
	if s.Staging() {
		return "", false, nil
	}
	db, err := s.DB(ctx)
	if err != nil {
		return "", false, err
	}
	robots, err := LoadRobots(db)
	if err != nil {
		return "", false, err
	}
	if !robots.LlmsEnabled {
		return "", false, nil
	}
	settings, err := LoadSettings(db)
	if err != nil {
		return "", false, err
	}
	locale := s.DefaultLocale(ctx, db)
	var b strings.Builder
	name := s.SiteName(ctx, settings, locale)
	if name == "" {
		name = s.SiteURL()
	}
	b.WriteString("# " + llmsLine(name) + "\n")
	intro := strings.TrimSpace(robots.LlmsIntro)
	if intro == "" {
		intro = strings.TrimSpace(s.translatedSetting(ctx, db, &settings, "default_description", settings.DefaultDescription, locale))
	}
	if intro != "" {
		b.WriteString("\n> " + llmsLine(intro) + "\n")
	}
	groups, err := s.llmsGroups(ctx, locale)
	if err != nil {
		return "", false, err
	}
	for _, g := range groups {
		b.WriteString("\n## " + llmsLine(g.title) + "\n\n")
		for _, e := range g.entries {
			title := strings.TrimSpace(e.Title)
			if title == "" {
				title = e.URL
			}
			b.WriteString("- [" + llmsText(title) + "](" + llmsURL(e.URL) + ")\n")
		}
	}
	return b.String(), true, nil
}

// llmsGroups reads the listed addresses of every sitemap group in locale,
// ordered by the content type of their first address (registration order),
// then by key.
func (s *Service) llmsGroups(ctx context.Context, locale string) ([]llmsGroup, error) {
	l := s.listing()
	if l == nil {
		return nil, nil
	}
	st, err := l.Settings(ctx)
	if err != nil {
		return nil, err
	}
	order := map[string]int{}
	if reg := s.Registry(); reg != nil {
		for i, typ := range reg.ContentTypes() {
			order[typ.Key] = i
		}
	}
	total := 0
	var out []llmsGroup
	for _, g := range st.Groups {
		if g.Count == 0 || total >= MaxLLMSEntries {
			continue
		}
		grp := llmsGroup{key: g.Key, order: len(order) + 1, title: groupTitle(g, locale, st.DefaultLocale)}
		after := ""
		for total < MaxLLMSEntries {
			page, err := l.Entries(ctx, contract.EntriesQuery{Group: g.Key, Locale: locale, After: after, Limit: 1000})
			if err != nil {
				return nil, err
			}
			for _, e := range page.Entries {
				if total >= MaxLLMSEntries {
					break
				}
				if len(grp.entries) == 0 {
					if i, ok := order[e.TypeKey]; ok {
						grp.order = i
					}
				}
				grp.entries = append(grp.entries, e)
				total++
			}
			if page.Next == "" || page.Next == after {
				break
			}
			after = page.Next
		}
		if len(grp.entries) > 0 {
			out = append(out, grp)
		}
	}
	sort.SliceStable(out, func(i, j int) bool {
		if out[i].order != out[j].order {
			return out[i].order < out[j].order
		}
		return out[i].key < out[j].key
	})
	return out, nil
}

// groupTitle is a group's heading in locale: its title there, else in the
// sitemap's default locale, else any, else its key.
func groupTitle(g contract.GroupInfo, locale, def string) string {
	for _, l := range []string{locale, def} {
		if t := strings.TrimSpace(g.Titles[l]); t != "" {
			return t
		}
	}
	keys := make([]string, 0, len(g.Titles))
	for k := range g.Titles {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		if t := strings.TrimSpace(g.Titles[k]); t != "" {
			return t
		}
	}
	return g.Key
}

// llmsLine collapses text to one line.
func llmsLine(text string) string {
	return strings.Join(strings.Fields(text), " ")
}

// llmsText is a link text: one line with the Markdown brackets and
// backslashes escaped.
func llmsText(text string) string {
	return strings.NewReplacer(`\`, `\\`, `[`, `\[`, `]`, `\]`).Replace(llmsLine(text))
}

// llmsURL is a link target: spaces and parentheses percent-encoded, so
// the Markdown link stays whole.
func llmsURL(u string) string {
	return strings.NewReplacer(" ", "%20", "(", "%28", ")", "%29", "<", "%3C", ">", "%3E").Replace(strings.TrimSpace(u))
}