package classes import ( "context" "sort" "strings" "git.golem15.com/golem15/sm-sitemap-plugin/contract" ) // MaxLLMSEntries caps the links llms.txt lists. const MaxLLMSEntries = 10000 // llmsGroup is one section of llms.txt. type llmsGroup struct { title string order int key string entries []contract.Entry } // LLMSText builds /llms.txt (D-25, UI-SPEC C25): "# ", the // introduction (llms_intro, else the default meta description) as a quote, // then one "## " section per sitemap group with listed // addresses in the default locale, each a "- [title](absolute url)" line. // It reads the sitemaps through contract.Listing, so an address the // sitemaps leave out (noindex, unpublished, canonicalised) never appears // (T-33-20). The groups follow the content types' order, as in the HTML // sitemap. ok is false while llms.txt is off and outside production. func (s *Service) LLMSText(ctx context.Context) (string, bool, error) { if s.Staging() { return "", false, nil } db, err := s.DB(ctx) if err != nil { return "", false, err } robots, err := LoadRobots(db) if err != nil { return "", false, err } if !robots.LlmsEnabled { return "", false, nil } settings, err := LoadSettings(db) if err != nil { return "", false, err } locale := s.DefaultLocale(ctx, db) var b strings.Builder name := s.SiteName(ctx, settings, locale) if name == "" { name = s.SiteURL() } b.WriteString("# " + llmsLine(name) + "\n") intro := strings.TrimSpace(robots.LlmsIntro) if intro == "" { intro = strings.TrimSpace(s.translatedSetting(ctx, db, &settings, "default_description", settings.DefaultDescription, locale)) } if intro != "" { b.WriteString("\n> " + llmsLine(intro) + "\n") } groups, err := s.llmsGroups(ctx, locale) if err != nil { return "", false, err } for _, g := range groups { b.WriteString("\n## " + llmsLine(g.title) + "\n\n") for _, e := range g.entries { title := strings.TrimSpace(e.Title) if title == "" { title = e.URL } b.WriteString("- [" + llmsText(title) + "](" + llmsURL(e.URL) + ")\n") } } return b.String(), true, nil } // llmsGroups reads the listed addresses of every sitemap group in locale, // ordered by the content type of their first address (registration order), // then by key. func (s *Service) llmsGroups(ctx context.Context, locale string) ([]llmsGroup, error) { l := s.listing() if l == nil { return nil, nil } st, err := l.Settings(ctx) if err != nil { return nil, err } order := map[string]int{} if reg := s.Registry(); reg != nil { for i, typ := range reg.ContentTypes() { order[typ.Key] = i } } total := 0 var out []llmsGroup for _, g := range st.Groups { if g.Count == 0 || total >= MaxLLMSEntries { continue } grp := llmsGroup{key: g.Key, order: len(order) + 1, title: groupTitle(g, locale, st.DefaultLocale)} after := "" for total < MaxLLMSEntries { page, err := l.Entries(ctx, contract.EntriesQuery{Group: g.Key, Locale: locale, After: after, Limit: 1000}) if err != nil { return nil, err } for _, e := range page.Entries { if total >= MaxLLMSEntries { break } if len(grp.entries) == 0 { if i, ok := order[e.TypeKey]; ok { grp.order = i } } grp.entries = append(grp.entries, e) total++ } if page.Next == "" || page.Next == after { break } after = page.Next } if len(grp.entries) > 0 { out = append(out, grp) } } sort.SliceStable(out, func(i, j int) bool { if out[i].order != out[j].order { return out[i].order < out[j].order } return out[i].key < out[j].key }) return out, nil } // groupTitle is a group's heading in locale: its title there, else in the // sitemap's default locale, else any, else its key. func groupTitle(g contract.GroupInfo, locale, def string) string { for _, l := range []string{locale, def} { if t := strings.TrimSpace(g.Titles[l]); t != "" { return t } } keys := make([]string, 0, len(g.Titles)) for k := range g.Titles { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { if t := strings.TrimSpace(g.Titles[k]); t != "" { return t } } return g.Key } // llmsLine collapses text to one line. func llmsLine(text string) string { return strings.Join(strings.Fields(text), " ") } // llmsText is a link text: one line with the Markdown brackets and // backslashes escaped. func llmsText(text string) string { return strings.NewReplacer(`\`, `\\`, `[`, `\[`, `]`, `\]`).Replace(llmsLine(text)) } // llmsURL is a link target: spaces and parentheses percent-encoded, so // the Markdown link stays whole. func llmsURL(u string) string { return strings.NewReplacer(" ", "%20", "(", "%28", ")", "%29", "<", "%3C", ">", "%3E").Replace(strings.TrimSpace(u)) }