package classes import ( "context" "encoding/json" "slices" "strings" "time" "git.golem15.com/golem15/sm-sitemap-plugin/contract" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/phrasebook" "git.golem15.com/golem15/summercms/modules/signpost" "git.golem15.com/golem15/summercms/modules/towel" "gorm.io/gorm" ) // HTMLEntry is one link of the HTML sitemap: the address's title (its // path when it has none) and absolute URL; Children nest a category tree. type HTMLEntry struct { Title string `json:"title"` URL string `json:"url"` Children []HTMLEntry `json:"children,omitempty"` } // HTMLGroup is one group of the HTML sitemap overview: Key and Slug name // the file group (the group page is addressed by Slug), Title is its // heading in the requested locale, Total its addresses, Entries the first // page of them and More is true when the group has more than one page. type HTMLGroup struct { Key string `json:"key"` Slug string `json:"slug"` Title string `json:"title"` Total int `json:"total"` More bool `json:"more"` Entries []HTMLEntry `json:"entries"` } // HTMLOverview is the answer of GET /_sitemap/api/v1/html: the locale and // the non-empty groups in the HTML sitemap order. type HTMLOverview struct { Locale string `json:"locale"` PageSize int `json:"page_size"` Groups []HTMLGroup `json:"groups"` } // HTMLGroupPage is the answer of GET /_sitemap/api/v1/html/{group}: one // page (from 1) of a group's links out of Pages. type HTMLGroupPage struct { Group string `json:"group"` Key string `json:"key"` Title string `json:"title"` Locale string `json:"locale"` Page int `json:"page"` Pages int `json:"pages"` Total int `json:"total"` Entries []HTMLEntry `json:"entries"` } // htmlGroupOrder is the order of the groups the HTML sitemap knows by // name (UI-SPEC L3): pages, offers and services, categories, posts, tags, // authors. The groups of other content types follow in registration order, // then any other group (a Filter's) alphabetically. var htmlGroupOrder = []string{"pages", "offers", "services", "categories", "posts", "tags", "authors"} // maxTreeRows bounds the rows a category tree is built from; a larger // group is listed alphabetically. const maxTreeRows = 10000 // The orders of a group's links. const ( orderTitle = "title" // alphabetical by title orderNewest = "newest" // newest first (articles) orderTree = "tree" // tree order (collections with parents) ) // htmlRow is what the HTML sitemap reads of a slot table row. type htmlRow struct { ID uint64 `gorm:"column:id"` Grp string `gorm:"column:grp"` TypeKey string `gorm:"column:type_key"` RecordID string `gorm:"column:record_id"` URL string `gorm:"column:url"` Title string `gorm:"column:title"` PublishedAt *time.Time `gorm:"column:published_at"` ParentRefs string `gorm:"column:parent_refs"` } // htmlGroupInfo is one group of the HTML sitemap before its links are // read: its slug, title, total and order. type htmlGroupInfo struct { slug string title string total int order string } // HTMLSitemap reads the HTML sitemap (D-34): the listed addresses of the // XML sitemaps (never an excluded one, D-24) of locale and those without // one, by group. locale "" means the default_locale setting (no locale // filter when that is empty too). type HTMLSitemap struct { App *backpack.App DB *gorm.DB } // scope is the listed rows of locale. func (h HTMLSitemap) scope(locale string) *gorm.DB { q := h.DB.Table("golem15_sitemap_urls").Where("excluded_reason = ''") if locale != "" { q = q.Where("locale = ? OR locale = ''", locale) } return q } // locale is the locale the HTML sitemap filters by. func (h HTMLSitemap) locale(s models.Settings, locale string) string { if locale = strings.TrimSpace(locale); locale != "" { return locale } return models.Text(s.DefaultLocale) } // groups lists the groups with listed rows of locale in the HTML sitemap // order, with their titles in locale and their orders. func (h HTMLSitemap) groups(ctx context.Context, s models.Settings, locale string) ([]htmlGroupInfo, error) { var counts []struct { Grp string `gorm:"column:grp"` N int `gorm:"column:n"` Articles int `gorm:"column:articles"` Nested int `gorm:"column:nested"` } if err := h.scope(locale).Select(`grp, COUNT(*) AS n, COUNT(*) FILTER (WHERE kind = ?) AS articles, COUNT(*) FILTER (WHERE kind = ? AND jsonb_array_length(parent_refs) > 0) AS nested`, pact.ContentKindArticle, pact.ContentKindCollection). Group("grp").Scan(&counts).Error; err != nil { return nil, err } titles, labels, typeOrder, err := groupTitles(ctx, h.App, h.DB) if err != nil { return nil, err } def := models.Text(s.DefaultLocale) out := make([]htmlGroupInfo, 0, len(counts)) for _, c := range counts { if c.N == 0 { continue } g := htmlGroupInfo{slug: c.Grp, total: c.N, order: orderTitle} switch { case c.Articles == c.N: g.order = orderNewest case c.Nested > 0 && c.N <= maxTreeRows: g.order = orderTree } g.title = pickTitle(titles[c.Grp], locale, def) if g.title == "" { if label := labels[c.Grp]; label != "" { g.title = translateIn(ctx, h.App, locale, label) } } if g.title == "" { g.title = c.Grp } out = append(out, g) } rank := func(slug string) (int, int) { if i := slices.Index(htmlGroupOrder, slug); i >= 0 { return 0, i } if i, ok := typeOrder[slug]; ok { return 1, i } return 2, 0 } slices.SortStableFunc(out, func(a, b htmlGroupInfo) int { ta, ia := rank(a.slug) tb, ib := rank(b.slug) if ta != tb { return ta - tb } if ia != ib { return ia - ib } return strings.Compare(a.slug, b.slug) }) return out, nil } // pickTitle is the title of locale, else of the default locale, else "". func pickTitle(titles map[string]string, locale, def string) string { if t := strings.TrimSpace(titles[locale]); t != "" { return t } if t := strings.TrimSpace(titles[def]); t != "" { return t } return "" } // translateIn translates a phrase key (a content type label) in locale; // without a translator, or for a plain label, it is returned as is. func translateIn(ctx context.Context, app *backpack.App, locale, key string) string { if app == nil { return key } tr, ok := app.Lookup[*phrasebook.Translator]() if !ok || tr == nil { return key } if locale != "" { ctx = towel.WithLocale(ctx, locale) } return tr.Get(ctx, key, nil) } // groupTitles collects the group headings (by group slug, per locale): // the published Filter's group titles, overridden by the content types' // saved HTML sitemap headings; the content types' labels by slug; and the // content types' registration order by slug. func groupTitles(ctx context.Context, app *backpack.App, db *gorm.DB) (titles map[string]map[string]string, labels map[string]string, order map[string]int, err error) { titles, labels, order = map[string]map[string]string{}, map[string]string{}, map[string]int{} if app == nil { return titles, labels, order, nil } if filter, ok := app.Lookup[contract.Filter](); ok && filter != nil { groups, err := filter.Groups(ctx) if err != nil { return nil, nil, nil, err } for _, g := range groups { if len(g.Titles) > 0 { titles[g.Key] = g.Titles } } } if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil { rows, err := LoadTypeSettings(db) if err != nil { return nil, nil, nil, err } for i, typ := range reg.ContentTypes() { cfg := EffectiveType(typ, rows) if len(cfg.Titles) > 0 { titles[cfg.Slug] = cfg.Titles } if _, seen := labels[cfg.Slug]; !seen { labels[cfg.Slug] = typ.Label order[cfg.Slug] = i } } } return titles, labels, order, nil } // entry is one link with what ordering and the tree need. type entry struct { HTMLEntry node models.Ref parent *models.Ref } // rows reads the rows of one group in its order: newest first for // articles, alphabetical by title otherwise (a tree is ordered later). // offset and limit page the read; limit < 0 reads every row. func (h HTMLSitemap) rows(g htmlGroupInfo, locale string, offset, limit int) ([]htmlRow, error) { q := h.scope(locale).Select("id, grp, type_key, record_id, url, title, published_at, parent_refs::text AS parent_refs").Where("grp = ?", g.slug) switch g.order { case orderNewest: q = q.Order("published_at DESC NULLS LAST, first_seen DESC, id DESC") default: q = q.Order("lower(COALESCE(NULLIF(title, ''), url)) ASC, id ASC") } if limit >= 0 { q = q.Offset(offset).Limit(limit) } var out []htmlRow return out, q.Scan(&out).Error } // toEntries turns rows into links on site; a row whose address cannot be // made absolute is skipped. func toEntries(rows []htmlRow, site string) []entry { out := make([]entry, 0, len(rows)) for _, r := range rows { u := Absolute(site, r.URL) if u == "" { continue } title := strings.TrimSpace(r.Title) if title == "" { title = r.URL } e := entry{HTMLEntry: HTMLEntry{Title: title, URL: u}, node: models.Ref{Type: r.TypeKey, ID: r.RecordID}} var parents []models.Ref if r.ParentRefs != "" { _ = json.Unmarshal([]byte(r.ParentRefs), &parents) } if len(parents) > 0 { e.parent = &parents[0] } out = append(out, e) } return out } // treeOrder orders entries as a tree (alphabetical siblings, depth first) // and returns them flat with each entry's parent index in the result (-1 // for a root). An entry whose first parent is not in the group, or that // would close a cycle, is a root. func treeOrder(entries []entry) ([]entry, []int) { index := make(map[models.Ref]int, len(entries)) for i, e := range entries { index[e.node] = i } children := map[int][]int{} var roots []int for i, e := range entries { if e.parent != nil { if p, ok := index[*e.parent]; ok && p != i { children[p] = append(children[p], i) continue } } roots = append(roots, i) } flat := make([]entry, 0, len(entries)) parents := make([]int, 0, len(entries)) visited := make([]bool, len(entries)) var walk func(i, parent int) walk = func(i, parent int) { if visited[i] { return } visited[i] = true at := len(flat) flat = append(flat, entries[i]) parents = append(parents, parent) for _, c := range children[i] { walk(c, at) } } for _, r := range roots { walk(r, -1) } // Entries only reachable through a cycle are roots. for i := range entries { walk(i, -1) } return flat, parents } // nest builds the links of flat[start:end] (tree order, parents indices // into flat): an entry whose parent is on the same page is its child, // every other is a top-level link. func nest(flat []entry, parents []int, start, end int) []HTMLEntry { nodes := make([]*HTMLEntry, end-start) for i := start; i < end; i++ { e := flat[i].HTMLEntry e.Children = nil nodes[i-start] = &e } // Children are attached deepest first so a parent copies its finished // children. for i := end - 1; i >= start; i-- { p := parents[i] if p >= start && p < end { parent := nodes[p-start] parent.Children = append([]HTMLEntry{*nodes[i-start]}, parent.Children...) nodes[i-start] = nil } } out := []HTMLEntry{} for _, n := range nodes { if n != nil { out = append(out, *n) } } return out } // page reads page n (from 1) of group g: its links and the number of // pages. func (h HTMLSitemap) page(g htmlGroupInfo, locale, site string, size, n int) ([]HTMLEntry, int, error) { pages := (g.total + size - 1) / size if g.order == orderTree { rows, err := h.rows(g, locale, 0, -1) if err != nil { return nil, 0, err } flat, parents := treeOrder(toEntries(rows, site)) pages = (len(flat) + size - 1) / size start := (n - 1) * size if start >= len(flat) { return []HTMLEntry{}, pages, nil } return nest(flat, parents, start, min(start+size, len(flat))), pages, nil } rows, err := h.rows(g, locale, (n-1)*size, size) if err != nil { return nil, 0, err } entries := toEntries(rows, site) out := make([]HTMLEntry, 0, len(entries)) for _, e := range entries { out = append(out, e.HTMLEntry) } return out, pages, nil } // Overview answers the HTML sitemap overview of locale: every non-empty // group in order with its first html_page_size links and more set when it // has more. func (h HTMLSitemap) Overview(ctx context.Context, s models.Settings, locale string) (HTMLOverview, error) { locale = h.locale(s, locale) size := s.EffectiveHTMLPageSize() out := HTMLOverview{Locale: locale, PageSize: size, Groups: []HTMLGroup{}} groups, err := h.groups(ctx, s, locale) if err != nil { return out, err } site := SiteURL(h.App, s) for _, g := range groups { entries, pages, err := h.page(g, locale, site, size, 1) if err != nil { return out, err } if len(entries) == 0 { continue } out.Groups = append(out.Groups, HTMLGroup{Key: g.slug, Slug: g.slug, Title: g.title, Total: g.total, More: pages > 1, Entries: entries}) } return out, nil } // Group answers page n of the group addressed by slug in locale; ok is // false for an unknown or empty group and a page past the last. func (h HTMLSitemap) Group(ctx context.Context, s models.Settings, locale, slug string, n int) (HTMLGroupPage, bool, error) { locale = h.locale(s, locale) groups, err := h.groups(ctx, s, locale) if err != nil || n < 1 { return HTMLGroupPage{}, false, err } i := slices.IndexFunc(groups, func(g htmlGroupInfo) bool { return g.slug == slug }) if i < 0 { return HTMLGroupPage{}, false, nil } g := groups[i] entries, pages, err := h.page(g, locale, SiteURL(h.App, s), s.EffectiveHTMLPageSize(), n) if err != nil || n > pages || len(entries) == 0 { return HTMLGroupPage{}, false, err } return HTMLGroupPage{Group: g.slug, Key: g.slug, Title: g.title, Locale: locale, Page: n, Pages: pages, Total: g.total, Entries: entries}, true, nil }