sm-sitemap-plugin/classes/htmlgroups.go

package classes

import (
	"context"
	"encoding/json"
	"slices"
	"strings"
	"time"

	"git.golem15.com/golem15/sm-sitemap-plugin/contract"
	"git.golem15.com/golem15/sm-sitemap-plugin/models"
	"git.golem15.com/golem15/summercms/modules/backpack"
	"git.golem15.com/golem15/summercms/modules/pact"
	"git.golem15.com/golem15/summercms/modules/phrasebook"
	"git.golem15.com/golem15/summercms/modules/signpost"
	"git.golem15.com/golem15/summercms/modules/towel"
	"gorm.io/gorm"
)

// HTMLEntry is one link of the HTML sitemap: the address's title (its
// path when it has none) and absolute URL; Children nest a category tree.
type HTMLEntry struct {
	Title    string      `json:"title"`
	URL      string      `json:"url"`
	Children []HTMLEntry `json:"children,omitempty"`
}

// HTMLGroup is one group of the HTML sitemap overview: Key and Slug name
// the file group (the group page is addressed by Slug), Title is its
// heading in the requested locale, Total its addresses, Entries the first
// page of them and More is true when the group has more than one page.
type HTMLGroup struct {
	Key     string      `json:"key"`
	Slug    string      `json:"slug"`
	Title   string      `json:"title"`
	Total   int         `json:"total"`
	More    bool        `json:"more"`
	Entries []HTMLEntry `json:"entries"`
}

// HTMLOverview is the answer of GET /_sitemap/api/v1/html: the locale and
// the non-empty groups in the HTML sitemap order.
type HTMLOverview struct {
	Locale   string      `json:"locale"`
	PageSize int         `json:"page_size"`
	Groups   []HTMLGroup `json:"groups"`
}

// HTMLGroupPage is the answer of GET /_sitemap/api/v1/html/{group}: one
// page (from 1) of a group's links out of Pages.
type HTMLGroupPage struct {
	Group   string      `json:"group"`
	Key     string      `json:"key"`
	Title   string      `json:"title"`
	Locale  string      `json:"locale"`
	Page    int         `json:"page"`
	Pages   int         `json:"pages"`
	Total   int         `json:"total"`
	Entries []HTMLEntry `json:"entries"`
}

// htmlGroupOrder is the order of the groups the HTML sitemap knows by
// name (UI-SPEC L3): pages, offers and services, categories, posts, tags,
// authors. The groups of other content types follow in registration order,
// then any other group (a Filter's) alphabetically.
var htmlGroupOrder = []string{"pages", "offers", "services", "categories", "posts", "tags", "authors"}

// maxTreeRows bounds the rows a category tree is built from; a larger
// group is listed alphabetically.
const maxTreeRows = 10000

// The orders of a group's links.
const (
	orderTitle  = "title"  // alphabetical by title
	orderNewest = "newest" // newest first (articles)
	orderTree   = "tree"   // tree order (collections with parents)
)

// htmlRow is what the HTML sitemap reads of a slot table row.
type htmlRow struct {
	ID          uint64     `gorm:"column:id"`
	Grp         string     `gorm:"column:grp"`
	TypeKey     string     `gorm:"column:type_key"`
	RecordID    string     `gorm:"column:record_id"`
	URL         string     `gorm:"column:url"`
	Title       string     `gorm:"column:title"`
	PublishedAt *time.Time `gorm:"column:published_at"`
	ParentRefs  string     `gorm:"column:parent_refs"`
}

// htmlGroupInfo is one group of the HTML sitemap before its links are
// read: its slug, title, total and order.
type htmlGroupInfo struct {
	slug  string
	title string
	total int
	order string
}

// HTMLSitemap reads the HTML sitemap (D-34): the listed addresses of the
// XML sitemaps (never an excluded one, D-24) of locale and those without
// one, by group. locale "" means the default_locale setting (no locale
// filter when that is empty too).
type HTMLSitemap struct {
	App *backpack.App
	DB  *gorm.DB
}

// scope is the listed rows of locale.
func (h HTMLSitemap) scope(locale string) *gorm.DB {
	q := h.DB.Table("golem15_sitemap_urls").Where("excluded_reason = ''")
	if locale != "" {
		q = q.Where("locale = ? OR locale = ''", locale)
	}
	return q
}

// locale is the locale the HTML sitemap filters by.
func (h HTMLSitemap) locale(s models.Settings, locale string) string {
	if locale = strings.TrimSpace(locale); locale != "" {
		return locale
	}
	return models.Text(s.DefaultLocale)
}

// groups lists the groups with listed rows of locale in the HTML sitemap
// order, with their titles in locale and their orders.
func (h HTMLSitemap) groups(ctx context.Context, s models.Settings, locale string) ([]htmlGroupInfo, error) {
	var counts []struct {
		Grp      string `gorm:"column:grp"`
		N        int    `gorm:"column:n"`
		Articles int    `gorm:"column:articles"`
		Nested   int    `gorm:"column:nested"`
	}
	if err := h.scope(locale).Select(`grp, COUNT(*) AS n,
COUNT(*) FILTER (WHERE kind = ?) AS articles,
COUNT(*) FILTER (WHERE kind = ? AND jsonb_array_length(parent_refs) > 0) AS nested`, pact.ContentKindArticle, pact.ContentKindCollection).
		Group("grp").Scan(&counts).Error; err != nil {
		return nil, err
	}
	titles, labels, typeOrder, err := groupTitles(ctx, h.App, h.DB)
	if err != nil {
		return nil, err
	}
	def := models.Text(s.DefaultLocale)
	out := make([]htmlGroupInfo, 0, len(counts))
	for _, c := range counts {
		if c.N == 0 {
			continue
		}
		g := htmlGroupInfo{slug: c.Grp, total: c.N, order: orderTitle}
		switch {
		case c.Articles == c.N:
			g.order = orderNewest
		case c.Nested > 0 && c.N <= maxTreeRows:
			g.order = orderTree
		}
		g.title = pickTitle(titles[c.Grp], locale, def)
		if g.title == "" {
			if label := labels[c.Grp]; label != "" {
				g.title = translateIn(ctx, h.App, locale, label)
			}
		}
		if g.title == "" {
			g.title = c.Grp
		}
		out = append(out, g)
	}
	rank := func(slug string) (int, int) {
		if i := slices.Index(htmlGroupOrder, slug); i >= 0 {
			return 0, i
		}
		if i, ok := typeOrder[slug]; ok {
			return 1, i
		}
		return 2, 0
	}
	slices.SortStableFunc(out, func(a, b htmlGroupInfo) int {
		ta, ia := rank(a.slug)
		tb, ib := rank(b.slug)
		if ta != tb {
			return ta - tb
		}
		if ia != ib {
			return ia - ib
		}
		return strings.Compare(a.slug, b.slug)
	})
	return out, nil
}

// pickTitle is the title of locale, else of the default locale, else "".
func pickTitle(titles map[string]string, locale, def string) string {
	if t := strings.TrimSpace(titles[locale]); t != "" {
		return t
	}
	if t := strings.TrimSpace(titles[def]); t != "" {
		return t
	}
	return ""
}

// translateIn translates a phrase key (a content type label) in locale;
// without a translator, or for a plain label, it is returned as is.
func translateIn(ctx context.Context, app *backpack.App, locale, key string) string {
	if app == nil {
		return key
	}
	tr, ok := app.Lookup[*phrasebook.Translator]()
	if !ok || tr == nil {
		return key
	}
	if locale != "" {
		ctx = towel.WithLocale(ctx, locale)
	}
	return tr.Get(ctx, key, nil)
}

// groupTitles collects the group headings (by group slug, per locale):
// the published Filter's group titles, overridden by the content types'
// saved HTML sitemap headings; the content types' labels by slug; and the
// content types' registration order by slug.
func groupTitles(ctx context.Context, app *backpack.App, db *gorm.DB) (titles map[string]map[string]string, labels map[string]string, order map[string]int, err error) {
	titles, labels, order = map[string]map[string]string{}, map[string]string{}, map[string]int{}
	if app == nil {
		return titles, labels, order, nil
	}
	if filter, ok := app.Lookup[contract.Filter](); ok && filter != nil {
		groups, err := filter.Groups(ctx)
		if err != nil {
			return nil, nil, nil, err
		}
		for _, g := range groups {
			if len(g.Titles) > 0 {
				titles[g.Key] = g.Titles
			}
		}
	}
	if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil {
		rows, err := LoadTypeSettings(db)
		if err != nil {
			return nil, nil, nil, err
		}
		for i, typ := range reg.ContentTypes() {
			cfg := EffectiveType(typ, rows)
			if len(cfg.Titles) > 0 {
				titles[cfg.Slug] = cfg.Titles
			}
			if _, seen := labels[cfg.Slug]; !seen {
				labels[cfg.Slug] = typ.Label
				order[cfg.Slug] = i
			}
		}
	}
	return titles, labels, order, nil
}

// entry is one link with what ordering and the tree need.
type entry struct {
	HTMLEntry
	node   models.Ref
	parent *models.Ref
}

// rows reads the rows of one group in its order: newest first for
// articles, alphabetical by title otherwise (a tree is ordered later).
// offset and limit page the read; limit < 0 reads every row.
func (h HTMLSitemap) rows(g htmlGroupInfo, locale string, offset, limit int) ([]htmlRow, error) {
	q := h.scope(locale).Select("id, grp, type_key, record_id, url, title, published_at, parent_refs::text AS parent_refs").Where("grp = ?", g.slug)
	switch g.order {
	case orderNewest:
		q = q.Order("published_at DESC NULLS LAST, first_seen DESC, id DESC")
	default:
		q = q.Order("lower(COALESCE(NULLIF(title, ''), url)) ASC, id ASC")
	}
	if limit >= 0 {
		q = q.Offset(offset).Limit(limit)
	}
	var out []htmlRow
	return out, q.Scan(&out).Error
}

// toEntries turns rows into links on site; a row whose address cannot be
// made absolute is skipped.
func toEntries(rows []htmlRow, site string) []entry {
	out := make([]entry, 0, len(rows))
	for _, r := range rows {
		u := Absolute(site, r.URL)
		if u == "" {
			continue
		}
		title := strings.TrimSpace(r.Title)
		if title == "" {
			title = r.URL
		}
		e := entry{HTMLEntry: HTMLEntry{Title: title, URL: u}, node: models.Ref{Type: r.TypeKey, ID: r.RecordID}}
		var parents []models.Ref
		if r.ParentRefs != "" {
			_ = json.Unmarshal([]byte(r.ParentRefs), &parents)
		}
		if len(parents) > 0 {
			e.parent = &parents[0]
		}
		out = append(out, e)
	}
	return out
}

// treeOrder orders entries as a tree (alphabetical siblings, depth first)
// and returns them flat with each entry's parent index in the result (-1
// for a root). An entry whose first parent is not in the group, or that
// would close a cycle, is a root.
func treeOrder(entries []entry) ([]entry, []int) {
	index := make(map[models.Ref]int, len(entries))
	for i, e := range entries {
		index[e.node] = i
	}
	children := map[int][]int{}
	var roots []int
	for i, e := range entries {
		if e.parent != nil {
			if p, ok := index[*e.parent]; ok && p != i {
				children[p] = append(children[p], i)
				continue
			}
		}
		roots = append(roots, i)
	}
	flat := make([]entry, 0, len(entries))
	parents := make([]int, 0, len(entries))
	visited := make([]bool, len(entries))
	var walk func(i, parent int)
	walk = func(i, parent int) {
		if visited[i] {
			return
		}
		visited[i] = true
		at := len(flat)
		flat = append(flat, entries[i])
		parents = append(parents, parent)
		for _, c := range children[i] {
			walk(c, at)
		}
	}
	for _, r := range roots {
		walk(r, -1)
	}
	// Entries only reachable through a cycle are roots.
	for i := range entries {
		walk(i, -1)
	}
	return flat, parents
}

// nest builds the links of flat[start:end] (tree order, parents indices
// into flat): an entry whose parent is on the same page is its child,
// every other is a top-level link.
func nest(flat []entry, parents []int, start, end int) []HTMLEntry {
	nodes := make([]*HTMLEntry, end-start)
	for i := start; i < end; i++ {
		e := flat[i].HTMLEntry
		e.Children = nil
		nodes[i-start] = &e
	}
	// Children are attached deepest first so a parent copies its finished
	// children.
	for i := end - 1; i >= start; i-- {
		p := parents[i]
		if p >= start && p < end {
			parent := nodes[p-start]
			parent.Children = append([]HTMLEntry{*nodes[i-start]}, parent.Children...)
			nodes[i-start] = nil
		}
	}
	out := []HTMLEntry{}
	for _, n := range nodes {
		if n != nil {
			out = append(out, *n)
		}
	}
	return out
}

// page reads page n (from 1) of group g: its links and the number of
// pages.
func (h HTMLSitemap) page(g htmlGroupInfo, locale, site string, size, n int) ([]HTMLEntry, int, error) {
	pages := (g.total + size - 1) / size
	if g.order == orderTree {
		rows, err := h.rows(g, locale, 0, -1)
		if err != nil {
			return nil, 0, err
		}
		flat, parents := treeOrder(toEntries(rows, site))
		pages = (len(flat) + size - 1) / size
		start := (n - 1) * size
		if start >= len(flat) {
			return []HTMLEntry{}, pages, nil
		}
		return nest(flat, parents, start, min(start+size, len(flat))), pages, nil
	}
	rows, err := h.rows(g, locale, (n-1)*size, size)
	if err != nil {
		return nil, 0, err
	}
	entries := toEntries(rows, site)
	out := make([]HTMLEntry, 0, len(entries))
	for _, e := range entries {
		out = append(out, e.HTMLEntry)
	}
	return out, pages, nil
}

// Overview answers the HTML sitemap overview of locale: every non-empty
// group in order with its first html_page_size links and more set when it
// has more.
func (h HTMLSitemap) Overview(ctx context.Context, s models.Settings, locale string) (HTMLOverview, error) {
	locale = h.locale(s, locale)
	size := s.EffectiveHTMLPageSize()
	out := HTMLOverview{Locale: locale, PageSize: size, Groups: []HTMLGroup{}}
	groups, err := h.groups(ctx, s, locale)
	if err != nil {
		return out, err
	}
	site := SiteURL(h.App, s)
	for _, g := range groups {
		entries, pages, err := h.page(g, locale, site, size, 1)
		if err != nil {
			return out, err
		}
		if len(entries) == 0 {
			continue
		}
		out.Groups = append(out.Groups, HTMLGroup{Key: g.slug, Slug: g.slug, Title: g.title, Total: g.total, More: pages > 1, Entries: entries})
	}
	return out, nil
}

// Group answers page n of the group addressed by slug in locale; ok is
// false for an unknown or empty group and a page past the last.
func (h HTMLSitemap) Group(ctx context.Context, s models.Settings, locale, slug string, n int) (HTMLGroupPage, bool, error) {
	locale = h.locale(s, locale)
	groups, err := h.groups(ctx, s, locale)
	if err != nil || n < 1 {
		return HTMLGroupPage{}, false, err
	}
	i := slices.IndexFunc(groups, func(g htmlGroupInfo) bool { return g.slug == slug })
	if i < 0 {
		return HTMLGroupPage{}, false, nil
	}
	g := groups[i]
	entries, pages, err := h.page(g, locale, SiteURL(h.App, s), s.EffectiveHTMLPageSize(), n)
	if err != nil || n > pages || len(entries) == 0 {
		return HTMLGroupPage{}, false, err
	}
	return HTMLGroupPage{Group: g.slug, Key: g.slug, Title: g.title, Locale: locale, Page: n, Pages: pages, Total: g.total, Entries: entries}, true, nil
}