sm-sitemap-plugin/classes/rebuild.go

package classes

import (
	"context"
	"crypto/sha256"
	"database/sql/driver"
	"encoding/hex"
	"errors"
	"fmt"
	"strings"
	"time"

	"git.golem15.com/golem15/sm-sitemap-plugin/contract"
	"git.golem15.com/golem15/sm-sitemap-plugin/models"
	"git.golem15.com/golem15/summercms/modules/backpack"
	"git.golem15.com/golem15/summercms/modules/lagoon"
	"git.golem15.com/golem15/summercms/modules/signpost"
	"gorm.io/gorm"
)

// RebuildKind is the River kind of the rebuild job.
const RebuildKind = "golem15.sitemap.rebuild"

// RebuildArgs are the arguments of the rebuild job: it has none, a rebuild
// always covers every content type and the additional addresses.
type RebuildArgs struct{}

// Kind is RebuildKind.
func (RebuildArgs) Kind() string { return RebuildKind }

// recordsPage is the page size the rebuild asks the content types for.
const recordsPage = 500

// ErrNoDatabase is returned when the application publishes no database.
var ErrNoDatabase = errors.New("golem15.sitemap: database is not configured")

// RebuildResult is what one rebuild changed, as absolute URLs: Added are
// newly listed addresses, Changed listed addresses whose URL changed and
// Removed addresses that left the files. Listed counts the listed
// addresses and Files the files they fill.
type RebuildResult struct {
	Added   []string
	Changed []string
	Removed []string
	Listed  int
	Files   int
}

// DefaultSlug is the file name part of a content type's group: the last
// segment of its key in plural form (journal-post: posts, journal-category:
// categories, seo-page: pages).
func DefaultSlug(key string) string {
	key = strings.ToLower(strings.TrimSpace(key))
	if i := strings.LastIndexAny(key, "-_."); i >= 0 {
		key = key[i+1:]
	}
	var b strings.Builder
	for _, r := range key {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
			b.WriteRune(r)
		}
	}
	s := b.String()
	switch {
	case s == "":
		return "pages"
	case strings.HasSuffix(s, "y") && len(s) > 1 && !strings.ContainsRune("aeiou", rune(s[len(s)-2])):
		return s[:len(s)-1] + "ies"
	case strings.HasSuffix(s, "s"):
		return s
	}
	return s + "s"
}

// ExtraRecordID is the record_id of an additional address: the hex SHA-256
// of the address.
func ExtraRecordID(u string) string {
	sum := sha256.Sum256([]byte(u))
	return hex.EncodeToString(sum[:])
}

// slotKey identifies a row of the slot table.
type slotKey struct {
	typeKey, recordID, locale string
}

// wanted is one address a rebuild found: its group, its row key, its path
// and title, its first lastmod (for a new row) and the reason it is left
// out ("" when listed). An additional address carries its own lastmod,
// which the editor sets, so it is written on every rebuild. images, kind,
// published and parents are the record's facts for the image and News
// sitemaps and the HTML sitemap.
type wanted struct {
	key        slotKey
	grp        string
	url        string
	title      string
	firstMod   *time.Time
	reason     string
	extra      bool
	extraMod   *time.Time
	alternates []models.Alternate
	images     []string
	kind       string
	published  *time.Time
	parents    []models.Ref
}

// collected is what a rebuild read from the content types and settings.
type collected struct {
	rows        []wanted
	unpublished map[slotKey]bool
}

// Rebuild reads every content type's published records (signpost Records,
// D-93) and the additional addresses, writes them into the slot table and
// numbers the files, in one transaction that holds the settings row, so
// two rebuilds never interleave. A record that is gone or unpublished
// leaves its rows in the table with a reason, as a hole; only a changed
// file size renumbers the files (D-30). In production, with IndexNow on,
// the added, changed and removed addresses are queued for IndexNow after
// the commit (NotifyIndexNow).
func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
	var res RebuildResult
	if app == nil {
		return res, ErrNoDatabase
	}
	db, ok := app.Lookup[*gorm.DB]()
	if !ok || db == nil {
		return res, ErrNoDatabase
	}
	db = db.WithContext(ctx)
	settings, err := LoadSettings(db)
	if err != nil {
		return res, err
	}
	types, err := LoadTypeSettings(db)
	if err != nil {
		return res, err
	}
	reg, _ := app.Lookup[*signpost.Registry]()
	filter, _ := app.Lookup[contract.Filter]()
	site := SiteURL(app, settings)
	found, err := collect(ctx, reg, filter, settings, types, site)
	if err != nil {
		return res, err
	}
	err = db.Transaction(func(tx *gorm.DB) error {
		if err := tx.Exec(`SELECT id FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Error; err != nil {
			return err
		}
		locked, err := LoadSettings(tx)
		if err != nil {
			return err
		}
		res, err = apply(tx, found, locked, site)
		return err
	})
	if err != nil {
		return res, err
	}
	// IndexNow hears of the change once it is committed (D-31); a queue
	// error is logged and never fails the rebuild.
	if err := NotifyIndexNow(ctx, app, db, res); err != nil {
		pluginLog.Warn("golem15.sitemap: queue indexnow", "error", err)
	}
	return res, nil
}

// collect walks the content types page by page and adds the additional
// addresses. The records of a type switched off in the settings are kept
// in the table with the reason contract.ReasonType, so the settings screen
// counts them. Every other page of addresses (and the additional
// addresses) is one batch for the published contract.Filter, when there is
// one (D-95): an excluded address keeps its row with the Filter's reason, a
// Decision.Group moves it to that file group, and Decision.LastMod is the
// first lastmod of a new row. The hreflang alternates of a record are its
// listed addresses (D-55 to D-57). Each address keeps its page's images
// (PageImages, on site), the type's kind, the record's first publication
// and its parents. An additional address a listed record already has is
// left out as models.ReasonDuplicate (markListedExtras).
func collect(ctx context.Context, reg *signpost.Registry, filter contract.Filter, settings models.Settings, types map[string]models.TypeSetting, site string) (collected, error) {
	out := collected{unpublished: map[slotKey]bool{}}
	seen := map[slotKey]bool{}
	defaultLocale := models.Text(settings.DefaultLocale)
	// batch decides one batch and adds its rows; rows of one record are
	// adjacent, so the alternates are computed per run of a record.
	batch := func(rows []wanted) error {
		var fresh []wanted
		for _, w := range rows {
			if !seen[w.key] {
				seen[w.key] = true
				fresh = append(fresh, w)
			}
		}
		if err := decide(ctx, filter, fresh); err != nil {
			return err
		}
		setAlternates(fresh, defaultLocale)
		out.rows = append(out.rows, fresh...)
		return nil
	}
	if reg != nil {
		for _, typ := range reg.ContentTypes() {
			cfg := EffectiveType(typ, types)
			kind := ""
			if typ.Content != nil {
				kind = typ.Content.Kind
			}
			after := ""
			for {
				page, err := reg.Records(ctx, typ.Key, after, recordsPage, defaultLocale)
				if err != nil {
					return out, fmt.Errorf("golem15.sitemap: records of %s: %w", typ.Key, err)
				}
				var rows []wanted
				for _, rec := range page.Records {
					if !rec.Published {
						out.unpublished[slotKey{typ.Key, rec.ID, ""}] = true
						continue
					}
					parents := make([]models.Ref, 0, len(rec.Parents))
					for _, p := range rec.Parents {
						if p.Type != "" && p.ID != "" {
							parents = append(parents, models.Ref{Type: p.Type, ID: p.ID})
						}
					}
					for _, l := range rec.Locales {
						if strings.TrimSpace(l.URL) == "" {
							continue
						}
						w := wanted{
							key:       slotKey{typ.Key, rec.ID, l.Locale},
							grp:       cfg.Slug,
							url:       l.URL,
							title:     l.Title,
							firstMod:  rec.LastMod,
							images:    PageImages(l, site),
							kind:      kind,
							published: rec.PublishedAt,
							parents:   parents,
						}
						if !cfg.Include {
							w.reason = contract.ReasonType
						}
						rows = append(rows, w)
					}
				}
				if err := batch(rows); err != nil {
					return out, err
				}
				if page.Next == "" || page.Next == after {
					break
				}
				after = page.Next
			}
		}
	}
	extras := extraRows(settings.Extra())
	markListedExtras(out.rows, extras, site)
	if err := batch(extras); err != nil {
		return out, err
	}
	return out, nil
}

// extraRows turns the settings' additional addresses into rows; an address
// that is not allowed is skipped.
func extraRows(list []models.ExtraURL) []wanted {
	var extras []wanted
	for _, e := range list {
		u := strings.TrimSpace(e.URL)
		if !models.AllowedAddress(u) {
			continue
		}
		mod := models.ParseDate(e.LastMod)
		extras = append(extras, wanted{
			key:      slotKey{models.ExtraTypeKey, ExtraRecordID(u), ""},
			grp:      models.ExtraGroup,
			url:      u,
			extra:    true,
			extraMod: mod,
			firstMod: mod,
		})
	}
	return extras
}

// markListedExtras leaves out (models.ReasonDuplicate) every additional
// address whose absolute URL a content record already lists, so an address
// both in a content type and in the additional addresses (a definition
// migrated from the old sitemap, for example) is in the files once, with
// the record's lastmod and alternates. A record left out by the filter or
// the type switch does not count: the editor's additional address then
// stays listed. The rows keep their place in the slot table, so the
// settings screen counts them by reason.
func markListedExtras(records, extras []wanted, site string) {
	listed := map[string]bool{}
	for _, w := range records {
		if w.reason == "" && !w.extra {
			listed[addressKey(site, w.url)] = true
		}
	}
	for i := range extras {
		if extras[i].reason == "" && listed[addressKey(site, extras[i].url)] {
			extras[i].reason = models.ReasonDuplicate
		}
	}
}

// addressKey is the absolute form of an address for comparison, or the
// address itself when it has none (a path without a site URL).
func addressKey(site, u string) string {
	u = strings.TrimSpace(u)
	if a := Absolute(site, u); a != "" {
		return a
	}
	return u
}

// decide asks filter about the rows not already left out (a type switched
// off is decided by the settings) and applies its decisions. Without a
// Filter every row stays listed.
func decide(ctx context.Context, filter contract.Filter, rows []wanted) error {
	if filter == nil {
		return nil
	}
	var asked []int
	var candidates []contract.Candidate
	for i, w := range rows {
		if w.reason != "" {
			continue
		}
		asked = append(asked, i)
		candidates = append(candidates, contract.Candidate{TypeKey: w.key.typeKey, RecordID: w.key.recordID, Locale: w.key.locale, URL: w.url})
	}
	if len(candidates) == 0 {
		return nil
	}
	decisions, err := filter.Decide(ctx, candidates)
	if err != nil {
		return fmt.Errorf("golem15.sitemap: filter: %w", err)
	}
	if len(decisions) != len(candidates) {
		return fmt.Errorf("golem15.sitemap: filter answered %d decisions for %d candidates", len(decisions), len(candidates))
	}
	for j, d := range decisions {
		w := &rows[asked[j]]
		if d.Exclude {
			w.reason = strings.TrimSpace(d.Reason)
			if w.reason == "" {
				w.reason = contract.ReasonNoindex
			}
			continue
		}
		if g := strings.TrimSpace(d.Group); g != "" {
			if ValidSlug(g) && !ReservedSlugs[g] {
				w.grp = g
			} else {
				pluginLog.Warn("golem15.sitemap: filter group ignored", "group", g, "type", w.key.typeKey, "record", w.key.recordID)
			}
		}
		if d.LastMod != nil {
			w.firstMod = d.LastMod
		}
	}
	return nil
}

// setAlternates gives the listed addresses of every record with more than
// one listed locale its hreflang alternates: each listed locale with its
// address, itself included, then x-default on the default locale's address
// (the record's first listed locale when the default has none). A locale
// without an address or left out has no alternate; a record with one
// listed locale gets none (D-55 to D-57, RESEARCH Pitfall 3).
func setAlternates(rows []wanted, defaultLocale string) {
	type record struct{ typeKey, id string }
	listed := map[record][]int{}
	var order []record
	for i, w := range rows {
		if w.reason != "" || w.extra || w.key.locale == "" || w.key.locale == "x-default" {
			continue
		}
		r := record{w.key.typeKey, w.key.recordID}
		if _, ok := listed[r]; !ok {
			order = append(order, r)
		}
		listed[r] = append(listed[r], i)
	}
	for _, r := range order {
		idx := listed[r]
		if len(idx) < 2 {
			continue
		}
		alts := make([]models.Alternate, 0, len(idx)+1)
		xDefault := rows[idx[0]].url
		for _, i := range idx {
			alts = append(alts, models.Alternate{Hreflang: rows[i].key.locale, Href: rows[i].url})
			if rows[i].key.locale == defaultLocale {
				xDefault = rows[i].url
			}
		}
		alts = append(alts, models.Alternate{Hreflang: "x-default", Href: xDefault})
		for _, i := range idx {
			rows[i].alternates = alts
		}
	}
}

// apply writes the found addresses into the slot table and numbers the
// files, inside the rebuild's transaction.
func apply(tx *gorm.DB, found collected, settings models.Settings, site string) (RebuildResult, error) {
	var res RebuildResult
	var existing []models.URL
	if err := tx.Order("id ASC").Find(&existing).Error; err != nil {
		return res, err
	}
	now := time.Now().UTC().Truncate(time.Microsecond)
	byKey := make(map[slotKey]*models.URL, len(existing))
	for i := range existing {
		r := &existing[i]
		byKey[slotKey{r.TypeKey, r.RecordID, r.Locale}] = r
	}
	dirty := map[uint64]bool{}
	var created []*models.URL
	matched := map[slotKey]bool{}
	for _, w := range found.rows {
		matched[w.key] = true
		r, ok := byKey[w.key]
		if !ok {
			mod := w.firstMod
			if mod == nil && !w.extra {
				mod = &now
			}
			row := &models.URL{
				Grp:            w.grp,
				TypeKey:        w.key.typeKey,
				RecordID:       w.key.recordID,
				Locale:         w.key.locale,
				URL:            w.url,
				Title:          w.title,
				FirstSeen:      now,
				LastMod:        utcPtr(mod),
				Alternates:     lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true},
				Images:         lagoon.Jsonable[[]string]{Data: nonNil(w.images), Valid: true},
				Kind:           w.kind,
				PublishedAt:    utcPtr(w.published),
				ParentRefs:     lagoon.Jsonable[[]models.Ref]{Data: nonNil(w.parents), Valid: true},
				ExcludedReason: w.reason,
				UpdatedAt:      now,
			}
			created = append(created, row)
			if w.reason == "" {
				res.Added = append(res.Added, Absolute(site, w.url))
			}
			continue
		}
		wasListed := r.ExcludedReason == ""
		before := *r
		if r.Grp != w.grp {
			r.Grp = w.grp
			r.FileNo = nil
		}
		r.URL = w.url
		r.Title = w.title
		r.ExcludedReason = w.reason
		r.Alternates = lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true}
		r.Images = lagoon.Jsonable[[]string]{Data: nonNil(w.images), Valid: true}
		r.Kind = w.kind
		r.PublishedAt = utcPtr(w.published)
		r.ParentRefs = lagoon.Jsonable[[]models.Ref]{Data: nonNil(w.parents), Valid: true}
		if w.extra {
			r.LastMod = utcPtr(w.extraMod)
		}
		listed := w.reason == ""
		switch {
		case listed && !wasListed:
			res.Added = append(res.Added, Absolute(site, r.URL))
		case !listed && wasListed:
			res.Removed = append(res.Removed, Absolute(site, before.URL))
		case listed && before.URL != r.URL:
			res.Changed = append(res.Changed, Absolute(site, r.URL))
		}
		if rowChanged(before, *r) {
			dirty[r.ID] = true
		}
	}
	for i := range existing {
		r := &existing[i]
		key := slotKey{r.TypeKey, r.RecordID, r.Locale}
		if matched[key] {
			continue
		}
		reason := models.ReasonDeleted
		if found.unpublished[slotKey{r.TypeKey, r.RecordID, ""}] {
			reason = models.ReasonUnpublished
		}
		if r.ExcludedReason == reason {
			continue
		}
		if r.ExcludedReason == "" {
			res.Removed = append(res.Removed, Absolute(site, r.URL))
		}
		r.ExcludedReason = reason
		dirty[r.ID] = true
	}
	if len(created) > 0 {
		if err := tx.CreateInBatches(created, 500).Error; err != nil {
			return res, err
		}
	}
	size := settings.EffectiveFileSize()
	renumber := settings.NumberedSize == nil || *settings.NumberedSize != size
	slots := make([]*Slot, 0, len(existing)+len(created))
	rowsByID := make(map[uint64]*models.URL, len(existing)+len(created))
	for i := range existing {
		r := &existing[i]
		rowsByID[r.ID] = r
		slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""})
	}
	for _, r := range created {
		rowsByID[r.ID] = r
		slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""})
	}
	AssignSlots(slots, size, renumber)
	files := map[string]bool{}
	for _, s := range slots {
		r := rowsByID[s.ID]
		if !sameInt(r.FileNo, s.FileNo) {
			r.FileNo = s.FileNo
			dirty[r.ID] = true
		}
		if s.Listed {
			res.Listed++
			if s.FileNo != nil {
				files[FileName(s.Grp, *s.FileNo)] = true
			}
		}
	}
	// The image files: the listed rows with images, numbered like the
	// per-group files (stable, holes kept, renumbered with them).
	imageSlots := make([]*Slot, 0, len(slots))
	for _, s := range slots {
		r := rowsByID[s.ID]
		imageSlots = append(imageSlots, &Slot{ID: r.ID, Grp: ImagesGroup, FileNo: r.ImageFileNo, FirstSeen: r.FirstSeen, Listed: s.Listed && len(r.Images.Get()) > 0})
	}
	AssignSlots(imageSlots, size, renumber)
	for _, s := range imageSlots {
		r := rowsByID[s.ID]
		if !sameInt(r.ImageFileNo, s.FileNo) {
			r.ImageFileNo = s.FileNo
			dirty[r.ID] = true
		}
	}
	res.Files = len(files)
	for id := range dirty {
		r := rowsByID[id]
		r.UpdatedAt = now
		if err := tx.Model(&models.URL{}).Where("id = ?", id).Updates(map[string]any{
			"grp":             r.Grp,
			"url":             r.URL,
			"title":           r.Title,
			"file_no":         r.FileNo,
			"lastmod":         r.LastMod,
			"alternates":      r.Alternates,
			"images":          r.Images,
			"image_file_no":   r.ImageFileNo,
			"kind":            r.Kind,
			"published_at":    r.PublishedAt,
			"parent_refs":     r.ParentRefs,
			"excluded_reason": r.ExcludedReason,
			"updated_at":      r.UpdatedAt,
		}).Error; err != nil {
			return res, err
		}
	}
	if err := tx.Exec(`UPDATE golem15_sitemap_settings SET numbered_size = ?, built_at = ? WHERE id = 1`, size, now).Error; err != nil {
		return res, err
	}
	return res, nil
}

// rowChanged reports whether a rebuild changed a stored row.
func rowChanged(a, b models.URL) bool {
	if a.Grp != b.Grp || a.URL != b.URL || a.Title != b.Title || a.ExcludedReason != b.ExcludedReason || a.Kind != b.Kind ||
		!sameInt(a.FileNo, b.FileNo) || !sameTime(a.LastMod, b.LastMod) || !sameTime(a.PublishedAt, b.PublishedAt) {
		return true
	}
	return jsonDiffers(a.Alternates, b.Alternates) || jsonDiffers(a.Images, b.Images) || jsonDiffers(a.ParentRefs, b.ParentRefs)
}

// jsonDiffers reports whether two stored JSON values differ.
func jsonDiffers(a, b driver.Valuer) bool {
	av, _ := a.Value()
	bv, _ := b.Value()
	return fmt.Sprint(av) != fmt.Sprint(bv)
}

func sameInt(a, b *int) bool {
	return (a == nil && b == nil) || (a != nil && b != nil && *a == *b)
}

func sameTime(a, b *time.Time) bool {
	return (a == nil && b == nil) || (a != nil && b != nil && a.Equal(*b))
}

func utcPtr(t *time.Time) *time.Time {
	if t == nil {
		return nil
	}
	u := t.UTC().Truncate(time.Microsecond)
	return &u
}

func nonNil[T any](s []T) []T {
	if s == nil {
		return []T{}
	}
	return s
}