sm-sitemap-plugin/classes/rebuild.go

package classes

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"strings"
	"time"

	"git.golem15.com/golem15/sm-sitemap-plugin/models"
	"git.golem15.com/golem15/summercms/modules/backpack"
	"git.golem15.com/golem15/summercms/modules/lagoon"
	"git.golem15.com/golem15/summercms/modules/signpost"
	"gorm.io/gorm"
)

// RebuildKind is the River kind of the rebuild job.
const RebuildKind = "golem15.sitemap.rebuild"

// RebuildArgs are the arguments of the rebuild job: it has none, a rebuild
// always covers every content type and the additional addresses.
type RebuildArgs struct{}

// Kind is RebuildKind.
func (RebuildArgs) Kind() string { return RebuildKind }

// recordsPage is the page size the rebuild asks the content types for.
const recordsPage = 500

// ErrNoDatabase is returned when the application publishes no database.
var ErrNoDatabase = errors.New("golem15.sitemap: database is not configured")

// RebuildResult is what one rebuild changed, as absolute URLs: Added are
// newly listed addresses, Changed listed addresses whose URL changed and
// Removed addresses that left the files. Listed counts the listed
// addresses and Files the files they fill.
type RebuildResult struct {
	Added   []string
	Changed []string
	Removed []string
	Listed  int
	Files   int
}

// DefaultSlug is the file name part of a content type's group: the last
// segment of its key in plural form (journal-post: posts, journal-category:
// categories, seo-page: pages).
func DefaultSlug(key string) string {
	key = strings.ToLower(strings.TrimSpace(key))
	if i := strings.LastIndexAny(key, "-_."); i >= 0 {
		key = key[i+1:]
	}
	var b strings.Builder
	for _, r := range key {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
			b.WriteRune(r)
		}
	}
	s := b.String()
	switch {
	case s == "":
		return "pages"
	case strings.HasSuffix(s, "y") && len(s) > 1 && !strings.ContainsRune("aeiou", rune(s[len(s)-2])):
		return s[:len(s)-1] + "ies"
	case strings.HasSuffix(s, "s"):
		return s
	}
	return s + "s"
}

// ExtraRecordID is the record_id of an additional address: the hex SHA-256
// of the address.
func ExtraRecordID(u string) string {
	sum := sha256.Sum256([]byte(u))
	return hex.EncodeToString(sum[:])
}

// slotKey identifies a row of the slot table.
type slotKey struct {
	typeKey, recordID, locale string
}

// wanted is one address a rebuild found: its group, its row key, its path
// and title, its first lastmod (for a new row) and the reason it is left
// out ("" when listed). An additional address carries its own lastmod,
// which the editor sets, so it is written on every rebuild.
type wanted struct {
	key        slotKey
	grp        string
	url        string
	title      string
	firstMod   *time.Time
	reason     string
	extra      bool
	extraMod   *time.Time
	alternates []models.Alternate
}

// collected is what a rebuild read from the content types and settings.
type collected struct {
	rows        []wanted
	unpublished map[slotKey]bool
}

// Rebuild reads every content type's published records (signpost Records,
// D-93) and the additional addresses, writes them into the slot table and
// numbers the files, in one transaction that holds the settings row, so
// two rebuilds never interleave. A record that is gone or unpublished
// leaves its rows in the table with a reason, as a hole; only a changed
// file size renumbers the files (D-30).
func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
	var res RebuildResult
	if app == nil {
		return res, ErrNoDatabase
	}
	db, ok := app.Lookup[*gorm.DB]()
	if !ok || db == nil {
		return res, ErrNoDatabase
	}
	db = db.WithContext(ctx)
	settings, err := LoadSettings(db)
	if err != nil {
		return res, err
	}
	reg, _ := app.Lookup[*signpost.Registry]()
	found, err := collect(ctx, reg, settings)
	if err != nil {
		return res, err
	}
	site := SiteURL(app, settings)
	err = db.Transaction(func(tx *gorm.DB) error {
		if err := tx.Exec(`SELECT id FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Error; err != nil {
			return err
		}
		locked, err := LoadSettings(tx)
		if err != nil {
			return err
		}
		res, err = apply(tx, found, locked, site)
		return err
	})
	return res, err
}

// collect walks the content types page by page and adds the additional
// addresses.
func collect(ctx context.Context, reg *signpost.Registry, settings models.Settings) (collected, error) {
	out := collected{unpublished: map[slotKey]bool{}}
	seen := map[slotKey]bool{}
	add := func(w wanted) {
		if seen[w.key] {
			return
		}
		seen[w.key] = true
		out.rows = append(out.rows, w)
	}
	unprefixed := models.Text(settings.DefaultLocale)
	if reg != nil {
		for _, typ := range reg.ContentTypes() {
			slug := DefaultSlug(typ.Key)
			after := ""
			for {
				page, err := reg.Records(ctx, typ.Key, after, recordsPage, unprefixed)
				if err != nil {
					return out, fmt.Errorf("golem15.sitemap: records of %s: %w", typ.Key, err)
				}
				for _, rec := range page.Records {
					if !rec.Published {
						out.unpublished[slotKey{typ.Key, rec.ID, ""}] = true
						continue
					}
					for _, l := range rec.Locales {
						if strings.TrimSpace(l.URL) == "" {
							continue
						}
						add(wanted{
							key:      slotKey{typ.Key, rec.ID, l.Locale},
							grp:      slug,
							url:      l.URL,
							title:    l.Title,
							firstMod: rec.LastMod,
						})
					}
				}
				if page.Next == "" || page.Next == after {
					break
				}
				after = page.Next
			}
		}
	}
	for _, e := range settings.Extra() {
		u := strings.TrimSpace(e.URL)
		if !models.AllowedAddress(u) {
			continue
		}
		mod := ParseDate(e.LastMod)
		add(wanted{
			key:      slotKey{models.ExtraTypeKey, ExtraRecordID(u), ""},
			grp:      models.ExtraGroup,
			url:      u,
			extra:    true,
			extraMod: mod,
			firstMod: mod,
		})
	}
	return out, nil
}

// ParseDate reads a date as the admin's date picker or an import sends it
// (RFC 3339, "2006-01-02 15:04:05" or "2006-01-02", UTC); nil when empty or
// unreadable.
func ParseDate(s string) *time.Time {
	s = strings.TrimSpace(s)
	if s == "" {
		return nil
	}
	for _, layout := range []string{time.RFC3339Nano, "2006-01-02 15:04:05", "2006-01-02T15:04:05", "2006-01-02"} {
		if t, err := time.Parse(layout, s); err == nil {
			t = t.UTC()
			return &t
		}
	}
	return nil
}

// apply writes the found addresses into the slot table and numbers the
// files, inside the rebuild's transaction.
func apply(tx *gorm.DB, found collected, settings models.Settings, site string) (RebuildResult, error) {
	var res RebuildResult
	var existing []models.URL
	if err := tx.Order("id ASC").Find(&existing).Error; err != nil {
		return res, err
	}
	now := time.Now().UTC().Truncate(time.Microsecond)
	byKey := make(map[slotKey]*models.URL, len(existing))
	for i := range existing {
		r := &existing[i]
		byKey[slotKey{r.TypeKey, r.RecordID, r.Locale}] = r
	}
	dirty := map[uint64]bool{}
	var created []*models.URL
	matched := map[slotKey]bool{}
	for _, w := range found.rows {
		matched[w.key] = true
		r, ok := byKey[w.key]
		if !ok {
			mod := w.firstMod
			if mod == nil && !w.extra {
				mod = &now
			}
			row := &models.URL{
				Grp:            w.grp,
				TypeKey:        w.key.typeKey,
				RecordID:       w.key.recordID,
				Locale:         w.key.locale,
				URL:            w.url,
				Title:          w.title,
				FirstSeen:      now,
				LastMod:        utcPtr(mod),
				Alternates:     lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true},
				Images:         lagoon.Jsonable[[]string]{Data: []string{}, Valid: true},
				ExcludedReason: w.reason,
				UpdatedAt:      now,
			}
			created = append(created, row)
			if w.reason == "" {
				res.Added = append(res.Added, Absolute(site, w.url))
			}
			continue
		}
		wasListed := r.ExcludedReason == ""
		before := *r
		if r.Grp != w.grp {
			r.Grp = w.grp
			r.FileNo = nil
		}
		r.URL = w.url
		r.Title = w.title
		r.ExcludedReason = w.reason
		r.Alternates = lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true}
		if w.extra {
			r.LastMod = utcPtr(w.extraMod)
		}
		listed := w.reason == ""
		switch {
		case listed && !wasListed:
			res.Added = append(res.Added, Absolute(site, r.URL))
		case !listed && wasListed:
			res.Removed = append(res.Removed, Absolute(site, before.URL))
		case listed && before.URL != r.URL:
			res.Changed = append(res.Changed, Absolute(site, r.URL))
		}
		if rowChanged(before, *r) {
			dirty[r.ID] = true
		}
	}
	for i := range existing {
		r := &existing[i]
		key := slotKey{r.TypeKey, r.RecordID, r.Locale}
		if matched[key] {
			continue
		}
		reason := models.ReasonDeleted
		if found.unpublished[slotKey{r.TypeKey, r.RecordID, ""}] {
			reason = models.ReasonUnpublished
		}
		if r.ExcludedReason == reason {
			continue
		}
		if r.ExcludedReason == "" {
			res.Removed = append(res.Removed, Absolute(site, r.URL))
		}
		r.ExcludedReason = reason
		dirty[r.ID] = true
	}
	if len(created) > 0 {
		if err := tx.CreateInBatches(created, 500).Error; err != nil {
			return res, err
		}
	}
	size := settings.EffectiveFileSize()
	renumber := settings.NumberedSize == nil || *settings.NumberedSize != size
	slots := make([]*Slot, 0, len(existing)+len(created))
	rowsByID := make(map[uint64]*models.URL, len(existing)+len(created))
	for i := range existing {
		r := &existing[i]
		rowsByID[r.ID] = r
		slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""})
	}
	for _, r := range created {
		rowsByID[r.ID] = r
		slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""})
	}
	AssignSlots(slots, size, renumber)
	files := map[string]bool{}
	for _, s := range slots {
		r := rowsByID[s.ID]
		if !sameInt(r.FileNo, s.FileNo) {
			r.FileNo = s.FileNo
			dirty[r.ID] = true
		}
		if s.Listed {
			res.Listed++
			if s.FileNo != nil {
				files[FileName(s.Grp, *s.FileNo)] = true
			}
		}
	}
	res.Files = len(files)
	for id := range dirty {
		r := rowsByID[id]
		r.UpdatedAt = now
		if err := tx.Model(&models.URL{}).Where("id = ?", id).Updates(map[string]any{
			"grp":             r.Grp,
			"url":             r.URL,
			"title":           r.Title,
			"file_no":         r.FileNo,
			"lastmod":         r.LastMod,
			"alternates":      r.Alternates,
			"excluded_reason": r.ExcludedReason,
			"updated_at":      r.UpdatedAt,
		}).Error; err != nil {
			return res, err
		}
	}
	if err := tx.Exec(`UPDATE golem15_sitemap_settings SET numbered_size = ?, built_at = ? WHERE id = 1`, size, now).Error; err != nil {
		return res, err
	}
	return res, nil
}

// rowChanged reports whether a rebuild changed a stored row.
func rowChanged(a, b models.URL) bool {
	if a.Grp != b.Grp || a.URL != b.URL || a.Title != b.Title || a.ExcludedReason != b.ExcludedReason || !sameInt(a.FileNo, b.FileNo) || !sameTime(a.LastMod, b.LastMod) {
		return true
	}
	av, _ := a.Alternates.Value()
	bv, _ := b.Alternates.Value()
	return av != bv
}

func sameInt(a, b *int) bool {
	return (a == nil && b == nil) || (a != nil && b != nil && *a == *b)
}

func sameTime(a, b *time.Time) bool {
	return (a == nil && b == nil) || (a != nil && b != nil && a.Equal(*b))
}

func utcPtr(t *time.Time) *time.Time {
	if t == nil {
		return nil
	}
	u := t.UTC().Truncate(time.Microsecond)
	return &u
}

func nonNil[T any](s []T) []T {
	if s == nil {
		return []T{}
	}
	return s
}