package classes import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "strings" "time" "git.golem15.com/golem15/sm-sitemap-plugin/contract" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/lagoon" "git.golem15.com/golem15/summercms/modules/signpost" "gorm.io/gorm" ) // RebuildKind is the River kind of the rebuild job. const RebuildKind = "golem15.sitemap.rebuild" // RebuildArgs are the arguments of the rebuild job: it has none, a rebuild // always covers every content type and the additional addresses. type RebuildArgs struct{} // Kind is RebuildKind. func (RebuildArgs) Kind() string { return RebuildKind } // recordsPage is the page size the rebuild asks the content types for. const recordsPage = 500 // ErrNoDatabase is returned when the application publishes no database. var ErrNoDatabase = errors.New("golem15.sitemap: database is not configured") // RebuildResult is what one rebuild changed, as absolute URLs: Added are // newly listed addresses, Changed listed addresses whose URL changed and // Removed addresses that left the files. Listed counts the listed // addresses and Files the files they fill. type RebuildResult struct { Added []string Changed []string Removed []string Listed int Files int } // DefaultSlug is the file name part of a content type's group: the last // segment of its key in plural form (journal-post: posts, journal-category: // categories, seo-page: pages). func DefaultSlug(key string) string { key = strings.ToLower(strings.TrimSpace(key)) if i := strings.LastIndexAny(key, "-_."); i >= 0 { key = key[i+1:] } var b strings.Builder for _, r := range key { if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { b.WriteRune(r) } } s := b.String() switch { case s == "": return "pages" case strings.HasSuffix(s, "y") && len(s) > 1 && !strings.ContainsRune("aeiou", rune(s[len(s)-2])): return s[:len(s)-1] + "ies" case strings.HasSuffix(s, "s"): return s } return s + "s" } // ExtraRecordID is the record_id of an additional address: the hex SHA-256 // of the address. func ExtraRecordID(u string) string { sum := sha256.Sum256([]byte(u)) return hex.EncodeToString(sum[:]) } // slotKey identifies a row of the slot table. type slotKey struct { typeKey, recordID, locale string } // wanted is one address a rebuild found: its group, its row key, its path // and title, its first lastmod (for a new row) and the reason it is left // out ("" when listed). An additional address carries its own lastmod, // which the editor sets, so it is written on every rebuild. type wanted struct { key slotKey grp string url string title string firstMod *time.Time reason string extra bool extraMod *time.Time alternates []models.Alternate } // collected is what a rebuild read from the content types and settings. type collected struct { rows []wanted unpublished map[slotKey]bool } // Rebuild reads every content type's published records (signpost Records, // D-93) and the additional addresses, writes them into the slot table and // numbers the files, in one transaction that holds the settings row, so // two rebuilds never interleave. A record that is gone or unpublished // leaves its rows in the table with a reason, as a hole; only a changed // file size renumbers the files (D-30). func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) { var res RebuildResult if app == nil { return res, ErrNoDatabase } db, ok := app.Lookup[*gorm.DB]() if !ok || db == nil { return res, ErrNoDatabase } db = db.WithContext(ctx) settings, err := LoadSettings(db) if err != nil { return res, err } types, err := LoadTypeSettings(db) if err != nil { return res, err } reg, _ := app.Lookup[*signpost.Registry]() found, err := collect(ctx, reg, settings, types) if err != nil { return res, err } site := SiteURL(app, settings) err = db.Transaction(func(tx *gorm.DB) error { if err := tx.Exec(`SELECT id FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Error; err != nil { return err } locked, err := LoadSettings(tx) if err != nil { return err } res, err = apply(tx, found, locked, site) return err }) return res, err } // collect walks the content types page by page and adds the additional // addresses. The records of a type switched off in the settings are kept // in the table with the reason contract.ReasonType, so the settings screen // counts them. func collect(ctx context.Context, reg *signpost.Registry, settings models.Settings, types map[string]models.TypeSetting) (collected, error) { out := collected{unpublished: map[slotKey]bool{}} seen := map[slotKey]bool{} add := func(w wanted) { if seen[w.key] { return } seen[w.key] = true out.rows = append(out.rows, w) } unprefixed := models.Text(settings.DefaultLocale) if reg != nil { for _, typ := range reg.ContentTypes() { cfg := EffectiveType(typ, types) reason := "" if !cfg.Include { reason = contract.ReasonType } after := "" for { page, err := reg.Records(ctx, typ.Key, after, recordsPage, unprefixed) if err != nil { return out, fmt.Errorf("golem15.sitemap: records of %s: %w", typ.Key, err) } for _, rec := range page.Records { if !rec.Published { out.unpublished[slotKey{typ.Key, rec.ID, ""}] = true continue } for _, l := range rec.Locales { if strings.TrimSpace(l.URL) == "" { continue } add(wanted{ key: slotKey{typ.Key, rec.ID, l.Locale}, grp: cfg.Slug, url: l.URL, title: l.Title, firstMod: rec.LastMod, reason: reason, }) } } if page.Next == "" || page.Next == after { break } after = page.Next } } } for _, e := range settings.Extra() { u := strings.TrimSpace(e.URL) if !models.AllowedAddress(u) { continue } mod := models.ParseDate(e.LastMod) add(wanted{ key: slotKey{models.ExtraTypeKey, ExtraRecordID(u), ""}, grp: models.ExtraGroup, url: u, extra: true, extraMod: mod, firstMod: mod, }) } return out, nil } // apply writes the found addresses into the slot table and numbers the // files, inside the rebuild's transaction. func apply(tx *gorm.DB, found collected, settings models.Settings, site string) (RebuildResult, error) { var res RebuildResult var existing []models.URL if err := tx.Order("id ASC").Find(&existing).Error; err != nil { return res, err } now := time.Now().UTC().Truncate(time.Microsecond) byKey := make(map[slotKey]*models.URL, len(existing)) for i := range existing { r := &existing[i] byKey[slotKey{r.TypeKey, r.RecordID, r.Locale}] = r } dirty := map[uint64]bool{} var created []*models.URL matched := map[slotKey]bool{} for _, w := range found.rows { matched[w.key] = true r, ok := byKey[w.key] if !ok { mod := w.firstMod if mod == nil && !w.extra { mod = &now } row := &models.URL{ Grp: w.grp, TypeKey: w.key.typeKey, RecordID: w.key.recordID, Locale: w.key.locale, URL: w.url, Title: w.title, FirstSeen: now, LastMod: utcPtr(mod), Alternates: lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true}, Images: lagoon.Jsonable[[]string]{Data: []string{}, Valid: true}, ExcludedReason: w.reason, UpdatedAt: now, } created = append(created, row) if w.reason == "" { res.Added = append(res.Added, Absolute(site, w.url)) } continue } wasListed := r.ExcludedReason == "" before := *r if r.Grp != w.grp { r.Grp = w.grp r.FileNo = nil } r.URL = w.url r.Title = w.title r.ExcludedReason = w.reason r.Alternates = lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true} if w.extra { r.LastMod = utcPtr(w.extraMod) } listed := w.reason == "" switch { case listed && !wasListed: res.Added = append(res.Added, Absolute(site, r.URL)) case !listed && wasListed: res.Removed = append(res.Removed, Absolute(site, before.URL)) case listed && before.URL != r.URL: res.Changed = append(res.Changed, Absolute(site, r.URL)) } if rowChanged(before, *r) { dirty[r.ID] = true } } for i := range existing { r := &existing[i] key := slotKey{r.TypeKey, r.RecordID, r.Locale} if matched[key] { continue } reason := models.ReasonDeleted if found.unpublished[slotKey{r.TypeKey, r.RecordID, ""}] { reason = models.ReasonUnpublished } if r.ExcludedReason == reason { continue } if r.ExcludedReason == "" { res.Removed = append(res.Removed, Absolute(site, r.URL)) } r.ExcludedReason = reason dirty[r.ID] = true } if len(created) > 0 { if err := tx.CreateInBatches(created, 500).Error; err != nil { return res, err } } size := settings.EffectiveFileSize() renumber := settings.NumberedSize == nil || *settings.NumberedSize != size slots := make([]*Slot, 0, len(existing)+len(created)) rowsByID := make(map[uint64]*models.URL, len(existing)+len(created)) for i := range existing { r := &existing[i] rowsByID[r.ID] = r slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""}) } for _, r := range created { rowsByID[r.ID] = r slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""}) } AssignSlots(slots, size, renumber) files := map[string]bool{} for _, s := range slots { r := rowsByID[s.ID] if !sameInt(r.FileNo, s.FileNo) { r.FileNo = s.FileNo dirty[r.ID] = true } if s.Listed { res.Listed++ if s.FileNo != nil { files[FileName(s.Grp, *s.FileNo)] = true } } } res.Files = len(files) for id := range dirty { r := rowsByID[id] r.UpdatedAt = now if err := tx.Model(&models.URL{}).Where("id = ?", id).Updates(map[string]any{ "grp": r.Grp, "url": r.URL, "title": r.Title, "file_no": r.FileNo, "lastmod": r.LastMod, "alternates": r.Alternates, "excluded_reason": r.ExcludedReason, "updated_at": r.UpdatedAt, }).Error; err != nil { return res, err } } if err := tx.Exec(`UPDATE golem15_sitemap_settings SET numbered_size = ?, built_at = ? WHERE id = 1`, size, now).Error; err != nil { return res, err } return res, nil } // rowChanged reports whether a rebuild changed a stored row. func rowChanged(a, b models.URL) bool { if a.Grp != b.Grp || a.URL != b.URL || a.Title != b.Title || a.ExcludedReason != b.ExcludedReason || !sameInt(a.FileNo, b.FileNo) || !sameTime(a.LastMod, b.LastMod) { return true } av, _ := a.Alternates.Value() bv, _ := b.Alternates.Value() return av != bv } func sameInt(a, b *int) bool { return (a == nil && b == nil) || (a != nil && b != nil && *a == *b) } func sameTime(a, b *time.Time) bool { return (a == nil && b == nil) || (a != nil && b != nil && a.Equal(*b)) } func utcPtr(t *time.Time) *time.Time { if t == nil { return nil } u := t.UTC().Truncate(time.Microsecond) return &u } func nonNil[T any](s []T) []T { if s == nil { return []T{} } return s }