feat(17-03): full sitemap generation, settings, hooks and import commands

- generator in Definition::addItemToSet order: 50000 URL cap, lastmod from
  the record or the updates manifest (regular file, 5 MiB cap, fail safe),
  BeforeAddItem (skip) and AddItem (extra XML) festival events, tier or item
  priority, per-alternate expansion of list type children
- priority tiers from golem15.sitemap.priority_tiers, compiled at boot,
  defaults equal to the PHP getPriorityForUrl rules
- frontend_url rewrite of loc, xhtml:link href and the stylesheet href;
  GET /sitemap.xsl serves the PHP stylesheet byte for byte
- golem15_sitemap_settings singleton and Settings model
- sitemap:import (PHP ImportCommand) and sitemap:import-winter with the
  read-only Winter export SQL
- urlto and full goldens generated from PHP; TestSitemapImportSmoke
This commit is contained in:
2026-10-12 17:07:39 +02:00
parent fe39e26ce0
commit 4a82240429
24 changed files with 2970 additions and 54 deletions

75
console/helpers.go Normal file
View File

@@ -0,0 +1,75 @@
// Package console holds the sitemap plugin's commands: sitemap:import (the
// PHP ImportCommand) and sitemap:import-winter (a copy of a WinterCMS
// site's definition and settings).
package console
import (
"context"
"fmt"
"strings"
"git.golem15.com/golem15/sm-sitemap-plugin/models"
"git.golem15.com/golem15/summercms/modules/backpack"
"git.golem15.com/golem15/summercms/modules/bonfire"
"git.golem15.com/golem15/summercms/modules/lagoon"
"gorm.io/gorm"
)
func flagValue(in bonfire.Input, name string) string {
v, ok := in.Flag(name)
if !ok {
return ""
}
return strings.TrimSpace(v)
}
func flagOn(in bonfire.Input, name string) bool {
switch strings.ToLower(flagValue(in, name)) {
case "true", "1", "yes":
return true
}
return false
}
// lookupDB returns the app database. The generated main.go publishes no
// database for plugin commands, so when none is published it opens
// database.dsn and publishes it the way lagoon's migrate does. The returned
// func closes a pool this call opened; it is a no-op otherwise.
func lookupDB(ctx context.Context, app *backpack.App) (*gorm.DB, func(), error) {
if app == nil {
return nil, nil, fmt.Errorf("golem15.sitemap: app is nil")
}
if db, ok := app.Lookup[*gorm.DB](); ok && db != nil {
return db, func() {}, nil
}
if lagoon.DSN(app.Config) == "" {
return nil, nil, fmt.Errorf("golem15.sitemap: database is not configured (set SUMMER_DATABASE__DSN)")
}
sqlDB, gdb, err := lagoon.OpenFromApp(ctx, app)
if err != nil {
return nil, nil, err
}
if err := lagoon.Publish(app, sqlDB, gdb); err != nil {
_ = sqlDB.Close()
return nil, nil, err
}
return gdb, func() { _ = sqlDB.Close() }, nil
}
// singletonDefinition loads the definition with the fixed theme key,
// creating it when there is none (PHP Definition::firstOrCreate).
func singletonDefinition(tx *gorm.DB) (models.Definition, error) {
var defs []models.Definition
if err := tx.Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&defs).Error; err != nil {
return models.Definition{}, err
}
if len(defs) > 0 {
return defs[0], nil
}
theme := models.DefinitionTheme
def := models.Definition{Theme: &theme}
if err := tx.Create(&def).Error; err != nil {
return models.Definition{}, err
}
return def, nil
}

187
console/import.go Normal file
View File

@@ -0,0 +1,187 @@
package console
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"git.golem15.com/golem15/sm-sitemap-plugin/models"
"git.golem15.com/golem15/summercms/modules/backpack"
"git.golem15.com/golem15/summercms/modules/bonfire"
"gorm.io/gorm"
)
// Import is sitemap:import FILE [--theme=] [--append], the PHP
// ImportCommand: it reads a JSON array of URL entries ({"url": ...,
// "changefreq": ..., "priority": ...}; changefreq defaults to monthly,
// priority to 0.5) into the sitemap definition. Without --append the
// definition keeps its non-url items first and the imported URLs replace
// its url items; with --append the URLs are added after every existing item.
// The definition is always the one /sitemap.xml serves (theme default);
// --theme is accepted for PHP compatibility and a theme other than default
// is reported.
func Import(app *backpack.App) bonfire.Command {
return bonfire.Command{
Name: "sitemap:import",
Description: "Import sitemap URLs from a JSON manifest file",
Args: []bonfire.Arg{
{Name: "file", Description: "Path to the JSON manifest (relative to the working directory or absolute)", Required: true},
},
Flags: []bonfire.Flag{
{Name: "theme", Description: "Theme code of the PHP plugin; /sitemap.xml serves only the default definition"},
{Name: "append", Description: "Add URLs without removing existing url-type items", Bare: true},
},
Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
file, _ := in.Argument("file")
db, done, err := lookupDB(ctx, app)
if err != nil {
return err
}
defer done()
return ImportFile(ctx, db, file, flagValue(in, "theme"), flagOn(in, "append"), out)
},
}
}
// ImportFile runs sitemap:import against db.
func ImportFile(ctx context.Context, db *gorm.DB, file, theme string, appendMode bool, out bonfire.Output) error {
raw, err := os.ReadFile(file)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("File not found: %s", file)
}
return fmt.Errorf("Cannot read %s: %w", file, err)
}
entries, labels, ok := orderedEntries(raw)
if !ok {
return errors.New("Invalid JSON: expected an array of URL entries")
}
newItems := make([]models.DefinitionItem, 0, len(entries))
for i, entry := range entries {
var fields map[string]json.RawMessage
_ = json.Unmarshal(entry, &fields)
url, err := models.DecodeText(fields["url"])
if fields["url"] == nil || err != nil || phpEmpty(fields["url"]) {
return fmt.Errorf("Entry %s is missing a 'url' field", labels[i])
}
item := models.DefinitionItem{Type: models.ItemTypeURL, URL: url}
item.Changefreq = textOr(fields["changefreq"], "monthly")
item.Priority = textOr(fields["priority"], "0.5")
newItems = append(newItems, item)
}
var kept int
err = db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
def, err := singletonDefinition(tx)
if err != nil {
return err
}
existing := def.Items()
if !appendMode {
var dynamic []models.DefinitionItem
for _, item := range existing {
if item.Type != models.ItemTypeURL {
dynamic = append(dynamic, item)
}
}
existing = dynamic
}
kept = len(existing)
def.SetItems(append(append([]models.DefinitionItem{}, existing...), newItems...))
return tx.Save(&def).Error
})
if err != nil {
return err
}
out.Info(fmt.Sprintf("Imported %d URLs into sitemap definition for theme '%s'.", len(newItems), models.DefinitionTheme))
if kept > 0 {
label := "existing non-URL (dynamic)"
if appendMode {
label = "existing"
}
out.Info(fmt.Sprintf("Kept %d %s items.", kept, label))
}
if theme != "" && theme != models.DefinitionTheme {
out.Warning(fmt.Sprintf("Theme '%s' was given, but /sitemap.xml serves only the '%s' definition, so the URLs were imported there.", theme, models.DefinitionTheme))
}
return nil
}
// orderedEntries splits a JSON array, or a JSON object in key order (a PHP
// array with keys), into its values with their PHP labels (index or key).
func orderedEntries(raw []byte) ([]json.RawMessage, []string, bool) {
raw = bytes.TrimSpace(raw)
if len(raw) == 0 {
return nil, nil, false
}
switch raw[0] {
case '[':
var list []json.RawMessage
if err := json.Unmarshal(raw, &list); err != nil {
return nil, nil, false
}
labels := make([]string, len(list))
for i := range list {
labels[i] = fmt.Sprint(i)
}
return list, labels, true
case '{':
dec := json.NewDecoder(bytes.NewReader(raw))
if _, err := dec.Token(); err != nil {
return nil, nil, false
}
var list []json.RawMessage
var labels []string
for dec.More() {
tok, err := dec.Token()
if err != nil {
return nil, nil, false
}
key, _ := tok.(string)
var v json.RawMessage
if err := dec.Decode(&v); err != nil {
return nil, nil, false
}
list = append(list, v)
labels = append(labels, key)
}
return list, labels, true
}
return nil, nil, false
}
// phpEmpty is PHP empty() of a decoded JSON value.
func phpEmpty(raw json.RawMessage) bool {
var v any
if err := json.Unmarshal(raw, &v); err != nil {
return true
}
switch t := v.(type) {
case nil:
return true
case string:
return t == "" || t == "0"
case bool:
return !t
case float64:
return t == 0
case []any:
return len(t) == 0
case map[string]any:
return len(t) == 0
}
return false
}
// textOr is PHP `$entry[key] ?? fallback` as text: a missing or null value
// is the fallback.
func textOr(raw json.RawMessage, fallback string) *string {
if raw != nil {
if s, err := models.DecodeText(raw); err == nil && s != nil {
return s
}
}
return &fallback
}

407
console/import_winter.go Normal file
View File

@@ -0,0 +1,407 @@
package console
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"slices"
"sort"
"strings"
"git.golem15.com/golem15/sm-sitemap-plugin/models"
"git.golem15.com/golem15/summercms/modules/backpack"
"git.golem15.com/golem15/summercms/modules/bonfire"
"git.golem15.com/golem15/summercms/modules/signpost"
"gorm.io/gorm"
)
// droppedTypes are the WinterCMS item types the port does not support:
// Winter.Pages static pages and CMS pages.
var droppedTypes = map[string]bool{
"static-page": true,
"all-static-pages": true,
"cms-page": true,
}
// WinterOptions are the inputs of sitemap:import-winter. DefinitionsFile is
// the output of scripts/winter-export-definitions.sql (one JSON object per
// line: id, theme, data, created_at, updated_at); SettingsFile the output
// of scripts/winter-export-settings.sql (the JSON object of the
// golem15_sitemap_settings system setting). Theme picks a definition when
// the export holds several. Maps rename item types (old to new) before they
// are checked. Force replaces a definition that has items; DryRun reports
// without writing.
type WinterOptions struct {
DefinitionsFile string
SettingsFile string
Theme string
Maps map[string]string
Force bool
DryRun bool
}
// WinterReport is what sitemap:import-winter did: the items kept, the
// report lines of every dropped or remapped item, and whether the settings
// were copied.
type WinterReport struct {
Kept int
Dropped int
Remapped int
Lines []string
Settings bool
}
// ImportWinter is sitemap:import-winter: it copies a WinterCMS site's
// sitemap definition and settings into the definition /sitemap.xml serves
// (theme default) and the settings row. Item types are mapped (url kept,
// static and CMS page types dropped, --map old=new renames applied first),
// an item of a registry type that takes references is dropped when its
// reference is not among the type's current references, an item of an
// unknown type is dropped, and every dropped or remapped item is printed. A
// definition that already has items is refused without --force. Settings
// keys missing from the export take the PHP defaults.
func ImportWinter(app *backpack.App) bonfire.Command {
return bonfire.Command{
Name: "sitemap:import-winter",
Description: "Copy a WinterCMS sitemap definition and settings export into this application",
Flags: []bonfire.Flag{
{Name: "definitions", Description: "Definitions export (JSON lines from scripts/winter-export-definitions.sql)"},
{Name: "settings", Description: "Settings export (the JSON object from scripts/winter-export-settings.sql)"},
{Name: "theme", Description: "The PHP theme whose definition to copy when the export holds several"},
{Name: "map", Description: "Rename an item type, old=new (repeatable)", Repeatable: true},
{Name: "force", Description: "Replace a definition that already has items", Bare: true},
{Name: "dry-run", Description: "Report what would be copied without writing", Bare: true},
},
Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
maps, err := parseMaps(in.Flags("map"))
if err != nil {
return err
}
opts := WinterOptions{
DefinitionsFile: flagValue(in, "definitions"),
SettingsFile: flagValue(in, "settings"),
Theme: flagValue(in, "theme"),
Maps: maps,
Force: flagOn(in, "force"),
DryRun: flagOn(in, "dry-run"),
}
db, done, err := lookupDB(ctx, app)
if err != nil {
return err
}
defer done()
var reg *signpost.Registry
if app != nil {
reg, _ = app.Lookup[*signpost.Registry]()
}
_, err = ImportWinterFiles(ctx, db, reg, opts, out)
return err
},
}
}
// parseMaps reads --map old=new values.
func parseMaps(values []string) (map[string]string, error) {
maps := map[string]string{}
for _, v := range values {
from, to, ok := strings.Cut(v, "=")
from, to = strings.TrimSpace(from), strings.TrimSpace(to)
if !ok || from == "" {
return nil, fmt.Errorf("--map %q: use old=new", v)
}
maps[from] = to
}
return maps, nil
}
// winterDefinition is one exported golem15_sitemap_definitions row.
type winterDefinition struct {
ID json.RawMessage `json:"id"`
Theme *string `json:"theme"`
Data json.RawMessage `json:"data"`
}
// ImportWinterFiles runs sitemap:import-winter against db; reg is the menu
// item type registry the item types are checked against (nil knows none).
func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, opts WinterOptions, out bonfire.Output) (WinterReport, error) {
var report WinterReport
if opts.DefinitionsFile == "" && opts.SettingsFile == "" {
return report, errors.New("give --definitions, --settings or both")
}
var (
items []models.DefinitionItem
haveDef bool
)
if opts.DefinitionsFile != "" {
row, err := pickDefinition(opts.DefinitionsFile, opts.Theme)
if err != nil {
return report, err
}
out.Info(fmt.Sprintf("Copying definition %s (theme '%s') into the '%s' definition.", strings.TrimSpace(string(row.ID)), models.Text(row.Theme), models.DefinitionTheme))
items, err = mapItems(ctx, reg, row.Data, opts.Maps, &report)
if err != nil {
return report, err
}
haveDef = true
}
var settings models.Settings
if opts.SettingsFile != "" {
var err error
settings, err = readWinterSettings(opts.SettingsFile, &report)
if err != nil {
return report, err
}
report.Settings = true
}
for _, line := range report.Lines {
out.Warning(line)
}
if haveDef {
var existing []models.Definition
if err := db.WithContext(ctx).Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&existing).Error; err != nil {
return report, err
}
if len(existing) > 0 && len(existing[0].Items()) > 0 && !opts.Force {
msg := fmt.Sprintf("The sitemap definition already has %d items; rerun with --force to replace them.", len(existing[0].Items()))
if !opts.DryRun {
return report, errors.New(msg)
}
out.Warning(msg)
}
}
summary := fmt.Sprintf("%d items kept, %d remapped, %d dropped.", report.Kept, report.Remapped, report.Dropped)
if opts.DryRun {
if haveDef {
out.Info("Dry run: " + summary)
}
if report.Settings {
out.Info("Dry run: the settings would be copied.")
}
out.Info("Dry run complete (no changes made).")
return report, nil
}
err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if haveDef {
def, err := singletonDefinition(tx)
if err != nil {
return err
}
def.SetItems(items)
if err := tx.Save(&def).Error; err != nil {
return err
}
}
if report.Settings {
settings.ID = 1
if err := tx.Save(&settings).Error; err != nil {
return err
}
}
return nil
})
if err != nil {
return report, err
}
if haveDef {
out.Info("Definition copied: " + summary)
}
if report.Settings {
out.Info("Settings copied.")
}
return report, nil
}
// pickDefinition reads the definitions export and returns the row of theme,
// or the only row when theme is empty.
func pickDefinition(file, theme string) (winterDefinition, error) {
f, err := os.Open(file)
if err != nil {
return winterDefinition{}, fmt.Errorf("File not found: %s", file)
}
defer f.Close()
var rows []winterDefinition
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 1<<20), 64<<20)
n := 0
for scanner.Scan() {
n++
line := bytes.TrimSpace(scanner.Bytes())
if len(line) == 0 {
continue
}
var row winterDefinition
if err := json.Unmarshal(line, &row); err != nil {
return winterDefinition{}, fmt.Errorf("%s line %d: %w", file, n, err)
}
rows = append(rows, row)
}
if err := scanner.Err(); err != nil {
return winterDefinition{}, err
}
if len(rows) == 0 {
return winterDefinition{}, fmt.Errorf("%s holds no definition", file)
}
if theme != "" {
for _, row := range rows {
if models.Text(row.Theme) == theme {
return row, nil
}
}
return winterDefinition{}, fmt.Errorf("%s holds no definition of theme '%s'", file, theme)
}
if len(rows) > 1 {
themes := make([]string, 0, len(rows))
for _, row := range rows {
themes = append(themes, "'"+models.Text(row.Theme)+"'")
}
return winterDefinition{}, fmt.Errorf("%s holds %d definitions (themes %s); pick one with --theme", file, len(rows), strings.Join(themes, ", "))
}
return rows[0], nil
}
// exportedJSON is a JSON value that the export may hold as text (a TEXT
// column selected as it is) or as JSON: a string is decoded once more.
func exportedJSON(raw json.RawMessage) json.RawMessage {
raw = bytes.TrimSpace(raw)
if len(raw) > 0 && raw[0] == '"' {
var text string
if json.Unmarshal(raw, &text) == nil {
return json.RawMessage(strings.TrimSpace(text))
}
}
return raw
}
// mapItems decodes the exported data and maps its items.
func mapItems(ctx context.Context, reg *signpost.Registry, data json.RawMessage, maps map[string]string, report *WinterReport) ([]models.DefinitionItem, error) {
data = exportedJSON(data)
if len(data) == 0 || bytes.Equal(data, []byte("null")) {
return []models.DefinitionItem{}, nil
}
rawItems, labels, ok := orderedEntries(data)
if !ok {
return nil, errors.New("the definition data is not a list of items")
}
out := []models.DefinitionItem{}
refs := map[string][]string{}
for i, raw := range rawItems {
var item models.DefinitionItem
if err := json.Unmarshal(raw, &item); err != nil {
return nil, fmt.Errorf("definition item %s: %w", labels[i], err)
}
var fields map[string]any
_ = json.Unmarshal(raw, &fields)
label := describeItem(labels[i], item)
if page, ok := fields["cmsPage"].(string); ok && page != "" {
report.Lines = append(report.Lines, fmt.Sprintf("%s: cmsPage '%s' dropped (CMS pages are not supported)", label, page))
}
if to, ok := maps[item.Type]; ok && to != item.Type {
report.Lines = append(report.Lines, fmt.Sprintf("%s: type '%s' mapped to '%s'", label, item.Type, to))
report.Remapped++
item.Type = to
if to == "" {
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (mapped to nothing)", label))
report.Dropped++
continue
}
}
if item.Type == models.ItemTypeURL {
out = append(out, item)
report.Kept++
continue
}
if droppedTypes[item.Type] {
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (static and CMS page types are not supported)", label))
report.Dropped++
continue
}
if _, known := reg.Type(item.Type); !known {
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (type '%s' is not registered; map it with --map %s=<type>)", label, item.Type, item.Type))
report.Dropped++
continue
}
values, cached := refs[item.Type]
if !cached {
set, err := signpost.ReferenceOptions(ctx, reg, item.Type)
if err != nil {
return nil, err
}
if set.Applies {
values = []string{}
for _, o := range set.Options {
values = append(values, o.Value)
}
}
refs[item.Type] = values
}
if values != nil && !slices.Contains(values, models.Text(item.Reference)) {
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (reference '%s' is not among the current references of '%s')", label, models.Text(item.Reference), item.Type))
report.Dropped++
continue
}
out = append(out, item)
report.Kept++
}
return out, nil
}
func describeItem(label string, item models.DefinitionItem) string {
s := fmt.Sprintf("item %s (type '%s'", label, item.Type)
if item.URL != nil && *item.URL != "" {
s += fmt.Sprintf(", url '%s'", *item.URL)
}
if item.Reference != nil && *item.Reference != "" {
s += fmt.Sprintf(", reference '%s'", *item.Reference)
}
return s + ")"
}
// readWinterSettings reads the exported settings object onto the PHP
// defaults.
func readWinterSettings(file string, report *WinterReport) (models.Settings, error) {
raw, err := os.ReadFile(file)
if err != nil {
return models.Settings{}, fmt.Errorf("File not found: %s", file)
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(exportedJSON(raw), &fields); err != nil {
return models.Settings{}, fmt.Errorf("%s: the settings must be a JSON object: %w", file, err)
}
s := models.DefaultSettings()
texts := map[string]**string{
"frontend_url": &s.FrontendURL,
"frontend_default_locale": &s.FrontendDefaultLocale,
"updates_manifest_path": &s.UpdatesManifestPath,
models.PriorityHomepage: &s.PriorityHomepage,
models.PriorityServices: &s.PriorityServices,
models.PriorityBlog: &s.PriorityBlog,
models.PriorityCategories: &s.PriorityCategories,
models.PriorityLegal: &s.PriorityLegal,
}
keys := make([]string, 0, len(fields))
for key := range fields {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
value := fields[key]
if dest, ok := texts[key]; ok {
text, err := models.DecodeText(value)
if err != nil {
return models.Settings{}, fmt.Errorf("%s: setting %s: %w", file, key, err)
}
*dest = text
continue
}
if key == "enable_priority_tiers" {
s.EnablePriorityTiers = !phpEmpty(value)
continue
}
report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (not a sitemap setting of this port)", key))
}
return s, nil
}