feat: migrate definitions into additional addresses

- migration 202610280002 copies the served definition's url items into
  extra_urls, frontend_url into site_url and frontend_default_locale into
  default_locale; reference items are dropped
- the definitions table and the Phase 17 columns keep their data, so a
  rollback loses nothing (TestMigrateDefinitions)

sitemap_index_test.go

@@ -9,7 +9,11 @@ import (
 	"time"
 
 	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
+	"git.golem15.com/golem15/sm-sitemap-plugin/models"
+	"git.golem15.com/golem15/sm-sitemap-plugin/updates"
+	"git.golem15.com/golem15/summercms/modules/lagoon"
 	"git.golem15.com/golem15/summercms/modules/pact"
+	"github.com/go-gormigrate/gormigrate/v2"
 )
 
 // bootIndex boots the fake content plugin and the sitemap on a fresh
@@ -304,3 +308,97 @@ func TestSettingsSingleton(t *testing.T) {
 		}
 	}
 }
+
+// TestMigrateDefinitions pins the D-32 data move of migration
+// 202610280002: the url items of the served definition (theme default)
+// become the additional addresses as given (no lastmod, unsafe and repeated
+// ones skipped), frontend_url becomes site_url and frontend_default_locale
+// default_locale; reference items are dropped, the definitions table and
+// the Phase 17 columns keep their data, and a rollback removes only the
+// copied values.
+func TestMigrateDefinitions(t *testing.T) {
+	sqlDB := freshDB(t)
+	gdb, err := lagoon.Use(t.Context(), sqlDB)
+	if err != nil {
+		t.Fatal(err)
+	}
+	all := updates.All()
+	opts := &gormigrate.Options{TableName: "sitemap_migrations", IDColumnName: "id", IDColumnSize: 255, UseTransaction: true}
+	upTo := 0
+	for i, m := range all {
+		if m.ID == "202610110002_create_golem15_sitemap_settings" {
+			upTo = i + 1
+		}
+	}
+	if upTo == 0 {
+		t.Fatal("the Phase 17 settings migration is missing")
+	}
+	if err := gormigrate.New(gdb, opts, all[:upTo]).Migrate(); err != nil {
+		t.Fatal(err)
+	}
+	items := `[{"type":"url","url":"/","changefreq":"always","priority":"1.0"},` +
+		`{"type":"acme-post","reference":"7","nesting":true},` +
+		`{"type":"url","url":"/o-nas"},` +
+		`{"type":"url","url":"javascript:alert(1)"},` +
+		`{"type":"url","url":"/"},` +
+		`{"type":"url","url":"https://www.example.test/kontakt"}]`
+	for _, stmt := range []struct {
+		sql  string
+		args []any
+	}{
+		{`INSERT INTO golem15_sitemap_definitions (theme, data) VALUES (?, ?)`, []any{"default", items}},
+		{`INSERT INTO golem15_sitemap_definitions (theme, data) VALUES (?, ?)`, []any{"other", `[{"type":"url","url":"/other"}]`}},
+		{`UPDATE golem15_sitemap_settings SET frontend_url = ?, frontend_default_locale = ?, priority_blog = ? WHERE id = 1`, []any{"https://www.example.test/", "pl", "0.7"}},
+	} {
+		if err := gdb.Exec(stmt.sql, stmt.args...).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := gormigrate.New(gdb, opts, all).Migrate(); err != nil {
+		t.Fatal(err)
+	}
+	s, err := classes.LoadSettings(gdb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := extraURLs(s.Extra()); got != "/ /o-nas https://www.example.test/kontakt" {
+		t.Fatalf("extra_urls = %q", got)
+	}
+	for _, e := range s.Extra() {
+		if e.LastMod != "" {
+			t.Fatalf("a copied address has a lastmod: %+v", e)
+		}
+	}
+	if models.Text(s.SiteURL) != "https://www.example.test/" || models.Text(s.DefaultLocale) != "pl" || s.EffectiveFileSize() != 1000 {
+		t.Fatalf("settings = %+v", s)
+	}
+	var defs int64
+	gdb.Raw(`SELECT COUNT(*) FROM golem15_sitemap_definitions`).Scan(&defs)
+	var data string
+	gdb.Raw(`SELECT data FROM golem15_sitemap_definitions WHERE theme = 'default'`).Scan(&data)
+	if defs != 2 || data != items {
+		t.Fatalf("the definitions table changed: %d rows, data %s", defs, data)
+	}
+
+	if err := gormigrate.New(gdb, opts, all).RollbackTo("202610110002_create_golem15_sitemap_settings"); err != nil {
+		t.Fatal(err)
+	}
+	var old struct {
+		FrontendURL string `gorm:"column:frontend_url"`
+		Locale      string `gorm:"column:frontend_default_locale"`
+		Blog        string `gorm:"column:priority_blog"`
+	}
+	gdb.Raw(`SELECT frontend_url, frontend_default_locale, priority_blog FROM golem15_sitemap_settings WHERE id = 1`).Scan(&old)
+	if old.FrontendURL != "https://www.example.test/" || old.Locale != "pl" || old.Blog != "0.7" {
+		t.Fatalf("the rollback changed the Phase 17 settings: %+v", old)
+	}
+	gdb.Raw(`SELECT COUNT(*) FROM golem15_sitemap_definitions`).Scan(&defs)
+	if defs != 2 {
+		t.Fatalf("the rollback changed the definitions: %d rows", defs)
+	}
+	var columns int64
+	gdb.Raw(`SELECT COUNT(*) FROM information_schema.columns WHERE table_name = 'golem15_sitemap_settings' AND column_name IN ('site_url', 'extra_urls')`).Scan(&columns)
+	if columns != 0 {
+		t.Fatalf("the rollback left %d copied columns", columns)
+	}
+}

updates/202610280002_update_golem15_sitemap_settings.go

@@ -1,6 +1,10 @@
 package updates
 
 import (
+	"encoding/json"
+	"strings"
+
+	"git.golem15.com/golem15/sm-sitemap-plugin/models"
 	"github.com/go-gormigrate/gormigrate/v2"
 	"gorm.io/gorm"
 )
@@ -33,7 +37,10 @@ var updateSettings = []*gormigrate.Migration{
 					return err
 				}
 			}
-			return tx.Exec(`INSERT INTO golem15_sitemap_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING`).Error
+			if err := tx.Exec(`INSERT INTO golem15_sitemap_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING`).Error; err != nil {
+				return err
+			}
+			return copyDefinitions(tx)
 		},
 		Rollback: func(tx *gorm.DB) error {
 			for _, name := range []string{"site_url", "file_size", "default_locale", "extra_urls", "rebuild_pending_until", "numbered_size", "built_at", "rebuild_requested_at"} {
@@ -49,3 +56,56 @@ var updateSettings = []*gormigrate.Migration{
 func init() {
 	Register(updateSettings...)
 }
+
+// copyDefinitions moves the Phase 17 data into the new columns (D-32): the
+// url items of the definition /sitemap.xml served (theme default, the
+// first one) become the additional addresses as given, without a lastmod
+// (an address that models.AllowedAddress refuses, or one already copied,
+// is skipped); frontend_url becomes site_url when models.AllowedFrontendURL
+// accepts it, and frontend_default_locale default_locale. Reference items
+// are dropped: the content types list their records. The definitions table
+// and the Phase 17 columns are left as they are, so a rollback (which drops
+// the new columns) keeps every value.
+func copyDefinitions(tx *gorm.DB) error {
+	var data []string
+	if err := tx.Raw(`SELECT COALESCE(data, '') FROM golem15_sitemap_definitions WHERE theme = 'default' ORDER BY id ASC LIMIT 1`).Scan(&data).Error; err != nil {
+		return err
+	}
+	rows := []models.ExtraURL{}
+	if len(data) > 0 {
+		items, _, err := models.ParseLegacyItems([]byte(data[0]))
+		if err != nil {
+			// A definition the Phase 17 code could not read either: nothing
+			// is copied, and the row stays for a manual look.
+			items = nil
+		}
+		seen := map[string]bool{}
+		for _, item := range items {
+			u := strings.TrimSpace(item.URL)
+			if item.Type != models.LegacyItemURL || seen[u] || !models.AllowedAddress(u) {
+				continue
+			}
+			seen[u] = true
+			rows = append(rows, models.ExtraURL{URL: u})
+		}
+	}
+	raw, err := json.Marshal(rows)
+	if err != nil {
+		return err
+	}
+	var old struct {
+		FrontendURL *string `gorm:"column:frontend_url"`
+		Locale      *string `gorm:"column:frontend_default_locale"`
+	}
+	if err := tx.Raw(`SELECT frontend_url, frontend_default_locale FROM golem15_sitemap_settings WHERE id = 1`).Scan(&old).Error; err != nil {
+		return err
+	}
+	var site, locale any
+	if u := strings.TrimSpace(models.Text(old.FrontendURL)); u != "" && models.AllowedFrontendURL(u) {
+		site = u
+	}
+	if l := strings.TrimSpace(models.Text(old.Locale)); l != "" {
+		locale = l
+	}
+	return tx.Exec(`UPDATE golem15_sitemap_settings SET extra_urls = ?::jsonb, site_url = ?, default_locale = ? WHERE id = 1`, string(raw), site, locale).Error
+}