84cb27a810
BREAKING CHANGE: /sitemap.xsl is gone (404), the definitions screen and the Phase 17 settings tile are replaced by the settings singleton, and priority, change frequency, priority tiers and the update manifest are no longer written or read. The config keys golem15.sitemap.cache_seconds and golem15.sitemap.priority_tiers are removed; site_url, lastmod_ignore and rebuild_delay_seconds are new. - sitemap:import and sitemap:import-winter write the additional addresses (and site_url, default_locale) instead of the definition - the golem15_sitemap_definitions table stays in the database, unread
450 lines
16 KiB
Go
450 lines
16 KiB
Go
package console
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
|
|
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
|
"git.golem15.com/golem15/summercms/modules/backpack"
|
|
"git.golem15.com/golem15/summercms/modules/bonfire"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// droppedTypes are the WinterCMS item types the port does not support:
|
|
// Winter.Pages static pages and CMS pages.
|
|
var droppedTypes = map[string]bool{
|
|
"static-page": true,
|
|
"all-static-pages": true,
|
|
"cms-page": true,
|
|
}
|
|
|
|
// WinterOptions are the inputs of sitemap:import-winter. DefinitionsFile is
|
|
// the output of scripts/winter-export-definitions.sql (one JSON object per
|
|
// line: id, theme, data, created_at, updated_at); SettingsFile the output
|
|
// of scripts/winter-export-settings.sql (the JSON object of the
|
|
// golem15_sitemap_settings system setting). Theme picks a definition when
|
|
// the export holds several. Maps rename item types (old to new) before they
|
|
// are checked. PageURLs turns a static or CMS page item into an additional
|
|
// address: its key is the item's reference (the PHP page file name, for
|
|
// example blog), its value the page's URL (/blog). Force replaces
|
|
// additional addresses that are already stored; DryRun reports without
|
|
// writing. A frontend_url that models.AllowedFrontendURL refuses stops the
|
|
// import and nothing is written.
|
|
type WinterOptions struct {
|
|
DefinitionsFile string
|
|
SettingsFile string
|
|
Theme string
|
|
Maps map[string]string
|
|
PageURLs map[string]string
|
|
Force bool
|
|
DryRun bool
|
|
}
|
|
|
|
// WinterReport is what sitemap:import-winter did: the addresses kept, the
|
|
// report lines of every dropped or remapped item, and whether the settings
|
|
// were copied.
|
|
type WinterReport struct {
|
|
Kept int
|
|
Dropped int
|
|
Remapped int
|
|
Lines []string
|
|
Settings bool
|
|
}
|
|
|
|
// winterSettings are the WinterCMS settings the per-type sitemap reads.
|
|
type winterSettings struct {
|
|
SiteURL string
|
|
DefaultLocale string
|
|
}
|
|
|
|
// ImportWinter is sitemap:import-winter: it copies a WinterCMS site's
|
|
// sitemap definition into the settings' additional addresses and its
|
|
// frontend_url and frontend_default_locale into the site address and the
|
|
// default locale (D-32). url items are kept (static and CMS page items
|
|
// become addresses when --page-url reference=/path names their URL,
|
|
// --map old=new renames applied first); every other item is dropped and
|
|
// listed, because the content types list their records themselves. A url
|
|
// item whose URL is not a path or an http(s) URL (models.AllowedAddress)
|
|
// is dropped. Stored additional addresses are replaced only with --force.
|
|
// Settings keys of the PHP-parity format (priorities, tiers, the update
|
|
// manifest) are reported as ignored. A frontend_url that
|
|
// models.AllowedFrontendURL refuses stops the import and nothing is
|
|
// written.
|
|
func ImportWinter(app *backpack.App) bonfire.Command {
|
|
return bonfire.Command{
|
|
Name: "sitemap:import-winter",
|
|
Description: "Copy a WinterCMS sitemap definition and settings export into this application",
|
|
Long: `Copies the sitemap definition of a WinterCMS site (the JSON lines that
|
|
scripts/winter-export-definitions.sql prints) and its settings (the JSON
|
|
object from scripts/winter-export-settings.sql) in one transaction. The
|
|
definition's url items become the sitemap's additional addresses; --page-url
|
|
turns a static or CMS page item into an address with the given path, and
|
|
--map renames a type first (old=new). Every other item is dropped and listed:
|
|
the content types (posts, categories and so on) list their records
|
|
themselves. The settings' frontend_url becomes the site address and
|
|
frontend_default_locale the default locale; priorities, tiers and the update
|
|
manifest are no longer used. When the export holds several themes, --theme
|
|
picks the one to copy.
|
|
|
|
Run it once when an application moves from WinterCMS, --dry-run first:
|
|
--dry-run reports kept, remapped and dropped items and writes nothing. Stored
|
|
additional addresses are replaced only with --force. It ends with "Definition
|
|
copied: N items kept, N remapped, N dropped." and "Settings copied." for the
|
|
parts the export holds.`,
|
|
Flags: []bonfire.Flag{
|
|
{Name: "definitions", Description: "Definitions export (JSON lines from scripts/winter-export-definitions.sql)"},
|
|
{Name: "settings", Description: "Settings export (the JSON object from scripts/winter-export-settings.sql)"},
|
|
{Name: "theme", Description: "The PHP theme whose definition to copy when the export holds several"},
|
|
{Name: "map", Description: "Rename an item type, old=new (repeatable)", Repeatable: true},
|
|
{Name: "page-url", Description: "Copy a static or CMS page item as an address, reference=/path (repeatable)", Repeatable: true},
|
|
{Name: "force", Description: "Replace additional addresses that are already stored", Bare: true},
|
|
{Name: "dry-run", Description: "Report what would be copied without writing", Bare: true},
|
|
},
|
|
Examples: []bonfire.Example{
|
|
{Command: "sitemap:import-winter --definitions definitions.jsonl --settings settings.json --dry-run", Description: "Show what would be kept, remapped and dropped, without writing"},
|
|
{Command: "sitemap:import-winter --definitions definitions.jsonl --theme main", Description: "Copy the url items of the main theme's definition"},
|
|
{Command: "sitemap:import-winter --definitions definitions.jsonl --page-url about=/about --force", Description: "Replace the additional addresses, turning the about page into an address"},
|
|
},
|
|
Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
|
|
maps, err := parseMaps(in.Flags("map"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pageURLs, err := parsePageURLs(in.Flags("page-url"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
opts := WinterOptions{
|
|
DefinitionsFile: flagValue(in, "definitions"),
|
|
SettingsFile: flagValue(in, "settings"),
|
|
Theme: flagValue(in, "theme"),
|
|
Maps: maps,
|
|
PageURLs: pageURLs,
|
|
Force: flagOn(in, "force"),
|
|
DryRun: flagOn(in, "dry-run"),
|
|
}
|
|
db, done, err := lookupDB(ctx, app)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer done()
|
|
_, err = ImportWinterFiles(ctx, db, opts, out)
|
|
return err
|
|
},
|
|
}
|
|
}
|
|
|
|
// parseMaps reads --map old=new values.
|
|
func parseMaps(values []string) (map[string]string, error) {
|
|
maps := map[string]string{}
|
|
for _, v := range values {
|
|
from, to, ok := strings.Cut(v, "=")
|
|
from, to = strings.TrimSpace(from), strings.TrimSpace(to)
|
|
if !ok || from == "" {
|
|
return nil, fmt.Errorf("--map %q: use old=new", v)
|
|
}
|
|
maps[from] = to
|
|
}
|
|
return maps, nil
|
|
}
|
|
|
|
// parsePageURLs reads --page-url reference=/path values. The URL must be a
|
|
// root-relative path or an http(s) URL.
|
|
func parsePageURLs(values []string) (map[string]string, error) {
|
|
urls := map[string]string{}
|
|
for _, v := range values {
|
|
ref, url, ok := strings.Cut(v, "=")
|
|
ref, url = strings.TrimSpace(ref), strings.TrimSpace(url)
|
|
if !ok || ref == "" || url == "" {
|
|
return nil, fmt.Errorf("--page-url %q: use reference=/path", v)
|
|
}
|
|
lower := strings.ToLower(url)
|
|
if !strings.HasPrefix(url, "/") && !strings.HasPrefix(lower, "http://") && !strings.HasPrefix(lower, "https://") {
|
|
return nil, fmt.Errorf("--page-url %q: the URL must start with / or http(s)://", v)
|
|
}
|
|
urls[ref] = url
|
|
}
|
|
return urls, nil
|
|
}
|
|
|
|
// winterDefinition is one exported golem15_sitemap_definitions row.
|
|
type winterDefinition struct {
|
|
ID json.RawMessage `json:"id"`
|
|
Theme *string `json:"theme"`
|
|
Data json.RawMessage `json:"data"`
|
|
}
|
|
|
|
// ImportWinterFiles runs sitemap:import-winter against db.
|
|
func ImportWinterFiles(ctx context.Context, db *gorm.DB, opts WinterOptions, out bonfire.Output) (WinterReport, error) {
|
|
var report WinterReport
|
|
if opts.DefinitionsFile == "" && opts.SettingsFile == "" {
|
|
return report, errors.New("give --definitions, --settings or both")
|
|
}
|
|
var (
|
|
rows []models.ExtraURL
|
|
haveDef bool
|
|
)
|
|
if opts.DefinitionsFile != "" {
|
|
row, err := pickDefinition(opts.DefinitionsFile, opts.Theme)
|
|
if err != nil {
|
|
return report, err
|
|
}
|
|
out.Info(fmt.Sprintf("Copying the url items of definition %s (theme '%s') into the additional addresses.", strings.TrimSpace(string(row.ID)), models.Text(row.Theme)))
|
|
rows, err = mapItems(row.Data, opts.Maps, opts.PageURLs, &report)
|
|
if err != nil {
|
|
return report, err
|
|
}
|
|
haveDef = true
|
|
}
|
|
var settings winterSettings
|
|
if opts.SettingsFile != "" {
|
|
var err error
|
|
settings, err = readWinterSettings(opts.SettingsFile, &report)
|
|
if err != nil {
|
|
return report, err
|
|
}
|
|
report.Settings = true
|
|
}
|
|
for _, line := range report.Lines {
|
|
out.Warning(line)
|
|
}
|
|
if haveDef {
|
|
var existing models.Settings
|
|
if err := db.WithContext(ctx).Where("id = ?", 1).Limit(1).Find(&existing).Error; err != nil {
|
|
return report, err
|
|
}
|
|
if n := len(existing.Extra()); n > 0 && !opts.Force {
|
|
msg := fmt.Sprintf("The sitemap already has %d additional addresses; rerun with --force to replace them.", n)
|
|
if !opts.DryRun {
|
|
return report, errors.New(msg)
|
|
}
|
|
out.Warning(msg)
|
|
}
|
|
}
|
|
summary := fmt.Sprintf("%d items kept, %d remapped, %d dropped.", report.Kept, report.Remapped, report.Dropped)
|
|
if opts.DryRun {
|
|
if haveDef {
|
|
out.Info("Dry run: " + summary)
|
|
}
|
|
if report.Settings {
|
|
out.Info("Dry run: the settings would be copied.")
|
|
}
|
|
out.Info("Dry run complete (no changes made).")
|
|
return report, nil
|
|
}
|
|
err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
|
if _, err := loadSettingsForUpdate(tx); err != nil {
|
|
return err
|
|
}
|
|
if haveDef {
|
|
if err := saveExtra(tx, rows); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if report.Settings {
|
|
if err := tx.Exec(`UPDATE golem15_sitemap_settings SET site_url = ?, default_locale = ? WHERE id = 1`,
|
|
nullable(settings.SiteURL), nullable(settings.DefaultLocale)).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return report, err
|
|
}
|
|
if haveDef {
|
|
out.Success("Definition copied: " + summary)
|
|
}
|
|
if report.Settings {
|
|
out.Success("Settings copied.")
|
|
}
|
|
return report, nil
|
|
}
|
|
|
|
// nullable is nil for "", else s.
|
|
func nullable(s string) any {
|
|
if s == "" {
|
|
return nil
|
|
}
|
|
return s
|
|
}
|
|
|
|
// pickDefinition reads the definitions export and returns the row of theme,
|
|
// or the only row when theme is empty.
|
|
func pickDefinition(file, theme string) (winterDefinition, error) {
|
|
f, err := os.Open(file)
|
|
if err != nil {
|
|
return winterDefinition{}, fmt.Errorf("File not found: %s", file)
|
|
}
|
|
defer f.Close()
|
|
var rows []winterDefinition
|
|
scanner := bufio.NewScanner(f)
|
|
scanner.Buffer(make([]byte, 1<<20), 64<<20)
|
|
n := 0
|
|
for scanner.Scan() {
|
|
n++
|
|
line := bytes.TrimSpace(scanner.Bytes())
|
|
if len(line) == 0 {
|
|
continue
|
|
}
|
|
var row winterDefinition
|
|
if err := json.Unmarshal(line, &row); err != nil {
|
|
return winterDefinition{}, fmt.Errorf("%s line %d: %w", file, n, err)
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return winterDefinition{}, err
|
|
}
|
|
if len(rows) == 0 {
|
|
return winterDefinition{}, fmt.Errorf("%s holds no definition", file)
|
|
}
|
|
if theme != "" {
|
|
for _, row := range rows {
|
|
if models.Text(row.Theme) == theme {
|
|
return row, nil
|
|
}
|
|
}
|
|
return winterDefinition{}, fmt.Errorf("%s holds no definition of theme '%s'", file, theme)
|
|
}
|
|
if len(rows) > 1 {
|
|
themes := make([]string, 0, len(rows))
|
|
for _, row := range rows {
|
|
themes = append(themes, "'"+models.Text(row.Theme)+"'")
|
|
}
|
|
return winterDefinition{}, fmt.Errorf("%s holds %d definitions (themes %s); pick one with --theme", file, len(rows), strings.Join(themes, ", "))
|
|
}
|
|
return rows[0], nil
|
|
}
|
|
|
|
// mapItems decodes the exported data and keeps its url items as
|
|
// additional addresses.
|
|
func mapItems(data json.RawMessage, maps, pageURLs map[string]string, report *WinterReport) ([]models.ExtraURL, error) {
|
|
items, labels, err := models.ParseLegacyItems(data)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := []models.ExtraURL{}
|
|
seen := map[string]bool{}
|
|
keep := func(label, u string) {
|
|
if seen[u] {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (address '%s' is already listed)", label, u))
|
|
report.Dropped++
|
|
return
|
|
}
|
|
seen[u] = true
|
|
out = append(out, models.ExtraURL{URL: u})
|
|
report.Kept++
|
|
}
|
|
for i, item := range items {
|
|
label := describeItem(labels[i], item)
|
|
if item.CMSPage != "" {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: cmsPage '%s' dropped (CMS pages are not supported)", label, item.CMSPage))
|
|
}
|
|
if url, ok := pageURLs[item.Reference]; ok && droppedTypes[item.Type] {
|
|
if !models.AllowedAddress(url) {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (--page-url '%s' is not a path or an http(s) URL)", label, url))
|
|
report.Dropped++
|
|
continue
|
|
}
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: copied as an additional address '%s'", label, url))
|
|
report.Remapped++
|
|
keep(label, url)
|
|
continue
|
|
}
|
|
if to, ok := maps[item.Type]; ok && to != item.Type {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: type '%s' mapped to '%s'", label, item.Type, to))
|
|
report.Remapped++
|
|
item.Type = to
|
|
}
|
|
switch {
|
|
case item.Type == models.LegacyItemURL:
|
|
if !models.AllowedAddress(item.URL) {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (the url is not a path or an http(s) URL)", label))
|
|
report.Dropped++
|
|
continue
|
|
}
|
|
keep(label, item.URL)
|
|
case item.Type == "":
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (mapped to nothing)", label))
|
|
report.Dropped++
|
|
case droppedTypes[item.Type]:
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (static and CMS page types are not supported)", label))
|
|
report.Dropped++
|
|
default:
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (the content types list their records themselves)", label))
|
|
report.Dropped++
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func describeItem(label string, item models.LegacyItem) string {
|
|
s := fmt.Sprintf("item %s (type '%s'", label, item.Type)
|
|
if item.URL != "" {
|
|
s += fmt.Sprintf(", url '%s'", item.URL)
|
|
}
|
|
if item.Reference != "" {
|
|
s += fmt.Sprintf(", reference '%s'", item.Reference)
|
|
}
|
|
return s + ")"
|
|
}
|
|
|
|
// ignoredWinterSettings are the PHP settings of the PHP-parity format the
|
|
// per-type sitemap no longer reads.
|
|
var ignoredWinterSettings = map[string]bool{
|
|
"updates_manifest_path": true, "enable_priority_tiers": true,
|
|
"priority_homepage": true, "priority_services": true, "priority_blog": true,
|
|
"priority_categories": true, "priority_legal": true,
|
|
}
|
|
|
|
// readWinterSettings reads the exported settings object. A frontend_url
|
|
// that models.AllowedFrontendURL refuses is an error; the value is quoted so
|
|
// its control characters are not printed raw.
|
|
func readWinterSettings(file string, report *WinterReport) (winterSettings, error) {
|
|
raw, err := os.ReadFile(file)
|
|
if err != nil {
|
|
return winterSettings{}, fmt.Errorf("File not found: %s", file)
|
|
}
|
|
var fields map[string]json.RawMessage
|
|
if err := json.Unmarshal(models.LegacyJSON(raw), &fields); err != nil {
|
|
return winterSettings{}, fmt.Errorf("%s: the settings must be a JSON object: %w", file, err)
|
|
}
|
|
var s winterSettings
|
|
keys := make([]string, 0, len(fields))
|
|
for key := range fields {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, key := range keys {
|
|
value := fields[key]
|
|
switch {
|
|
case key == "frontend_url" || key == "frontend_default_locale":
|
|
text, err := models.DecodeText(value)
|
|
if err != nil {
|
|
return winterSettings{}, fmt.Errorf("%s: setting %s: %w", file, key, err)
|
|
}
|
|
if key == "frontend_url" {
|
|
s.SiteURL = models.Text(text)
|
|
} else {
|
|
s.DefaultLocale = strings.TrimSpace(models.Text(text))
|
|
}
|
|
case ignoredWinterSettings[key]:
|
|
report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (the per-type sitemaps do not use it)", key))
|
|
default:
|
|
report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (not a sitemap setting of this port)", key))
|
|
}
|
|
}
|
|
if !models.AllowedFrontendURL(s.SiteURL) {
|
|
return winterSettings{}, fmt.Errorf("%s: setting frontend_url %q must be empty or an absolute http:// or https:// URL without whitespace, control characters, quotes or the characters <, > and &", file, s.SiteURL)
|
|
}
|
|
return s, nil
|
|
}
|