1bb89ce006
- Settings.BeforeValidate answers a cabana validation error on frontend_url, so the settings save is a 422 and the row is unchanged - sitemap:import-winter stops on a frontend_url that models.AllowedFrontendURL refuses, before the dry-run branch and before any database use; the value is printed quoted - errors.frontend_url phrase in en and pl - TestSettingsBeforeValidate, TestWinterSettingsFrontendURL and the save and import refusals in the Postgres smokes
471 lines
16 KiB
Go
471 lines
16 KiB
Go
package console
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
|
|
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
|
|
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
|
"git.golem15.com/golem15/summercms/modules/backpack"
|
|
"git.golem15.com/golem15/summercms/modules/bonfire"
|
|
"git.golem15.com/golem15/summercms/modules/signpost"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// droppedTypes are the WinterCMS item types the port does not support:
|
|
// Winter.Pages static pages and CMS pages.
|
|
var droppedTypes = map[string]bool{
|
|
"static-page": true,
|
|
"all-static-pages": true,
|
|
"cms-page": true,
|
|
}
|
|
|
|
// WinterOptions are the inputs of sitemap:import-winter. DefinitionsFile is
|
|
// the output of scripts/winter-export-definitions.sql (one JSON object per
|
|
// line: id, theme, data, created_at, updated_at); SettingsFile the output
|
|
// of scripts/winter-export-settings.sql (the JSON object of the
|
|
// golem15_sitemap_settings system setting). Theme picks a definition when
|
|
// the export holds several. Maps rename item types (old to new) before they
|
|
// are checked. PageURLs turns a static or CMS page item into a url item:
|
|
// its key is the item's reference (the PHP page file name, for example
|
|
// blog), its value the page's URL (/blog). Force replaces a definition that
|
|
// has items; DryRun reports without writing. A frontend_url that
|
|
// models.AllowedFrontendURL refuses stops the import and nothing is written.
|
|
type WinterOptions struct {
|
|
DefinitionsFile string
|
|
SettingsFile string
|
|
Theme string
|
|
Maps map[string]string
|
|
PageURLs map[string]string
|
|
Force bool
|
|
DryRun bool
|
|
}
|
|
|
|
// WinterReport is what sitemap:import-winter did: the items kept, the
|
|
// report lines of every dropped or remapped item, and whether the settings
|
|
// were copied.
|
|
type WinterReport struct {
|
|
Kept int
|
|
Dropped int
|
|
Remapped int
|
|
Lines []string
|
|
Settings bool
|
|
}
|
|
|
|
// ImportWinter is sitemap:import-winter: it copies a WinterCMS site's
|
|
// sitemap definition and settings into the definition /sitemap.xml serves
|
|
// (theme default) and the settings row. Item types are mapped (url kept,
|
|
// static and CMS page types dropped unless --page-url reference=/path names
|
|
// their URL, --map old=new renames applied first),
|
|
// an item of a registry type that takes references is dropped when its
|
|
// reference is not among the type's current references, an item of an
|
|
// unknown type is dropped, a url item whose URL is not a path or an http(s)
|
|
// URL (classes.AllowedItemURL) is dropped, a URL pattern that is not a path
|
|
// (signpost.ValidItemPattern) is cleared, and every dropped, remapped or
|
|
// cleared item is printed. A definition that already has items is refused
|
|
// without --force. Settings keys missing from the export take the PHP
|
|
// defaults. A frontend_url that models.AllowedFrontendURL refuses stops the
|
|
// import and nothing is written.
|
|
func ImportWinter(app *backpack.App) bonfire.Command {
|
|
return bonfire.Command{
|
|
Name: "sitemap:import-winter",
|
|
Description: "Copy a WinterCMS sitemap definition and settings export into this application",
|
|
Flags: []bonfire.Flag{
|
|
{Name: "definitions", Description: "Definitions export (JSON lines from scripts/winter-export-definitions.sql)"},
|
|
{Name: "settings", Description: "Settings export (the JSON object from scripts/winter-export-settings.sql)"},
|
|
{Name: "theme", Description: "The PHP theme whose definition to copy when the export holds several"},
|
|
{Name: "map", Description: "Rename an item type, old=new (repeatable)", Repeatable: true},
|
|
{Name: "page-url", Description: "Copy a static or CMS page item as a url item, reference=/path (repeatable)", Repeatable: true},
|
|
{Name: "force", Description: "Replace a definition that already has items", Bare: true},
|
|
{Name: "dry-run", Description: "Report what would be copied without writing", Bare: true},
|
|
},
|
|
Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
|
|
maps, err := parseMaps(in.Flags("map"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pageURLs, err := parsePageURLs(in.Flags("page-url"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
opts := WinterOptions{
|
|
DefinitionsFile: flagValue(in, "definitions"),
|
|
SettingsFile: flagValue(in, "settings"),
|
|
Theme: flagValue(in, "theme"),
|
|
Maps: maps,
|
|
PageURLs: pageURLs,
|
|
Force: flagOn(in, "force"),
|
|
DryRun: flagOn(in, "dry-run"),
|
|
}
|
|
db, done, err := lookupDB(ctx, app)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer done()
|
|
var reg *signpost.Registry
|
|
if app != nil {
|
|
reg, _ = app.Lookup[*signpost.Registry]()
|
|
}
|
|
_, err = ImportWinterFiles(ctx, db, reg, opts, out)
|
|
return err
|
|
},
|
|
}
|
|
}
|
|
|
|
// parseMaps reads --map old=new values.
|
|
func parseMaps(values []string) (map[string]string, error) {
|
|
maps := map[string]string{}
|
|
for _, v := range values {
|
|
from, to, ok := strings.Cut(v, "=")
|
|
from, to = strings.TrimSpace(from), strings.TrimSpace(to)
|
|
if !ok || from == "" {
|
|
return nil, fmt.Errorf("--map %q: use old=new", v)
|
|
}
|
|
maps[from] = to
|
|
}
|
|
return maps, nil
|
|
}
|
|
|
|
// parsePageURLs reads --page-url reference=/path values. The URL must be a
|
|
// root-relative path or an http(s) URL.
|
|
func parsePageURLs(values []string) (map[string]string, error) {
|
|
urls := map[string]string{}
|
|
for _, v := range values {
|
|
ref, url, ok := strings.Cut(v, "=")
|
|
ref, url = strings.TrimSpace(ref), strings.TrimSpace(url)
|
|
if !ok || ref == "" || url == "" {
|
|
return nil, fmt.Errorf("--page-url %q: use reference=/path", v)
|
|
}
|
|
lower := strings.ToLower(url)
|
|
if !strings.HasPrefix(url, "/") && !strings.HasPrefix(lower, "http://") && !strings.HasPrefix(lower, "https://") {
|
|
return nil, fmt.Errorf("--page-url %q: the URL must start with / or http(s)://", v)
|
|
}
|
|
urls[ref] = url
|
|
}
|
|
return urls, nil
|
|
}
|
|
|
|
// winterDefinition is one exported golem15_sitemap_definitions row.
|
|
type winterDefinition struct {
|
|
ID json.RawMessage `json:"id"`
|
|
Theme *string `json:"theme"`
|
|
Data json.RawMessage `json:"data"`
|
|
}
|
|
|
|
// ImportWinterFiles runs sitemap:import-winter against db; reg is the menu
|
|
// item type registry the item types are checked against (nil knows none).
|
|
func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, opts WinterOptions, out bonfire.Output) (WinterReport, error) {
|
|
var report WinterReport
|
|
if opts.DefinitionsFile == "" && opts.SettingsFile == "" {
|
|
return report, errors.New("give --definitions, --settings or both")
|
|
}
|
|
var (
|
|
items []models.DefinitionItem
|
|
haveDef bool
|
|
)
|
|
if opts.DefinitionsFile != "" {
|
|
row, err := pickDefinition(opts.DefinitionsFile, opts.Theme)
|
|
if err != nil {
|
|
return report, err
|
|
}
|
|
out.Info(fmt.Sprintf("Copying definition %s (theme '%s') into the '%s' definition.", strings.TrimSpace(string(row.ID)), models.Text(row.Theme), models.DefinitionTheme))
|
|
items, err = mapItems(ctx, reg, row.Data, opts.Maps, opts.PageURLs, &report)
|
|
if err != nil {
|
|
return report, err
|
|
}
|
|
haveDef = true
|
|
}
|
|
var settings models.Settings
|
|
if opts.SettingsFile != "" {
|
|
var err error
|
|
settings, err = readWinterSettings(opts.SettingsFile, &report)
|
|
if err != nil {
|
|
return report, err
|
|
}
|
|
report.Settings = true
|
|
}
|
|
for _, line := range report.Lines {
|
|
out.Warning(line)
|
|
}
|
|
if haveDef {
|
|
var existing []models.Definition
|
|
if err := db.WithContext(ctx).Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&existing).Error; err != nil {
|
|
return report, err
|
|
}
|
|
if len(existing) > 0 && len(existing[0].Items()) > 0 && !opts.Force {
|
|
msg := fmt.Sprintf("The sitemap definition already has %d items; rerun with --force to replace them.", len(existing[0].Items()))
|
|
if !opts.DryRun {
|
|
return report, errors.New(msg)
|
|
}
|
|
out.Warning(msg)
|
|
}
|
|
}
|
|
summary := fmt.Sprintf("%d items kept, %d remapped, %d dropped.", report.Kept, report.Remapped, report.Dropped)
|
|
if opts.DryRun {
|
|
if haveDef {
|
|
out.Info("Dry run: " + summary)
|
|
}
|
|
if report.Settings {
|
|
out.Info("Dry run: the settings would be copied.")
|
|
}
|
|
out.Info("Dry run complete (no changes made).")
|
|
return report, nil
|
|
}
|
|
err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
|
if haveDef {
|
|
def, err := singletonDefinition(tx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
def.SetItems(items)
|
|
if err := tx.Save(&def).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if report.Settings {
|
|
settings.ID = 1
|
|
if err := tx.Save(&settings).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return report, err
|
|
}
|
|
if haveDef {
|
|
out.Info("Definition copied: " + summary)
|
|
}
|
|
if report.Settings {
|
|
out.Info("Settings copied.")
|
|
}
|
|
return report, nil
|
|
}
|
|
|
|
// pickDefinition reads the definitions export and returns the row of theme,
|
|
// or the only row when theme is empty.
|
|
func pickDefinition(file, theme string) (winterDefinition, error) {
|
|
f, err := os.Open(file)
|
|
if err != nil {
|
|
return winterDefinition{}, fmt.Errorf("File not found: %s", file)
|
|
}
|
|
defer f.Close()
|
|
var rows []winterDefinition
|
|
scanner := bufio.NewScanner(f)
|
|
scanner.Buffer(make([]byte, 1<<20), 64<<20)
|
|
n := 0
|
|
for scanner.Scan() {
|
|
n++
|
|
line := bytes.TrimSpace(scanner.Bytes())
|
|
if len(line) == 0 {
|
|
continue
|
|
}
|
|
var row winterDefinition
|
|
if err := json.Unmarshal(line, &row); err != nil {
|
|
return winterDefinition{}, fmt.Errorf("%s line %d: %w", file, n, err)
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return winterDefinition{}, err
|
|
}
|
|
if len(rows) == 0 {
|
|
return winterDefinition{}, fmt.Errorf("%s holds no definition", file)
|
|
}
|
|
if theme != "" {
|
|
for _, row := range rows {
|
|
if models.Text(row.Theme) == theme {
|
|
return row, nil
|
|
}
|
|
}
|
|
return winterDefinition{}, fmt.Errorf("%s holds no definition of theme '%s'", file, theme)
|
|
}
|
|
if len(rows) > 1 {
|
|
themes := make([]string, 0, len(rows))
|
|
for _, row := range rows {
|
|
themes = append(themes, "'"+models.Text(row.Theme)+"'")
|
|
}
|
|
return winterDefinition{}, fmt.Errorf("%s holds %d definitions (themes %s); pick one with --theme", file, len(rows), strings.Join(themes, ", "))
|
|
}
|
|
return rows[0], nil
|
|
}
|
|
|
|
// exportedJSON is a JSON value that the export may hold as text (a TEXT
|
|
// column selected as it is) or as JSON: a string is decoded once more.
|
|
func exportedJSON(raw json.RawMessage) json.RawMessage {
|
|
raw = bytes.TrimSpace(raw)
|
|
if len(raw) > 0 && raw[0] == '"' {
|
|
var text string
|
|
if json.Unmarshal(raw, &text) == nil {
|
|
return json.RawMessage(strings.TrimSpace(text))
|
|
}
|
|
}
|
|
return raw
|
|
}
|
|
|
|
// mapItems decodes the exported data and maps its items.
|
|
func mapItems(ctx context.Context, reg *signpost.Registry, data json.RawMessage, maps, pageURLs map[string]string, report *WinterReport) ([]models.DefinitionItem, error) {
|
|
data = exportedJSON(data)
|
|
if len(data) == 0 || bytes.Equal(data, []byte("null")) {
|
|
return []models.DefinitionItem{}, nil
|
|
}
|
|
rawItems, labels, ok := orderedEntries(data)
|
|
if !ok {
|
|
return nil, errors.New("the definition data is not a list of items")
|
|
}
|
|
out := []models.DefinitionItem{}
|
|
refs := map[string][]string{}
|
|
for i, raw := range rawItems {
|
|
var item models.DefinitionItem
|
|
if err := json.Unmarshal(raw, &item); err != nil {
|
|
return nil, fmt.Errorf("definition item %s: %w", labels[i], err)
|
|
}
|
|
var fields map[string]any
|
|
_ = json.Unmarshal(raw, &fields)
|
|
label := describeItem(labels[i], item)
|
|
if page, ok := fields["cmsPage"].(string); ok && page != "" {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: cmsPage '%s' dropped (CMS pages are not supported)", label, page))
|
|
}
|
|
if p := models.Text(item.URLPattern); p != "" && !signpost.ValidItemPattern(p) {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: urlPattern '%s' cleared (not a path starting with a single /)", label, p))
|
|
item.URLPattern = nil
|
|
}
|
|
if url, ok := pageURLs[models.Text(item.Reference)]; ok && droppedTypes[item.Type] {
|
|
if !classes.AllowedItemURL(url) {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (--page-url '%s' is not a path or an http(s) URL)", label, url))
|
|
report.Dropped++
|
|
continue
|
|
}
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: copied as type 'url' with url '%s'", label, url))
|
|
report.Remapped++
|
|
item.Type = models.ItemTypeURL
|
|
item.URL = &url
|
|
item.Reference = nil
|
|
out = append(out, item)
|
|
report.Kept++
|
|
continue
|
|
}
|
|
if to, ok := maps[item.Type]; ok && to != item.Type {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: type '%s' mapped to '%s'", label, item.Type, to))
|
|
report.Remapped++
|
|
item.Type = to
|
|
if to == "" {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (mapped to nothing)", label))
|
|
report.Dropped++
|
|
continue
|
|
}
|
|
}
|
|
if item.Type == models.ItemTypeURL {
|
|
if !classes.AllowedItemURL(models.Text(item.URL)) {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (the url is not a path or an http(s) URL)", label))
|
|
report.Dropped++
|
|
continue
|
|
}
|
|
out = append(out, item)
|
|
report.Kept++
|
|
continue
|
|
}
|
|
if droppedTypes[item.Type] {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (static and CMS page types are not supported)", label))
|
|
report.Dropped++
|
|
continue
|
|
}
|
|
if _, known := reg.Type(item.Type); !known {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (type '%s' is not registered; map it with --map %s=<type>)", label, item.Type, item.Type))
|
|
report.Dropped++
|
|
continue
|
|
}
|
|
values, cached := refs[item.Type]
|
|
if !cached {
|
|
set, err := signpost.ReferenceOptions(ctx, reg, item.Type)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if set.Applies {
|
|
values = []string{}
|
|
for _, o := range set.Options {
|
|
values = append(values, o.Value)
|
|
}
|
|
}
|
|
refs[item.Type] = values
|
|
}
|
|
if values != nil && !slices.Contains(values, models.Text(item.Reference)) {
|
|
report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (reference '%s' is not among the current references of '%s')", label, models.Text(item.Reference), item.Type))
|
|
report.Dropped++
|
|
continue
|
|
}
|
|
out = append(out, item)
|
|
report.Kept++
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func describeItem(label string, item models.DefinitionItem) string {
|
|
s := fmt.Sprintf("item %s (type '%s'", label, item.Type)
|
|
if item.URL != nil && *item.URL != "" {
|
|
s += fmt.Sprintf(", url '%s'", *item.URL)
|
|
}
|
|
if item.Reference != nil && *item.Reference != "" {
|
|
s += fmt.Sprintf(", reference '%s'", *item.Reference)
|
|
}
|
|
return s + ")"
|
|
}
|
|
|
|
// readWinterSettings reads the exported settings object onto the PHP
|
|
// defaults. A frontend_url that models.AllowedFrontendURL refuses is an
|
|
// error; the value is quoted so its control characters are not printed raw.
|
|
func readWinterSettings(file string, report *WinterReport) (models.Settings, error) {
|
|
raw, err := os.ReadFile(file)
|
|
if err != nil {
|
|
return models.Settings{}, fmt.Errorf("File not found: %s", file)
|
|
}
|
|
var fields map[string]json.RawMessage
|
|
if err := json.Unmarshal(exportedJSON(raw), &fields); err != nil {
|
|
return models.Settings{}, fmt.Errorf("%s: the settings must be a JSON object: %w", file, err)
|
|
}
|
|
s := models.DefaultSettings()
|
|
texts := map[string]**string{
|
|
"frontend_url": &s.FrontendURL,
|
|
"frontend_default_locale": &s.FrontendDefaultLocale,
|
|
"updates_manifest_path": &s.UpdatesManifestPath,
|
|
models.PriorityHomepage: &s.PriorityHomepage,
|
|
models.PriorityServices: &s.PriorityServices,
|
|
models.PriorityBlog: &s.PriorityBlog,
|
|
models.PriorityCategories: &s.PriorityCategories,
|
|
models.PriorityLegal: &s.PriorityLegal,
|
|
}
|
|
keys := make([]string, 0, len(fields))
|
|
for key := range fields {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, key := range keys {
|
|
value := fields[key]
|
|
if dest, ok := texts[key]; ok {
|
|
text, err := models.DecodeText(value)
|
|
if err != nil {
|
|
return models.Settings{}, fmt.Errorf("%s: setting %s: %w", file, key, err)
|
|
}
|
|
*dest = text
|
|
continue
|
|
}
|
|
if key == "enable_priority_tiers" {
|
|
s.EnablePriorityTiers = !phpEmpty(value)
|
|
continue
|
|
}
|
|
report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (not a sitemap setting of this port)", key))
|
|
}
|
|
if url := models.Text(s.FrontendURL); !models.AllowedFrontendURL(url) {
|
|
return models.Settings{}, fmt.Errorf("%s: setting frontend_url %q must be empty or an absolute http:// or https:// URL without whitespace, control characters, quotes or the characters <, > and &", file, url)
|
|
}
|
|
return s, nil
|
|
}
|