package console import ( "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "os" "slices" "sort" "strings" "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/bonfire" "git.golem15.com/golem15/summercms/modules/signpost" "gorm.io/gorm" ) // droppedTypes are the WinterCMS item types the port does not support: // Winter.Pages static pages and CMS pages. var droppedTypes = map[string]bool{ "static-page": true, "all-static-pages": true, "cms-page": true, } // WinterOptions are the inputs of sitemap:import-winter. DefinitionsFile is // the output of scripts/winter-export-definitions.sql (one JSON object per // line: id, theme, data, created_at, updated_at); SettingsFile the output // of scripts/winter-export-settings.sql (the JSON object of the // golem15_sitemap_settings system setting). Theme picks a definition when // the export holds several. Maps rename item types (old to new) before they // are checked. PageURLs turns a static or CMS page item into a url item: // its key is the item's reference (the PHP page file name, for example // blog), its value the page's URL (/blog). Force replaces a definition that // has items; DryRun reports without writing. A frontend_url that // models.AllowedFrontendURL refuses stops the import and nothing is written. type WinterOptions struct { DefinitionsFile string SettingsFile string Theme string Maps map[string]string PageURLs map[string]string Force bool DryRun bool } // WinterReport is what sitemap:import-winter did: the items kept, the // report lines of every dropped or remapped item, and whether the settings // were copied. type WinterReport struct { Kept int Dropped int Remapped int Lines []string Settings bool } // ImportWinter is sitemap:import-winter: it copies a WinterCMS site's // sitemap definition and settings into the definition /sitemap.xml serves // (theme default) and the settings row. Item types are mapped (url kept, // static and CMS page types dropped unless --page-url reference=/path names // their URL, --map old=new renames applied first), // an item of a registry type that takes references is dropped when its // reference is not among the type's current references, an item of an // unknown type is dropped, a url item whose URL is not a path or an http(s) // URL (classes.AllowedItemURL) is dropped, a URL pattern that is not a path // (signpost.ValidItemPattern) is cleared, and every dropped, remapped or // cleared item is printed. A definition that already has items is refused // without --force. Settings keys missing from the export take the PHP // defaults. A frontend_url that models.AllowedFrontendURL refuses stops the // import and nothing is written. func ImportWinter(app *backpack.App) bonfire.Command { return bonfire.Command{ Name: "sitemap:import-winter", Description: "Copy a WinterCMS sitemap definition and settings export into this application", Long: `Copies the sitemap definition of a WinterCMS site (the JSON lines that scripts/winter-export-definitions.sql prints) and its settings (the JSON object from scripts/winter-export-settings.sql) in one transaction. Item types this application knows are kept; --map renames a type (old=new) and --page-url turns a static or CMS page item into a url item with the given path. Items that still have no known type are dropped and listed. When the export holds several themes, --theme picks the one to copy. Run it once when an application moves from WinterCMS, --dry-run first: --dry-run reports kept, remapped and dropped items and writes nothing. A definition that already has items is replaced only with --force. It ends with "Definition copied: N items kept, N remapped, N dropped." and "Settings copied." for the parts the export holds.`, Flags: []bonfire.Flag{ {Name: "definitions", Description: "Definitions export (JSON lines from scripts/winter-export-definitions.sql)"}, {Name: "settings", Description: "Settings export (the JSON object from scripts/winter-export-settings.sql)"}, {Name: "theme", Description: "The PHP theme whose definition to copy when the export holds several"}, {Name: "map", Description: "Rename an item type, old=new (repeatable)", Repeatable: true}, {Name: "page-url", Description: "Copy a static or CMS page item as a url item, reference=/path (repeatable)", Repeatable: true}, {Name: "force", Description: "Replace a definition that already has items", Bare: true}, {Name: "dry-run", Description: "Report what would be copied without writing", Bare: true}, }, Examples: []bonfire.Example{ {Command: "sitemap:import-winter --definitions definitions.jsonl --settings settings.json --dry-run", Description: "Show what would be kept, remapped and dropped, without writing"}, {Command: "sitemap:import-winter --definitions definitions.jsonl --theme main --map blog-post=journal-post", Description: "Copy the main theme's definition, renaming an item type"}, {Command: "sitemap:import-winter --definitions definitions.jsonl --page-url about=/about --force", Description: "Replace the definition, turning the about page into a url item"}, }, Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { maps, err := parseMaps(in.Flags("map")) if err != nil { return err } pageURLs, err := parsePageURLs(in.Flags("page-url")) if err != nil { return err } opts := WinterOptions{ DefinitionsFile: flagValue(in, "definitions"), SettingsFile: flagValue(in, "settings"), Theme: flagValue(in, "theme"), Maps: maps, PageURLs: pageURLs, Force: flagOn(in, "force"), DryRun: flagOn(in, "dry-run"), } db, done, err := lookupDB(ctx, app) if err != nil { return err } defer done() var reg *signpost.Registry if app != nil { reg, _ = app.Lookup[*signpost.Registry]() } _, err = ImportWinterFiles(ctx, db, reg, opts, out) return err }, } } // parseMaps reads --map old=new values. func parseMaps(values []string) (map[string]string, error) { maps := map[string]string{} for _, v := range values { from, to, ok := strings.Cut(v, "=") from, to = strings.TrimSpace(from), strings.TrimSpace(to) if !ok || from == "" { return nil, fmt.Errorf("--map %q: use old=new", v) } maps[from] = to } return maps, nil } // parsePageURLs reads --page-url reference=/path values. The URL must be a // root-relative path or an http(s) URL. func parsePageURLs(values []string) (map[string]string, error) { urls := map[string]string{} for _, v := range values { ref, url, ok := strings.Cut(v, "=") ref, url = strings.TrimSpace(ref), strings.TrimSpace(url) if !ok || ref == "" || url == "" { return nil, fmt.Errorf("--page-url %q: use reference=/path", v) } lower := strings.ToLower(url) if !strings.HasPrefix(url, "/") && !strings.HasPrefix(lower, "http://") && !strings.HasPrefix(lower, "https://") { return nil, fmt.Errorf("--page-url %q: the URL must start with / or http(s)://", v) } urls[ref] = url } return urls, nil } // winterDefinition is one exported golem15_sitemap_definitions row. type winterDefinition struct { ID json.RawMessage `json:"id"` Theme *string `json:"theme"` Data json.RawMessage `json:"data"` } // ImportWinterFiles runs sitemap:import-winter against db; reg is the menu // item type registry the item types are checked against (nil knows none). func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, opts WinterOptions, out bonfire.Output) (WinterReport, error) { var report WinterReport if opts.DefinitionsFile == "" && opts.SettingsFile == "" { return report, errors.New("give --definitions, --settings or both") } var ( items []models.DefinitionItem haveDef bool ) if opts.DefinitionsFile != "" { row, err := pickDefinition(opts.DefinitionsFile, opts.Theme) if err != nil { return report, err } out.Info(fmt.Sprintf("Copying definition %s (theme '%s') into the '%s' definition.", strings.TrimSpace(string(row.ID)), models.Text(row.Theme), models.DefinitionTheme)) items, err = mapItems(ctx, reg, row.Data, opts.Maps, opts.PageURLs, &report) if err != nil { return report, err } haveDef = true } var settings models.Settings if opts.SettingsFile != "" { var err error settings, err = readWinterSettings(opts.SettingsFile, &report) if err != nil { return report, err } report.Settings = true } for _, line := range report.Lines { out.Warning(line) } if haveDef { var existing []models.Definition if err := db.WithContext(ctx).Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&existing).Error; err != nil { return report, err } if len(existing) > 0 && len(existing[0].Items()) > 0 && !opts.Force { msg := fmt.Sprintf("The sitemap definition already has %d items; rerun with --force to replace them.", len(existing[0].Items())) if !opts.DryRun { return report, errors.New(msg) } out.Warning(msg) } } summary := fmt.Sprintf("%d items kept, %d remapped, %d dropped.", report.Kept, report.Remapped, report.Dropped) if opts.DryRun { if haveDef { out.Info("Dry run: " + summary) } if report.Settings { out.Info("Dry run: the settings would be copied.") } out.Info("Dry run complete (no changes made).") return report, nil } err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { if haveDef { def, err := singletonDefinition(tx) if err != nil { return err } def.SetItems(items) if err := tx.Save(&def).Error; err != nil { return err } } if report.Settings { settings.ID = 1 if err := tx.Save(&settings).Error; err != nil { return err } } return nil }) if err != nil { return report, err } if haveDef { out.Success("Definition copied: " + summary) } if report.Settings { out.Success("Settings copied.") } return report, nil } // pickDefinition reads the definitions export and returns the row of theme, // or the only row when theme is empty. func pickDefinition(file, theme string) (winterDefinition, error) { f, err := os.Open(file) if err != nil { return winterDefinition{}, fmt.Errorf("File not found: %s", file) } defer f.Close() var rows []winterDefinition scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 1<<20), 64<<20) n := 0 for scanner.Scan() { n++ line := bytes.TrimSpace(scanner.Bytes()) if len(line) == 0 { continue } var row winterDefinition if err := json.Unmarshal(line, &row); err != nil { return winterDefinition{}, fmt.Errorf("%s line %d: %w", file, n, err) } rows = append(rows, row) } if err := scanner.Err(); err != nil { return winterDefinition{}, err } if len(rows) == 0 { return winterDefinition{}, fmt.Errorf("%s holds no definition", file) } if theme != "" { for _, row := range rows { if models.Text(row.Theme) == theme { return row, nil } } return winterDefinition{}, fmt.Errorf("%s holds no definition of theme '%s'", file, theme) } if len(rows) > 1 { themes := make([]string, 0, len(rows)) for _, row := range rows { themes = append(themes, "'"+models.Text(row.Theme)+"'") } return winterDefinition{}, fmt.Errorf("%s holds %d definitions (themes %s); pick one with --theme", file, len(rows), strings.Join(themes, ", ")) } return rows[0], nil } // exportedJSON is a JSON value that the export may hold as text (a TEXT // column selected as it is) or as JSON: a string is decoded once more. func exportedJSON(raw json.RawMessage) json.RawMessage { raw = bytes.TrimSpace(raw) if len(raw) > 0 && raw[0] == '"' { var text string if json.Unmarshal(raw, &text) == nil { return json.RawMessage(strings.TrimSpace(text)) } } return raw } // mapItems decodes the exported data and maps its items. func mapItems(ctx context.Context, reg *signpost.Registry, data json.RawMessage, maps, pageURLs map[string]string, report *WinterReport) ([]models.DefinitionItem, error) { data = exportedJSON(data) if len(data) == 0 || bytes.Equal(data, []byte("null")) { return []models.DefinitionItem{}, nil } rawItems, labels, ok := orderedEntries(data) if !ok { return nil, errors.New("the definition data is not a list of items") } out := []models.DefinitionItem{} refs := map[string][]string{} for i, raw := range rawItems { var item models.DefinitionItem if err := json.Unmarshal(raw, &item); err != nil { return nil, fmt.Errorf("definition item %s: %w", labels[i], err) } var fields map[string]any _ = json.Unmarshal(raw, &fields) label := describeItem(labels[i], item) if page, ok := fields["cmsPage"].(string); ok && page != "" { report.Lines = append(report.Lines, fmt.Sprintf("%s: cmsPage '%s' dropped (CMS pages are not supported)", label, page)) } if p := models.Text(item.URLPattern); p != "" && !signpost.ValidItemPattern(p) { report.Lines = append(report.Lines, fmt.Sprintf("%s: urlPattern '%s' cleared (not a path starting with a single /)", label, p)) item.URLPattern = nil } if url, ok := pageURLs[models.Text(item.Reference)]; ok && droppedTypes[item.Type] { if !classes.AllowedItemURL(url) { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (--page-url '%s' is not a path or an http(s) URL)", label, url)) report.Dropped++ continue } report.Lines = append(report.Lines, fmt.Sprintf("%s: copied as type 'url' with url '%s'", label, url)) report.Remapped++ item.Type = models.ItemTypeURL item.URL = &url item.Reference = nil out = append(out, item) report.Kept++ continue } if to, ok := maps[item.Type]; ok && to != item.Type { report.Lines = append(report.Lines, fmt.Sprintf("%s: type '%s' mapped to '%s'", label, item.Type, to)) report.Remapped++ item.Type = to if to == "" { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (mapped to nothing)", label)) report.Dropped++ continue } } if item.Type == models.ItemTypeURL { if !classes.AllowedItemURL(models.Text(item.URL)) { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (the url is not a path or an http(s) URL)", label)) report.Dropped++ continue } out = append(out, item) report.Kept++ continue } if droppedTypes[item.Type] { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (static and CMS page types are not supported)", label)) report.Dropped++ continue } if _, known := reg.Type(item.Type); !known { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (type '%s' is not registered; map it with --map %s=)", label, item.Type, item.Type)) report.Dropped++ continue } values, cached := refs[item.Type] if !cached { set, err := signpost.ReferenceOptions(ctx, reg, item.Type) if err != nil { return nil, err } if set.Applies { values = []string{} for _, o := range set.Options { values = append(values, o.Value) } } refs[item.Type] = values } if values != nil && !slices.Contains(values, models.Text(item.Reference)) { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (reference '%s' is not among the current references of '%s')", label, models.Text(item.Reference), item.Type)) report.Dropped++ continue } out = append(out, item) report.Kept++ } return out, nil } func describeItem(label string, item models.DefinitionItem) string { s := fmt.Sprintf("item %s (type '%s'", label, item.Type) if item.URL != nil && *item.URL != "" { s += fmt.Sprintf(", url '%s'", *item.URL) } if item.Reference != nil && *item.Reference != "" { s += fmt.Sprintf(", reference '%s'", *item.Reference) } return s + ")" } // readWinterSettings reads the exported settings object onto the PHP // defaults. A frontend_url that models.AllowedFrontendURL refuses is an // error; the value is quoted so its control characters are not printed raw. func readWinterSettings(file string, report *WinterReport) (models.Settings, error) { raw, err := os.ReadFile(file) if err != nil { return models.Settings{}, fmt.Errorf("File not found: %s", file) } var fields map[string]json.RawMessage if err := json.Unmarshal(exportedJSON(raw), &fields); err != nil { return models.Settings{}, fmt.Errorf("%s: the settings must be a JSON object: %w", file, err) } s := models.DefaultSettings() texts := map[string]**string{ "frontend_url": &s.FrontendURL, "frontend_default_locale": &s.FrontendDefaultLocale, "updates_manifest_path": &s.UpdatesManifestPath, models.PriorityHomepage: &s.PriorityHomepage, models.PriorityServices: &s.PriorityServices, models.PriorityBlog: &s.PriorityBlog, models.PriorityCategories: &s.PriorityCategories, models.PriorityLegal: &s.PriorityLegal, } keys := make([]string, 0, len(fields)) for key := range fields { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { value := fields[key] if dest, ok := texts[key]; ok { text, err := models.DecodeText(value) if err != nil { return models.Settings{}, fmt.Errorf("%s: setting %s: %w", file, key, err) } *dest = text continue } if key == "enable_priority_tiers" { s.EnablePriorityTiers = !phpEmpty(value) continue } report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (not a sitemap setting of this port)", key)) } if url := models.Text(s.FrontendURL); !models.AllowedFrontendURL(url) { return models.Settings{}, fmt.Errorf("%s: setting frontend_url %q must be empty or an absolute http:// or https:// URL without whitespace, control characters, quotes or the characters <, > and &", file, url) } return s, nil }