package console import ( "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "os" "sort" "strings" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/bonfire" "gorm.io/gorm" ) // droppedTypes are the WinterCMS item types the port does not support: // Winter.Pages static pages and CMS pages. var droppedTypes = map[string]bool{ "static-page": true, "all-static-pages": true, "cms-page": true, } // WinterOptions are the inputs of sitemap:import-winter. DefinitionsFile is // the output of scripts/winter-export-definitions.sql (one JSON object per // line: id, theme, data, created_at, updated_at); SettingsFile the output // of scripts/winter-export-settings.sql (the JSON object of the // golem15_sitemap_settings system setting). Theme picks a definition when // the export holds several. Maps rename item types (old to new) before they // are checked. PageURLs turns a static or CMS page item into an additional // address: its key is the item's reference (the PHP page file name, for // example blog), its value the page's URL (/blog). Force replaces // additional addresses that are already stored; DryRun reports without // writing. A frontend_url that models.AllowedFrontendURL refuses stops the // import and nothing is written. type WinterOptions struct { DefinitionsFile string SettingsFile string Theme string Maps map[string]string PageURLs map[string]string Force bool DryRun bool } // WinterReport is what sitemap:import-winter did: the addresses kept, the // report lines of every dropped or remapped item, and whether the settings // were copied. type WinterReport struct { Kept int Dropped int Remapped int Lines []string Settings bool } // winterSettings are the WinterCMS settings the per-type sitemap reads. type winterSettings struct { SiteURL string DefaultLocale string } // ImportWinter is sitemap:import-winter: it copies a WinterCMS site's // sitemap definition into the settings' additional addresses and its // frontend_url and frontend_default_locale into the site address and the // default locale (D-32). url items are kept (static and CMS page items // become addresses when --page-url reference=/path names their URL, // --map old=new renames applied first); every other item is dropped and // listed, because the content types list their records themselves. A url // item whose URL is not a path or an http(s) URL (models.AllowedAddress) // is dropped. Stored additional addresses are replaced only with --force. // Settings keys of the PHP-parity format (priorities, tiers, the update // manifest) are reported as ignored. A frontend_url that // models.AllowedFrontendURL refuses stops the import and nothing is // written. func ImportWinter(app *backpack.App) bonfire.Command { return bonfire.Command{ Name: "sitemap:import-winter", Description: "Copy a WinterCMS sitemap definition and settings export into this application", Long: `Copies the sitemap definition of a WinterCMS site (the JSON lines that scripts/winter-export-definitions.sql prints) and its settings (the JSON object from scripts/winter-export-settings.sql) in one transaction. The definition's url items become the sitemap's additional addresses; --page-url turns a static or CMS page item into an address with the given path, and --map renames a type first (old=new). Every other item is dropped and listed: the content types (posts, categories and so on) list their records themselves. The settings' frontend_url becomes the site address and frontend_default_locale the default locale; priorities, tiers and the update manifest are no longer used. When the export holds several themes, --theme picks the one to copy. Run it once when an application moves from WinterCMS, --dry-run first: --dry-run reports kept, remapped and dropped items and writes nothing. Stored additional addresses are replaced only with --force. It ends with "Definition copied: N items kept, N remapped, N dropped." and "Settings copied." for the parts the export holds.`, Flags: []bonfire.Flag{ {Name: "definitions", Description: "Definitions export (JSON lines from scripts/winter-export-definitions.sql)"}, {Name: "settings", Description: "Settings export (the JSON object from scripts/winter-export-settings.sql)"}, {Name: "theme", Description: "The PHP theme whose definition to copy when the export holds several"}, {Name: "map", Description: "Rename an item type, old=new (repeatable)", Repeatable: true}, {Name: "page-url", Description: "Copy a static or CMS page item as an address, reference=/path (repeatable)", Repeatable: true}, {Name: "force", Description: "Replace additional addresses that are already stored", Bare: true}, {Name: "dry-run", Description: "Report what would be copied without writing", Bare: true}, }, Examples: []bonfire.Example{ {Command: "sitemap:import-winter --definitions definitions.jsonl --settings settings.json --dry-run", Description: "Show what would be kept, remapped and dropped, without writing"}, {Command: "sitemap:import-winter --definitions definitions.jsonl --theme main", Description: "Copy the url items of the main theme's definition"}, {Command: "sitemap:import-winter --definitions definitions.jsonl --page-url about=/about --force", Description: "Replace the additional addresses, turning the about page into an address"}, }, Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { maps, err := parseMaps(in.Flags("map")) if err != nil { return err } pageURLs, err := parsePageURLs(in.Flags("page-url")) if err != nil { return err } opts := WinterOptions{ DefinitionsFile: flagValue(in, "definitions"), SettingsFile: flagValue(in, "settings"), Theme: flagValue(in, "theme"), Maps: maps, PageURLs: pageURLs, Force: flagOn(in, "force"), DryRun: flagOn(in, "dry-run"), } db, done, err := lookupDB(ctx, app) if err != nil { return err } defer done() _, err = ImportWinterFiles(ctx, db, opts, out) return err }, } } // parseMaps reads --map old=new values. func parseMaps(values []string) (map[string]string, error) { maps := map[string]string{} for _, v := range values { from, to, ok := strings.Cut(v, "=") from, to = strings.TrimSpace(from), strings.TrimSpace(to) if !ok || from == "" { return nil, fmt.Errorf("--map %q: use old=new", v) } maps[from] = to } return maps, nil } // parsePageURLs reads --page-url reference=/path values. The URL must be a // root-relative path or an http(s) URL. func parsePageURLs(values []string) (map[string]string, error) { urls := map[string]string{} for _, v := range values { ref, url, ok := strings.Cut(v, "=") ref, url = strings.TrimSpace(ref), strings.TrimSpace(url) if !ok || ref == "" || url == "" { return nil, fmt.Errorf("--page-url %q: use reference=/path", v) } lower := strings.ToLower(url) if !strings.HasPrefix(url, "/") && !strings.HasPrefix(lower, "http://") && !strings.HasPrefix(lower, "https://") { return nil, fmt.Errorf("--page-url %q: the URL must start with / or http(s)://", v) } urls[ref] = url } return urls, nil } // winterDefinition is one exported golem15_sitemap_definitions row. type winterDefinition struct { ID json.RawMessage `json:"id"` Theme *string `json:"theme"` Data json.RawMessage `json:"data"` } // ImportWinterFiles runs sitemap:import-winter against db. func ImportWinterFiles(ctx context.Context, db *gorm.DB, opts WinterOptions, out bonfire.Output) (WinterReport, error) { var report WinterReport if opts.DefinitionsFile == "" && opts.SettingsFile == "" { return report, errors.New("give --definitions, --settings or both") } var ( rows []models.ExtraURL haveDef bool ) if opts.DefinitionsFile != "" { row, err := pickDefinition(opts.DefinitionsFile, opts.Theme) if err != nil { return report, err } out.Info(fmt.Sprintf("Copying the url items of definition %s (theme '%s') into the additional addresses.", strings.TrimSpace(string(row.ID)), models.Text(row.Theme))) rows, err = mapItems(row.Data, opts.Maps, opts.PageURLs, &report) if err != nil { return report, err } haveDef = true } var settings winterSettings if opts.SettingsFile != "" { var err error settings, err = readWinterSettings(opts.SettingsFile, &report) if err != nil { return report, err } report.Settings = true } for _, line := range report.Lines { out.Warning(line) } if haveDef { var existing models.Settings if err := db.WithContext(ctx).Where("id = ?", 1).Limit(1).Find(&existing).Error; err != nil { return report, err } if n := len(existing.Extra()); n > 0 && !opts.Force { msg := fmt.Sprintf("The sitemap already has %d additional addresses; rerun with --force to replace them.", n) if !opts.DryRun { return report, errors.New(msg) } out.Warning(msg) } } summary := fmt.Sprintf("%d items kept, %d remapped, %d dropped.", report.Kept, report.Remapped, report.Dropped) if opts.DryRun { if haveDef { out.Info("Dry run: " + summary) } if report.Settings { out.Info("Dry run: the settings would be copied.") } out.Info("Dry run complete (no changes made).") return report, nil } err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { if _, err := loadSettingsForUpdate(tx); err != nil { return err } if haveDef { if err := saveExtra(tx, rows); err != nil { return err } } if report.Settings { if err := tx.Exec(`UPDATE golem15_sitemap_settings SET site_url = ?, default_locale = ? WHERE id = 1`, nullable(settings.SiteURL), nullable(settings.DefaultLocale)).Error; err != nil { return err } } return nil }) if err != nil { return report, err } if haveDef { out.Success("Definition copied: " + summary) } if report.Settings { out.Success("Settings copied.") } return report, nil } // nullable is nil for "", else s. func nullable(s string) any { if s == "" { return nil } return s } // pickDefinition reads the definitions export and returns the row of theme, // or the only row when theme is empty. func pickDefinition(file, theme string) (winterDefinition, error) { f, err := os.Open(file) if err != nil { return winterDefinition{}, fmt.Errorf("File not found: %s", file) } defer f.Close() var rows []winterDefinition scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 1<<20), 64<<20) n := 0 for scanner.Scan() { n++ line := bytes.TrimSpace(scanner.Bytes()) if len(line) == 0 { continue } var row winterDefinition if err := json.Unmarshal(line, &row); err != nil { return winterDefinition{}, fmt.Errorf("%s line %d: %w", file, n, err) } rows = append(rows, row) } if err := scanner.Err(); err != nil { return winterDefinition{}, err } if len(rows) == 0 { return winterDefinition{}, fmt.Errorf("%s holds no definition", file) } if theme != "" { for _, row := range rows { if models.Text(row.Theme) == theme { return row, nil } } return winterDefinition{}, fmt.Errorf("%s holds no definition of theme '%s'", file, theme) } if len(rows) > 1 { themes := make([]string, 0, len(rows)) for _, row := range rows { themes = append(themes, "'"+models.Text(row.Theme)+"'") } return winterDefinition{}, fmt.Errorf("%s holds %d definitions (themes %s); pick one with --theme", file, len(rows), strings.Join(themes, ", ")) } return rows[0], nil } // mapItems decodes the exported data and keeps its url items as // additional addresses. func mapItems(data json.RawMessage, maps, pageURLs map[string]string, report *WinterReport) ([]models.ExtraURL, error) { items, labels, err := models.ParseLegacyItems(data) if err != nil { return nil, err } out := []models.ExtraURL{} seen := map[string]bool{} keep := func(label, u string) { if seen[u] { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (address '%s' is already listed)", label, u)) report.Dropped++ return } seen[u] = true out = append(out, models.ExtraURL{URL: u}) report.Kept++ } for i, item := range items { label := describeItem(labels[i], item) if item.CMSPage != "" { report.Lines = append(report.Lines, fmt.Sprintf("%s: cmsPage '%s' dropped (CMS pages are not supported)", label, item.CMSPage)) } if url, ok := pageURLs[item.Reference]; ok && droppedTypes[item.Type] { if !models.AllowedAddress(url) { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (--page-url '%s' is not a path or an http(s) URL)", label, url)) report.Dropped++ continue } report.Lines = append(report.Lines, fmt.Sprintf("%s: copied as an additional address '%s'", label, url)) report.Remapped++ keep(label, url) continue } if to, ok := maps[item.Type]; ok && to != item.Type { report.Lines = append(report.Lines, fmt.Sprintf("%s: type '%s' mapped to '%s'", label, item.Type, to)) report.Remapped++ item.Type = to } switch { case item.Type == models.LegacyItemURL: if !models.AllowedAddress(item.URL) { report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (the url is not a path or an http(s) URL)", label)) report.Dropped++ continue } keep(label, item.URL) case item.Type == "": report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (mapped to nothing)", label)) report.Dropped++ case droppedTypes[item.Type]: report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (static and CMS page types are not supported)", label)) report.Dropped++ default: report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (the content types list their records themselves)", label)) report.Dropped++ } } return out, nil } func describeItem(label string, item models.LegacyItem) string { s := fmt.Sprintf("item %s (type '%s'", label, item.Type) if item.URL != "" { s += fmt.Sprintf(", url '%s'", item.URL) } if item.Reference != "" { s += fmt.Sprintf(", reference '%s'", item.Reference) } return s + ")" } // ignoredWinterSettings are the PHP settings of the PHP-parity format the // per-type sitemap no longer reads. var ignoredWinterSettings = map[string]bool{ "updates_manifest_path": true, "enable_priority_tiers": true, "priority_homepage": true, "priority_services": true, "priority_blog": true, "priority_categories": true, "priority_legal": true, } // readWinterSettings reads the exported settings object. A frontend_url // that models.AllowedFrontendURL refuses is an error; the value is quoted so // its control characters are not printed raw. func readWinterSettings(file string, report *WinterReport) (winterSettings, error) { raw, err := os.ReadFile(file) if err != nil { return winterSettings{}, fmt.Errorf("File not found: %s", file) } var fields map[string]json.RawMessage if err := json.Unmarshal(models.LegacyJSON(raw), &fields); err != nil { return winterSettings{}, fmt.Errorf("%s: the settings must be a JSON object: %w", file, err) } var s winterSettings keys := make([]string, 0, len(fields)) for key := range fields { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { value := fields[key] switch { case key == "frontend_url" || key == "frontend_default_locale": text, err := models.DecodeText(value) if err != nil { return winterSettings{}, fmt.Errorf("%s: setting %s: %w", file, key, err) } if key == "frontend_url" { s.SiteURL = models.Text(text) } else { s.DefaultLocale = strings.TrimSpace(models.Text(text)) } case ignoredWinterSettings[key]: report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (the per-type sitemaps do not use it)", key)) default: report.Lines = append(report.Lines, fmt.Sprintf("setting '%s' ignored (not a sitemap setting of this port)", key)) } } if !models.AllowedFrontendURL(s.SiteURL) { return winterSettings{}, fmt.Errorf("%s: setting frontend_url %q must be empty or an absolute http:// or https:// URL without whitespace, control characters, quotes or the characters <, > and &", file, s.SiteURL) } return s, nil }