sm-menu-plugin/console/import_winter.go

package console

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"slices"
	"strings"
	"time"

	"git.golem15.com/golem15/sm-menu-plugin/classes"
	"git.golem15.com/golem15/sm-menu-plugin/models"
	"git.golem15.com/golem15/summercms/modules/backpack"
	"git.golem15.com/golem15/summercms/modules/bonfire"
	"git.golem15.com/golem15/summercms/modules/signpost"
	"gorm.io/gorm"
)

// DefaultTypeMap maps the WinterCMS entry types to this plugin's (D-05): the
// PHP link is the built-in url, and the PHP Active Blog Categories list is
// the Journal's all-journal-categories registry type. --map pairs extend
// and override it. PHP page entries (Winter.Pages) are dropped.
var DefaultTypeMap = map[string]string{
	"link":            models.TypeURL,
	"blog_categories": "all-journal-categories",
}

// droppedPHPType is the Winter.Pages entry type the port does not support.
const droppedPHPType = "page"

// WinterOptions are the inputs of menu:import-winter. MenusFile and
// EntriesFile are the outputs of scripts/winter-export-menus.sql and
// scripts/winter-export-entries.sql (one JSON object per line). Maps rename
// entry types (old to new) over DefaultTypeMap; a type mapped to "" is
// dropped. Force replaces the menus and entries the tables already hold;
// DryRun reports without writing.
type WinterOptions struct {
	MenusFile   string
	EntriesFile string
	Maps        map[string]string
	Force       bool
	DryRun      bool
}

// WinterReport is what menu:import-winter copied (or would copy on a dry
// run): menus and entries kept, entries remapped and dropped, and one report
// line per remapped or dropped entry.
type WinterReport struct {
	Menus    int
	Entries  int
	Remapped int
	Dropped  int
	Lines    []string
}

// ImportWinter is menu:import-winter: it copies a WinterCMS site's menus and
// menu entries, keeping their ids, menu_id and parent_id, the nest columns
// and sort_order. Entry types are mapped (link to url, blog_categories to
// all-journal-categories, --map old=new on top); a page entry, an entry of a
// type that is neither built-in nor registered, and a registry entry whose
// reference is not among the type's current references are dropped with
// their subtree, and every dropped or remapped entry is reported. The rules
// an admin save enforces are applied to the copies with a report line each:
// a URL that classes.AllowedURL refuses is trimmed when only surrounding
// whitespace is wrong and cleared otherwise, a target other than _self or
// _blank is cleared, and an entry whose parent is in another menu is copied
// as a top-level entry of its own menu. Tables
// that already hold rows are refused without --force, which replaces them in
// the same transaction. Both id sequences are moved past the copied ids.
func ImportWinter(app *backpack.App) bonfire.Command {
	return bonfire.Command{
		Name:        "menu:import-winter",
		Description: "Copy a WinterCMS menu and menu entry export into this application",
		Flags: []bonfire.Flag{
			{Name: "menus", Description: "Menus export (JSON lines from scripts/winter-export-menus.sql)"},
			{Name: "entries", Description: "Menu entries export (JSON lines from scripts/winter-export-entries.sql)"},
			{Name: "map", Description: "Rename an entry type, old=new (repeatable; new may be empty to drop the type)", Repeatable: true},
			{Name: "force", Description: "Replace the menus and entries the tables already hold", Bare: true},
			{Name: "dry-run", Description: "Report what would be copied without writing", Bare: true},
		},
		Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
			maps, err := parseMaps(in.Flags("map"))
			if err != nil {
				return err
			}
			opts := WinterOptions{
				MenusFile:   flagValue(in, "menus"),
				EntriesFile: flagValue(in, "entries"),
				Maps:        maps,
				Force:       flagOn(in, "force"),
				DryRun:      flagOn(in, "dry-run"),
			}
			db, done, err := lookupDB(ctx, app)
			if err != nil {
				return err
			}
			defer done()
			var reg *signpost.Registry
			if app != nil {
				reg, _ = app.Lookup[*signpost.Registry]()
			}
			_, err = ImportWinterFiles(ctx, db, reg, opts, out)
			return err
		},
	}
}

// parseMaps reads --map old=new values.
func parseMaps(values []string) (map[string]string, error) {
	maps := map[string]string{}
	for _, v := range values {
		from, to, ok := strings.Cut(v, "=")
		from, to = strings.TrimSpace(from), strings.TrimSpace(to)
		if !ok || from == "" {
			return nil, fmt.Errorf("--map %q: use old=new", v)
		}
		maps[from] = to
	}
	return maps, nil
}

// ImportWinterFiles runs menu:import-winter against db; reg is the menu item
// type registry the entry types are checked against (nil knows none).
func ImportWinterFiles(ctx context.Context, db *gorm.DB, reg *signpost.Registry, opts WinterOptions, out bonfire.Output) (WinterReport, error) {
	var report WinterReport
	if opts.MenusFile == "" || opts.EntriesFile == "" {
		return report, errors.New("give --menus and --entries")
	}
	menuRows, err := readJSONLines(opts.MenusFile)
	if err != nil {
		return report, err
	}
	entryRows, err := readJSONLines(opts.EntriesFile)
	if err != nil {
		return report, err
	}
	if len(menuRows) == 0 && len(entryRows) == 0 {
		out.Info("The export holds no menus and no entries; nothing to copy (0 menus, 0 entries).")
		return report, nil
	}
	menus := make([]models.Menu, 0, len(menuRows))
	menuIDs := map[uint]bool{}
	for i, row := range menuRows {
		m, err := decodeMenu(row)
		if err != nil {
			return report, fmt.Errorf("%s row %d: %w", opts.MenusFile, i+1, err)
		}
		menus = append(menus, m)
		menuIDs[m.ID] = true
	}
	entries := make([]models.MenuEntry, 0, len(entryRows))
	for i, row := range entryRows {
		e, err := decodeEntry(row)
		if err != nil {
			return report, fmt.Errorf("%s row %d: %w", opts.EntriesFile, i+1, err)
		}
		entries = append(entries, e)
	}
	kept, err := mapEntries(ctx, reg, entries, menuIDs, mergedMaps(opts.Maps), &report)
	if err != nil {
		return report, err
	}
	report.Menus, report.Entries = len(menus), len(kept)
	for _, line := range report.Lines {
		out.Warning(line)
	}

	db = db.WithContext(ctx)
	var haveMenus, haveEntries int64
	if err := db.Model(&models.Menu{}).Count(&haveMenus).Error; err != nil {
		return report, err
	}
	if err := db.Model(&models.MenuEntry{}).Count(&haveEntries).Error; err != nil {
		return report, err
	}
	if (haveMenus > 0 || haveEntries > 0) && !opts.Force {
		msg := fmt.Sprintf("The menu tables already hold %d menus and %d entries; rerun with --force to replace them.", haveMenus, haveEntries)
		if !opts.DryRun {
			return report, errors.New(msg)
		}
		out.Warning(msg)
	}
	summary := fmt.Sprintf("%d menus, %d entries kept, %d remapped, %d dropped.", report.Menus, report.Entries, report.Remapped, report.Dropped)
	if opts.DryRun {
		out.Info("Dry run: " + summary)
		out.Info("Dry run complete (no changes made).")
		return report, nil
	}
	err = db.Transaction(func(tx *gorm.DB) error {
		if haveMenus > 0 || haveEntries > 0 {
			for _, stmt := range []string{`DELETE FROM golem15_menu_menu_entries`, `DELETE FROM golem15_menu_menus`} {
				if err := tx.Exec(stmt).Error; err != nil {
					return err
				}
			}
		}
		// The rows are copied as they are: the entry save hook (sort_order,
		// menu from the parent) is for admin saves.
		raw := tx.Session(&gorm.Session{SkipHooks: true})
		for i := range menus {
			if err := raw.Create(&menus[i]).Error; err != nil {
				return fmt.Errorf("menu %d: %w", menus[i].ID, err)
			}
		}
		for i := range kept {
			if err := raw.Create(&kept[i]).Error; err != nil {
				return fmt.Errorf("entry %d: %w", kept[i].ID, err)
			}
		}
		for _, table := range []string{"golem15_menu_menus", "golem15_menu_menu_entries"} {
			if err := tx.Exec(`SELECT setval(pg_get_serial_sequence('` + table + `', 'id'), COALESCE(MAX(id), 1), MAX(id) IS NOT NULL) FROM ` + table).Error; err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		return report, err
	}
	out.Success("Copied " + summary)
	return report, nil
}

// mergedMaps is DefaultTypeMap with the --map pairs on top.
func mergedMaps(maps map[string]string) map[string]string {
	out := make(map[string]string, len(DefaultTypeMap)+len(maps))
	for k, v := range DefaultTypeMap {
		out[k] = v
	}
	for k, v := range maps {
		out[k] = v
	}
	return out
}

// mapEntries maps every entry's type and returns the entries to copy, in
// export order. An entry is dropped when its menu is not in the menus
// export, when its parent is missing or dropped (with the whole subtree),
// for a page type, for a type mapped to nothing, for a type that is neither
// built-in nor registered, and for a registry type that takes references
// when its reference is not among them. A kept entry's URL, target and
// parent are made to pass the admin save rules (see ImportWinter).
func mapEntries(ctx context.Context, reg *signpost.Registry, entries []models.MenuEntry, menuIDs map[uint]bool, maps map[string]string, report *WinterReport) ([]models.MenuEntry, error) {
	byID := make(map[uint]*models.MenuEntry, len(entries))
	for i := range entries {
		byID[entries[i].ID] = &entries[i]
	}
	refs := map[string][]string{}
	// reason is why an entry is dropped on its own, "" when it is kept;
	// oldType is the exported type the report names.
	reason := map[uint]string{}
	oldType := make(map[uint]string, len(entries))
	for i := range entries {
		e := &entries[i]
		old := e.Type
		oldType[e.ID] = old
		label := describe(e, old)
		if to, ok := maps[old]; ok && to != old {
			if to == "" {
				reason[e.ID] = fmt.Sprintf("type '%s' is mapped to nothing", old)
				continue
			}
			e.Type = to
			report.Lines = append(report.Lines, fmt.Sprintf("%s: type '%s' mapped to '%s'", label, old, to))
			report.Remapped++
		}
		switch {
		case !menuIDs[e.MenuID]:
			reason[e.ID] = fmt.Sprintf("menu %d is not in the menus export", e.MenuID)
		case e.Type == droppedPHPType:
			reason[e.ID] = "page entries (Winter.Pages) are not supported"
		case models.Builtin(e.Type):
		default:
			if _, known := reg.Type(e.Type); !known {
				reason[e.ID] = fmt.Sprintf("type '%s' is neither built-in nor registered; map it with --map %s=<type>", e.Type, old)
				continue
			}
			values, cached := refs[e.Type]
			if !cached {
				set, err := signpost.ReferenceOptions(ctx, reg, e.Type)
				if err != nil {
					return nil, err
				}
				if set.Applies {
					values = []string{}
					for _, o := range set.Options {
						values = append(values, o.Value)
					}
				}
				refs[e.Type] = values
			}
			if values != nil && !slices.Contains(values, models.Text(e.Reference)) {
				reason[e.ID] = fmt.Sprintf("reference '%s' is not among the current references of '%s'", models.Text(e.Reference), e.Type)
			}
		}
	}
	// dropped walks up the parents: an entry goes when it or an ancestor
	// goes, or when an ancestor is missing from the export.
	var dropped func(e *models.MenuEntry, seen map[uint]bool) (bool, string)
	dropped = func(e *models.MenuEntry, seen map[uint]bool) (bool, string) {
		if r := reason[e.ID]; r != "" {
			return true, r
		}
		if e.ParentID == nil {
			return false, ""
		}
		if seen[e.ID] {
			return true, "its parent chain loops"
		}
		seen[e.ID] = true
		parent, ok := byID[*e.ParentID]
		if !ok {
			return true, fmt.Sprintf("its parent %d is not in the entries export", *e.ParentID)
		}
		if gone, _ := dropped(parent, seen); gone {
			return true, fmt.Sprintf("dropped with its parent %d", parent.ID)
		}
		return false, ""
	}
	kept := make([]models.MenuEntry, 0, len(entries))
	for i := range entries {
		e := &entries[i]
		if gone, why := dropped(e, map[uint]bool{}); gone {
			report.Lines = append(report.Lines, fmt.Sprintf("%s: dropped (%s)", describe(e, oldType[e.ID]), why))
			report.Dropped++
			continue
		}
		report.Lines = append(report.Lines, adminRules(e, byID, oldType[e.ID])...)
		kept = append(kept, *e)
	}
	return kept, nil
}

// adminRules makes a kept entry pass the rules an admin save enforces and
// returns a report line per change: a URL classes.AllowedURL refuses is
// trimmed when that is enough and cleared otherwise, a target
// classes.AllowedTarget refuses is cleared, and a parent in another menu is
// dropped so the entry is a top-level entry of its own menu (the public tree
// would leave it out, the reorder page would show it at the top).
func adminRules(e *models.MenuEntry, byID map[uint]*models.MenuEntry, typ string) []string {
	var lines []string
	label := describe(e, typ)
	if url := models.Text(e.URL); !classes.AllowedURL(url) {
		if trimmed := strings.TrimSpace(url); trimmed != url && classes.AllowedURL(trimmed) {
			e.URL = &trimmed
			lines = append(lines, fmt.Sprintf("%s: url '%s' trimmed to '%s'", label, url, trimmed))
		} else {
			e.URL = nil
			lines = append(lines, fmt.Sprintf("%s: url %q cleared (only http, https, mailto, tel, / and # links are served)", label, url))
		}
	}
	if target := models.Text(e.Target); !classes.AllowedTarget(target) {
		e.Target = nil
		lines = append(lines, fmt.Sprintf("%s: target '%s' cleared (only _self and _blank are allowed)", label, target))
	}
	if e.ParentID != nil {
		if parent, ok := byID[*e.ParentID]; ok && parent.MenuID != e.MenuID {
			lines = append(lines, fmt.Sprintf("%s: copied as a top-level entry (its parent %d is in menu %d)", label, parent.ID, parent.MenuID))
			e.ParentID = nil
		}
	}
	return lines
}

// describe names an entry in a report line by id, title, menu, exported
// type and reference.
func describe(e *models.MenuEntry, typ string) string {
	s := fmt.Sprintf("entry %d '%s' (menu %d, type '%s'", e.ID, models.Text(e.Title), e.MenuID, typ)
	if e.Reference != nil && *e.Reference != "" {
		s += fmt.Sprintf(", reference '%s'", *e.Reference)
	}
	return s + ")"
}

// decodeMenu reads one exported golem15_menu_menus row.
func decodeMenu(row map[string]json.RawMessage) (models.Menu, error) {
	var m models.Menu
	id, err := jsonUint(row["id"])
	if err != nil || id == 0 {
		return m, fmt.Errorf("id: want a positive id, got %s", row["id"])
	}
	m.ID = id
	name, err := jsonText(row["name"])
	if err != nil || name == nil || *name == "" {
		return m, fmt.Errorf("name is missing")
	}
	code, err := jsonText(row["code"])
	if err != nil || code == nil || *code == "" {
		return m, fmt.Errorf("code is missing")
	}
	m.Name, m.Code = *name, *code
	if m.IsActive, err = jsonBool(row["is_active"], true); err != nil {
		return m, fmt.Errorf("is_active: %w", err)
	}
	if m.CSSClass, err = jsonText(row["css_class"]); err != nil {
		return m, fmt.Errorf("css_class: %w", err)
	}
	if err := decodeTimes(row, &m.CreatedAt, &m.UpdatedAt); err != nil {
		return m, err
	}
	return m, nil
}

// decodeEntry reads one exported golem15_menu_menu_entries row with every
// PHP column; NOT NULL columns missing from the export take the PHP
// defaults (nest columns 0, an empty title, published).
func decodeEntry(row map[string]json.RawMessage) (models.MenuEntry, error) {
	var e models.MenuEntry
	id, err := jsonUint(row["id"])
	if err != nil || id == 0 {
		return e, fmt.Errorf("id: want a positive id, got %s", row["id"])
	}
	e.ID = id
	if e.MenuID, err = jsonUint(row["menu_id"]); err != nil {
		return e, fmt.Errorf("menu_id: %w", err)
	}
	parent, err := jsonUint(row["parent_id"])
	if err != nil {
		return e, fmt.Errorf("parent_id: %w", err)
	}
	if parent != 0 {
		e.ParentID = &parent
	}
	for key, dest := range map[string]**int{"nest_left": &e.NestLeft, "nest_right": &e.NestRight, "nest_depth": &e.NestDepth, "sort_order": &e.SortOrder} {
		n, err := jsonInt(row[key])
		if err != nil {
			return e, fmt.Errorf("%s: %w", key, err)
		}
		*dest = n
	}
	for _, p := range []**int{&e.NestLeft, &e.NestRight, &e.NestDepth} {
		if *p == nil {
			zero := 0
			*p = &zero
		}
	}
	typ, err := jsonText(row["type"])
	if err != nil || typ == nil || *typ == "" {
		return e, fmt.Errorf("type is missing")
	}
	e.Type = *typ
	for key, dest := range map[string]**string{"title": &e.Title, "url": &e.URL, "reference": &e.Reference, "code": &e.Code, "icon": &e.Icon, "css_class": &e.CSSClass, "target": &e.Target} {
		if *dest, err = jsonText(row[key]); err != nil {
			return e, fmt.Errorf("%s: %w", key, err)
		}
	}
	if e.Title == nil {
		empty := ""
		e.Title = &empty
	}
	published, err := jsonBool(row["is_published"], true)
	if err != nil {
		return e, fmt.Errorf("is_published: %w", err)
	}
	e.IsPublished = &published
	if err := decodeTimes(row, &e.CreatedAt, &e.UpdatedAt); err != nil {
		return e, err
	}
	return e, nil
}

func decodeTimes(row map[string]json.RawMessage, created, updated **time.Time) error {
	var err error
	if *created, err = jsonTime(row["created_at"]); err != nil {
		return fmt.Errorf("created_at: %w", err)
	}
	if *updated, err = jsonTime(row["updated_at"]); err != nil {
		return fmt.Errorf("updated_at: %w", err)
	}
	return nil
}