feat(17-04): faq:import-winter and the read-only Winter export SQL
- JSON lines import keeping ids, locale_id looked up by locale code (an unknown code aborts before any write) - one transaction, setval past the copied ids, --force replaces, --dry-run reports, an empty file copies nothing - scripts/winter-export.sql: SELECT JSON_OBJECT with the translate locale code - TestFaqImportSmoke
This commit is contained in:
248
console/import_winter.go
Normal file
248
console/import_winter.go
Normal file
@@ -0,0 +1,248 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-faq-plugin/models"
|
||||
translate "git.golem15.com/golem15/sm-translate-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/bonfire"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// WinterOptions are the inputs of faq:import-winter. File is the output of
|
||||
// scripts/winter-export.sql: one JSON object per line with id, name, slug,
|
||||
// locale_code, is_active, description, content (the PHP JSON text or a
|
||||
// list), created_at and updated_at. Force replaces the FAQs a non-empty table
|
||||
// holds; DryRun reports without writing.
|
||||
type WinterOptions struct {
|
||||
File string
|
||||
Force bool
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// WinterReport is what faq:import-winter copied (or would copy on a dry
|
||||
// run): the number of FAQs and one line per FAQ.
|
||||
type WinterReport struct {
|
||||
FAQs int
|
||||
Lines []string
|
||||
}
|
||||
|
||||
// ImportWinter is faq:import-winter: it copies a WinterCMS site's FAQs into
|
||||
// golem15_faq_faqs, keeping their ids. Each row's locale is looked up by its
|
||||
// code among the translate locales; an unknown code aborts the import before
|
||||
// anything is written. A table that already holds FAQs is refused without
|
||||
// --force, which replaces them in the same transaction. The id sequence is
|
||||
// moved past the copied ids.
|
||||
func ImportWinter(app *backpack.App) bonfire.Command {
|
||||
return bonfire.Command{
|
||||
Name: "faq:import-winter",
|
||||
Description: "Copy a WinterCMS FAQ export into this application",
|
||||
Flags: []bonfire.Flag{
|
||||
{Name: "file", Description: "FAQ export (JSON lines from scripts/winter-export.sql)"},
|
||||
{Name: "force", Description: "Replace the FAQs the table already holds", Bare: true},
|
||||
{Name: "dry-run", Description: "Report what would be copied without writing", Bare: true},
|
||||
},
|
||||
Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
|
||||
opts := WinterOptions{
|
||||
File: flagValue(in, "file"),
|
||||
Force: flagOn(in, "force"),
|
||||
DryRun: flagOn(in, "dry-run"),
|
||||
}
|
||||
db, done, err := lookupDB(ctx, app)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done()
|
||||
_, err = ImportWinterFile(ctx, db, opts, out)
|
||||
return err
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ImportWinterFile runs faq:import-winter against db.
|
||||
func ImportWinterFile(ctx context.Context, db *gorm.DB, opts WinterOptions, out bonfire.Output) (WinterReport, error) {
|
||||
var report WinterReport
|
||||
if opts.File == "" {
|
||||
return report, errors.New("give --file")
|
||||
}
|
||||
rows, err := readJSONLines(opts.File)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
out.Info("The export holds no FAQs; nothing to copy (0 FAQs).")
|
||||
return report, nil
|
||||
}
|
||||
db = db.WithContext(ctx)
|
||||
locales, err := localeIDs(db)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
faqs := make([]models.FAQ, 0, len(rows))
|
||||
for i, row := range rows {
|
||||
faq, code, err := decodeFAQ(row)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("%s row %d: %w", opts.File, i+1, err)
|
||||
}
|
||||
id, ok := locales[code]
|
||||
if !ok {
|
||||
return report, fmt.Errorf("FAQ %d '%s': locale '%s' is not a translate locale; add it first (nothing was copied)", faq.ID, faq.Slug, code)
|
||||
}
|
||||
faq.LocaleID = id
|
||||
faqs = append(faqs, faq)
|
||||
report.Lines = append(report.Lines, fmt.Sprintf("FAQ %d '%s' (%s): %d questions", faq.ID, faq.Slug, code, len(faq.Items())))
|
||||
}
|
||||
report.FAQs = len(faqs)
|
||||
for _, line := range report.Lines {
|
||||
out.Info(line)
|
||||
}
|
||||
var existing int64
|
||||
if err := db.Model(&models.FAQ{}).Count(&existing).Error; err != nil {
|
||||
return report, err
|
||||
}
|
||||
if existing > 0 && !opts.Force {
|
||||
msg := fmt.Sprintf("golem15_faq_faqs already holds %d FAQs; rerun with --force to replace them.", existing)
|
||||
if !opts.DryRun {
|
||||
return report, errors.New(msg)
|
||||
}
|
||||
out.Warning(msg)
|
||||
}
|
||||
if opts.DryRun {
|
||||
out.Info(fmt.Sprintf("Dry run: %d FAQs would be copied.", report.FAQs))
|
||||
out.Info("Dry run complete (no changes made).")
|
||||
return report, nil
|
||||
}
|
||||
err = db.Transaction(func(tx *gorm.DB) error {
|
||||
if existing > 0 {
|
||||
if err := tx.Exec(`DELETE FROM golem15_faq_faqs`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for i := range faqs {
|
||||
if err := tx.Create(&faqs[i]).Error; err != nil {
|
||||
return fmt.Errorf("FAQ %d: %w", faqs[i].ID, err)
|
||||
}
|
||||
}
|
||||
return tx.Exec(`SELECT setval(pg_get_serial_sequence('golem15_faq_faqs', 'id'), COALESCE(MAX(id), 1), MAX(id) IS NOT NULL) FROM golem15_faq_faqs`).Error
|
||||
})
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
out.Success(fmt.Sprintf("Copied %d FAQs.", report.FAQs))
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// localeIDs maps every translate locale code to its id.
|
||||
func localeIDs(db *gorm.DB) (map[string]uint, error) {
|
||||
var locales []translate.Locale
|
||||
if err := db.Find(&locales).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]uint, len(locales))
|
||||
for _, l := range locales {
|
||||
out[l.Code] = l.ID
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// decodeFAQ reads one exported row and returns the FAQ and its locale code.
|
||||
func decodeFAQ(row map[string]json.RawMessage) (models.FAQ, string, error) {
|
||||
var faq models.FAQ
|
||||
id, err := jsonUint(row["id"])
|
||||
if err != nil || id == 0 {
|
||||
return faq, "", fmt.Errorf("id: want a positive id, got %s", row["id"])
|
||||
}
|
||||
faq.ID = id
|
||||
text := func(key string) (string, error) {
|
||||
s, err := jsonText(row[key])
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
if s == nil || strings.TrimSpace(*s) == "" {
|
||||
return "", fmt.Errorf("%s is missing", key)
|
||||
}
|
||||
return *s, nil
|
||||
}
|
||||
if faq.Name, err = text("name"); err != nil {
|
||||
return faq, "", err
|
||||
}
|
||||
if faq.Slug, err = text("slug"); err != nil {
|
||||
return faq, "", err
|
||||
}
|
||||
code, err := text("locale_code")
|
||||
if err != nil {
|
||||
return faq, "", err
|
||||
}
|
||||
if faq.IsActive, err = jsonBool(row["is_active"], false); err != nil {
|
||||
return faq, "", fmt.Errorf("is_active: %w", err)
|
||||
}
|
||||
if faq.Description, err = jsonText(row["description"]); err != nil {
|
||||
return faq, "", fmt.Errorf("description: %w", err)
|
||||
}
|
||||
content, err := decodeContent(row["content"])
|
||||
if err != nil {
|
||||
return faq, "", fmt.Errorf("content: %w", err)
|
||||
}
|
||||
faq.Content = lagoon.Jsonable[models.Content]{Data: content, Valid: content != nil}
|
||||
for key, dest := range map[string]**time.Time{"created_at": &faq.CreatedAt, "updated_at": &faq.UpdatedAt} {
|
||||
if *dest, err = jsonTime(row[key]); err != nil {
|
||||
return faq, "", fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
}
|
||||
return faq, code, nil
|
||||
}
|
||||
|
||||
// decodeContent reads the PHP content: a JSON list of rows, the same list as
|
||||
// JSON text (a TEXT column selected as it is), or an object keyed by row
|
||||
// number; nil for null.
|
||||
func decodeContent(raw json.RawMessage) (models.Content, error) {
|
||||
raw = bytes.TrimSpace(raw)
|
||||
if isNull(raw) {
|
||||
return nil, nil
|
||||
}
|
||||
if raw[0] == '"' {
|
||||
var text string
|
||||
if err := json.Unmarshal(raw, &text); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw = bytes.TrimSpace([]byte(text))
|
||||
if isNull(raw) {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
var list []map[string]any
|
||||
if err := json.Unmarshal(raw, &list); err == nil {
|
||||
return list, nil
|
||||
}
|
||||
var keyed map[string]map[string]any
|
||||
if err := json.Unmarshal(raw, &keyed); err != nil {
|
||||
return nil, errors.New("want a list of question and answer rows")
|
||||
}
|
||||
keys := make([]string, 0, len(keyed))
|
||||
for k := range keyed {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
a, errA := strconv.Atoi(keys[i])
|
||||
b, errB := strconv.Atoi(keys[j])
|
||||
if errA == nil && errB == nil {
|
||||
return a < b
|
||||
}
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
out := make(models.Content, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
out = append(out, keyed[k])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user