feat(17-04): faq:import-winter and the read-only Winter export SQL
- JSON lines import keeping ids, locale_id looked up by locale code (an unknown code aborts before any write) - one transaction, setval past the copied ids, --force replaces, --dry-run reports, an empty file copies nothing - scripts/winter-export.sql: SELECT JSON_OBJECT with the translate locale code - TestFaqImportSmoke
This commit is contained in:
166
console/helpers.go
Normal file
166
console/helpers.go
Normal file
@@ -0,0 +1,166 @@
|
||||
// Package console holds the FAQ plugin's commands: faq:import-winter, a copy
|
||||
// of a WinterCMS site's FAQs.
|
||||
package console
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/bonfire"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func flagValue(in bonfire.Input, name string) string {
|
||||
v, ok := in.Flag(name)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
|
||||
func flagOn(in bonfire.Input, name string) bool {
|
||||
switch strings.ToLower(flagValue(in, name)) {
|
||||
case "true", "1", "yes":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// lookupDB returns the app database. The generated main.go publishes no
|
||||
// database for plugin commands, so when none is published it opens
|
||||
// database.dsn and publishes it the way lagoon's migrate does. The returned
|
||||
// func closes a pool this call opened; it is a no-op otherwise.
|
||||
func lookupDB(ctx context.Context, app *backpack.App) (*gorm.DB, func(), error) {
|
||||
if app == nil {
|
||||
return nil, nil, fmt.Errorf("golem15.faq: app is nil")
|
||||
}
|
||||
if db, ok := app.Lookup[*gorm.DB](); ok && db != nil {
|
||||
return db, func() {}, nil
|
||||
}
|
||||
if lagoon.DSN(app.Config) == "" {
|
||||
return nil, nil, fmt.Errorf("golem15.faq: database is not configured (set SUMMER_DATABASE__DSN)")
|
||||
}
|
||||
sqlDB, gdb, err := lagoon.OpenFromApp(ctx, app)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := lagoon.Publish(app, sqlDB, gdb); err != nil {
|
||||
_ = sqlDB.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
return gdb, func() { _ = sqlDB.Close() }, nil
|
||||
}
|
||||
|
||||
// readJSONLines decodes one JSON object per non-blank line of file.
|
||||
func readJSONLines(file string) ([]map[string]json.RawMessage, error) {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("File not found: %s", file)
|
||||
}
|
||||
defer f.Close()
|
||||
var rows []map[string]json.RawMessage
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 1<<20), 64<<20)
|
||||
n := 0
|
||||
for scanner.Scan() {
|
||||
n++
|
||||
line := bytes.TrimSpace(scanner.Bytes())
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
var row map[string]json.RawMessage
|
||||
if err := json.Unmarshal(line, &row); err != nil {
|
||||
return nil, fmt.Errorf("%s line %d: %w", file, n, err)
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// isNull reports a missing or null JSON value.
|
||||
func isNull(raw json.RawMessage) bool {
|
||||
t := bytes.TrimSpace(raw)
|
||||
return len(t) == 0 || bytes.Equal(t, []byte("null"))
|
||||
}
|
||||
|
||||
// jsonText reads a string, number or boolean as text; nil for null.
|
||||
func jsonText(raw json.RawMessage) (*string, error) {
|
||||
if isNull(raw) {
|
||||
return nil, nil
|
||||
}
|
||||
var v any
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var s string
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
s = x
|
||||
case float64:
|
||||
s = strconv.FormatFloat(x, 'f', -1, 64)
|
||||
case bool:
|
||||
s = strconv.FormatBool(x)
|
||||
default:
|
||||
return nil, fmt.Errorf("want text, got %s", raw)
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// jsonUint reads a positive integer given as a number or a string; 0 for
|
||||
// null.
|
||||
func jsonUint(raw json.RawMessage) (uint, error) {
|
||||
s, err := jsonText(raw)
|
||||
if err != nil || s == nil || strings.TrimSpace(*s) == "" {
|
||||
return 0, err
|
||||
}
|
||||
n, err := strconv.ParseUint(strings.TrimSpace(*s), 10, 32)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("want an id, got %s", raw)
|
||||
}
|
||||
return uint(n), nil
|
||||
}
|
||||
|
||||
// jsonBool reads MySQL's 0 and 1, a JSON boolean or "true"/"false"; def for
|
||||
// null.
|
||||
func jsonBool(raw json.RawMessage, def bool) (bool, error) {
|
||||
s, err := jsonText(raw)
|
||||
if err != nil || s == nil {
|
||||
return def, err
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(*s)) {
|
||||
case "1", "true":
|
||||
return true, nil
|
||||
case "0", "false", "":
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("want a boolean, got %s", raw)
|
||||
}
|
||||
|
||||
// jsonTime reads a MySQL DATETIME text (with or without fractions) or an
|
||||
// RFC 3339 time as UTC; nil for null.
|
||||
func jsonTime(raw json.RawMessage) (*time.Time, error) {
|
||||
s, err := jsonText(raw)
|
||||
if err != nil || s == nil || strings.TrimSpace(*s) == "" {
|
||||
return nil, err
|
||||
}
|
||||
text := strings.TrimSpace(*s)
|
||||
for _, layout := range []string{"2006-01-02 15:04:05.999999", "2006-01-02 15:04:05", time.RFC3339Nano} {
|
||||
if t, err := time.ParseInLocation(layout, text, time.UTC); err == nil {
|
||||
t = t.UTC()
|
||||
return &t, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("want a date and time, got %s", raw)
|
||||
}
|
||||
248
console/import_winter.go
Normal file
248
console/import_winter.go
Normal file
@@ -0,0 +1,248 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-faq-plugin/models"
|
||||
translate "git.golem15.com/golem15/sm-translate-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/bonfire"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// WinterOptions are the inputs of faq:import-winter. File is the output of
|
||||
// scripts/winter-export.sql: one JSON object per line with id, name, slug,
|
||||
// locale_code, is_active, description, content (the PHP JSON text or a
|
||||
// list), created_at and updated_at. Force replaces the FAQs a non-empty table
|
||||
// holds; DryRun reports without writing.
|
||||
type WinterOptions struct {
|
||||
File string
|
||||
Force bool
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// WinterReport is what faq:import-winter copied (or would copy on a dry
|
||||
// run): the number of FAQs and one line per FAQ.
|
||||
type WinterReport struct {
|
||||
FAQs int
|
||||
Lines []string
|
||||
}
|
||||
|
||||
// ImportWinter is faq:import-winter: it copies a WinterCMS site's FAQs into
|
||||
// golem15_faq_faqs, keeping their ids. Each row's locale is looked up by its
|
||||
// code among the translate locales; an unknown code aborts the import before
|
||||
// anything is written. A table that already holds FAQs is refused without
|
||||
// --force, which replaces them in the same transaction. The id sequence is
|
||||
// moved past the copied ids.
|
||||
func ImportWinter(app *backpack.App) bonfire.Command {
|
||||
return bonfire.Command{
|
||||
Name: "faq:import-winter",
|
||||
Description: "Copy a WinterCMS FAQ export into this application",
|
||||
Flags: []bonfire.Flag{
|
||||
{Name: "file", Description: "FAQ export (JSON lines from scripts/winter-export.sql)"},
|
||||
{Name: "force", Description: "Replace the FAQs the table already holds", Bare: true},
|
||||
{Name: "dry-run", Description: "Report what would be copied without writing", Bare: true},
|
||||
},
|
||||
Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
|
||||
opts := WinterOptions{
|
||||
File: flagValue(in, "file"),
|
||||
Force: flagOn(in, "force"),
|
||||
DryRun: flagOn(in, "dry-run"),
|
||||
}
|
||||
db, done, err := lookupDB(ctx, app)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done()
|
||||
_, err = ImportWinterFile(ctx, db, opts, out)
|
||||
return err
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ImportWinterFile runs faq:import-winter against db.
|
||||
func ImportWinterFile(ctx context.Context, db *gorm.DB, opts WinterOptions, out bonfire.Output) (WinterReport, error) {
|
||||
var report WinterReport
|
||||
if opts.File == "" {
|
||||
return report, errors.New("give --file")
|
||||
}
|
||||
rows, err := readJSONLines(opts.File)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
out.Info("The export holds no FAQs; nothing to copy (0 FAQs).")
|
||||
return report, nil
|
||||
}
|
||||
db = db.WithContext(ctx)
|
||||
locales, err := localeIDs(db)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
faqs := make([]models.FAQ, 0, len(rows))
|
||||
for i, row := range rows {
|
||||
faq, code, err := decodeFAQ(row)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("%s row %d: %w", opts.File, i+1, err)
|
||||
}
|
||||
id, ok := locales[code]
|
||||
if !ok {
|
||||
return report, fmt.Errorf("FAQ %d '%s': locale '%s' is not a translate locale; add it first (nothing was copied)", faq.ID, faq.Slug, code)
|
||||
}
|
||||
faq.LocaleID = id
|
||||
faqs = append(faqs, faq)
|
||||
report.Lines = append(report.Lines, fmt.Sprintf("FAQ %d '%s' (%s): %d questions", faq.ID, faq.Slug, code, len(faq.Items())))
|
||||
}
|
||||
report.FAQs = len(faqs)
|
||||
for _, line := range report.Lines {
|
||||
out.Info(line)
|
||||
}
|
||||
var existing int64
|
||||
if err := db.Model(&models.FAQ{}).Count(&existing).Error; err != nil {
|
||||
return report, err
|
||||
}
|
||||
if existing > 0 && !opts.Force {
|
||||
msg := fmt.Sprintf("golem15_faq_faqs already holds %d FAQs; rerun with --force to replace them.", existing)
|
||||
if !opts.DryRun {
|
||||
return report, errors.New(msg)
|
||||
}
|
||||
out.Warning(msg)
|
||||
}
|
||||
if opts.DryRun {
|
||||
out.Info(fmt.Sprintf("Dry run: %d FAQs would be copied.", report.FAQs))
|
||||
out.Info("Dry run complete (no changes made).")
|
||||
return report, nil
|
||||
}
|
||||
err = db.Transaction(func(tx *gorm.DB) error {
|
||||
if existing > 0 {
|
||||
if err := tx.Exec(`DELETE FROM golem15_faq_faqs`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for i := range faqs {
|
||||
if err := tx.Create(&faqs[i]).Error; err != nil {
|
||||
return fmt.Errorf("FAQ %d: %w", faqs[i].ID, err)
|
||||
}
|
||||
}
|
||||
return tx.Exec(`SELECT setval(pg_get_serial_sequence('golem15_faq_faqs', 'id'), COALESCE(MAX(id), 1), MAX(id) IS NOT NULL) FROM golem15_faq_faqs`).Error
|
||||
})
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
out.Success(fmt.Sprintf("Copied %d FAQs.", report.FAQs))
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// localeIDs maps every translate locale code to its id.
|
||||
func localeIDs(db *gorm.DB) (map[string]uint, error) {
|
||||
var locales []translate.Locale
|
||||
if err := db.Find(&locales).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]uint, len(locales))
|
||||
for _, l := range locales {
|
||||
out[l.Code] = l.ID
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// decodeFAQ reads one exported row and returns the FAQ and its locale code.
|
||||
func decodeFAQ(row map[string]json.RawMessage) (models.FAQ, string, error) {
|
||||
var faq models.FAQ
|
||||
id, err := jsonUint(row["id"])
|
||||
if err != nil || id == 0 {
|
||||
return faq, "", fmt.Errorf("id: want a positive id, got %s", row["id"])
|
||||
}
|
||||
faq.ID = id
|
||||
text := func(key string) (string, error) {
|
||||
s, err := jsonText(row[key])
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
if s == nil || strings.TrimSpace(*s) == "" {
|
||||
return "", fmt.Errorf("%s is missing", key)
|
||||
}
|
||||
return *s, nil
|
||||
}
|
||||
if faq.Name, err = text("name"); err != nil {
|
||||
return faq, "", err
|
||||
}
|
||||
if faq.Slug, err = text("slug"); err != nil {
|
||||
return faq, "", err
|
||||
}
|
||||
code, err := text("locale_code")
|
||||
if err != nil {
|
||||
return faq, "", err
|
||||
}
|
||||
if faq.IsActive, err = jsonBool(row["is_active"], false); err != nil {
|
||||
return faq, "", fmt.Errorf("is_active: %w", err)
|
||||
}
|
||||
if faq.Description, err = jsonText(row["description"]); err != nil {
|
||||
return faq, "", fmt.Errorf("description: %w", err)
|
||||
}
|
||||
content, err := decodeContent(row["content"])
|
||||
if err != nil {
|
||||
return faq, "", fmt.Errorf("content: %w", err)
|
||||
}
|
||||
faq.Content = lagoon.Jsonable[models.Content]{Data: content, Valid: content != nil}
|
||||
for key, dest := range map[string]**time.Time{"created_at": &faq.CreatedAt, "updated_at": &faq.UpdatedAt} {
|
||||
if *dest, err = jsonTime(row[key]); err != nil {
|
||||
return faq, "", fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
}
|
||||
return faq, code, nil
|
||||
}
|
||||
|
||||
// decodeContent reads the PHP content: a JSON list of rows, the same list as
|
||||
// JSON text (a TEXT column selected as it is), or an object keyed by row
|
||||
// number; nil for null.
|
||||
func decodeContent(raw json.RawMessage) (models.Content, error) {
|
||||
raw = bytes.TrimSpace(raw)
|
||||
if isNull(raw) {
|
||||
return nil, nil
|
||||
}
|
||||
if raw[0] == '"' {
|
||||
var text string
|
||||
if err := json.Unmarshal(raw, &text); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw = bytes.TrimSpace([]byte(text))
|
||||
if isNull(raw) {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
var list []map[string]any
|
||||
if err := json.Unmarshal(raw, &list); err == nil {
|
||||
return list, nil
|
||||
}
|
||||
var keyed map[string]map[string]any
|
||||
if err := json.Unmarshal(raw, &keyed); err != nil {
|
||||
return nil, errors.New("want a list of question and answer rows")
|
||||
}
|
||||
keys := make([]string, 0, len(keyed))
|
||||
for k := range keyed {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
a, errA := strconv.Atoi(keys[i])
|
||||
b, errB := strconv.Atoi(keys[j])
|
||||
if errA == nil && errB == nil {
|
||||
return a < b
|
||||
}
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
out := make(models.Content, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
out = append(out, keyed[k])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user