diff --git a/console/helpers.go b/console/helpers.go new file mode 100644 index 0000000..0625c3a --- /dev/null +++ b/console/helpers.go @@ -0,0 +1,166 @@ +// Package console holds the FAQ plugin's commands: faq:import-winter, a copy +// of a WinterCMS site's FAQs. +package console + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "time" + + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/bonfire" + "git.golem15.com/golem15/summercms/modules/lagoon" + "gorm.io/gorm" +) + +func flagValue(in bonfire.Input, name string) string { + v, ok := in.Flag(name) + if !ok { + return "" + } + return strings.TrimSpace(v) +} + +func flagOn(in bonfire.Input, name string) bool { + switch strings.ToLower(flagValue(in, name)) { + case "true", "1", "yes": + return true + } + return false +} + +// lookupDB returns the app database. The generated main.go publishes no +// database for plugin commands, so when none is published it opens +// database.dsn and publishes it the way lagoon's migrate does. The returned +// func closes a pool this call opened; it is a no-op otherwise. +func lookupDB(ctx context.Context, app *backpack.App) (*gorm.DB, func(), error) { + if app == nil { + return nil, nil, fmt.Errorf("golem15.faq: app is nil") + } + if db, ok := app.Lookup[*gorm.DB](); ok && db != nil { + return db, func() {}, nil + } + if lagoon.DSN(app.Config) == "" { + return nil, nil, fmt.Errorf("golem15.faq: database is not configured (set SUMMER_DATABASE__DSN)") + } + sqlDB, gdb, err := lagoon.OpenFromApp(ctx, app) + if err != nil { + return nil, nil, err + } + if err := lagoon.Publish(app, sqlDB, gdb); err != nil { + _ = sqlDB.Close() + return nil, nil, err + } + return gdb, func() { _ = sqlDB.Close() }, nil +} + +// readJSONLines decodes one JSON object per non-blank line of file. +func readJSONLines(file string) ([]map[string]json.RawMessage, error) { + f, err := os.Open(file) + if err != nil { + return nil, fmt.Errorf("File not found: %s", file) + } + defer f.Close() + var rows []map[string]json.RawMessage + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 1<<20), 64<<20) + n := 0 + for scanner.Scan() { + n++ + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 { + continue + } + var row map[string]json.RawMessage + if err := json.Unmarshal(line, &row); err != nil { + return nil, fmt.Errorf("%s line %d: %w", file, n, err) + } + rows = append(rows, row) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return rows, nil +} + +// isNull reports a missing or null JSON value. +func isNull(raw json.RawMessage) bool { + t := bytes.TrimSpace(raw) + return len(t) == 0 || bytes.Equal(t, []byte("null")) +} + +// jsonText reads a string, number or boolean as text; nil for null. +func jsonText(raw json.RawMessage) (*string, error) { + if isNull(raw) { + return nil, nil + } + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return nil, err + } + var s string + switch x := v.(type) { + case string: + s = x + case float64: + s = strconv.FormatFloat(x, 'f', -1, 64) + case bool: + s = strconv.FormatBool(x) + default: + return nil, fmt.Errorf("want text, got %s", raw) + } + return &s, nil +} + +// jsonUint reads a positive integer given as a number or a string; 0 for +// null. +func jsonUint(raw json.RawMessage) (uint, error) { + s, err := jsonText(raw) + if err != nil || s == nil || strings.TrimSpace(*s) == "" { + return 0, err + } + n, err := strconv.ParseUint(strings.TrimSpace(*s), 10, 32) + if err != nil { + return 0, fmt.Errorf("want an id, got %s", raw) + } + return uint(n), nil +} + +// jsonBool reads MySQL's 0 and 1, a JSON boolean or "true"/"false"; def for +// null. +func jsonBool(raw json.RawMessage, def bool) (bool, error) { + s, err := jsonText(raw) + if err != nil || s == nil { + return def, err + } + switch strings.ToLower(strings.TrimSpace(*s)) { + case "1", "true": + return true, nil + case "0", "false", "": + return false, nil + } + return false, fmt.Errorf("want a boolean, got %s", raw) +} + +// jsonTime reads a MySQL DATETIME text (with or without fractions) or an +// RFC 3339 time as UTC; nil for null. +func jsonTime(raw json.RawMessage) (*time.Time, error) { + s, err := jsonText(raw) + if err != nil || s == nil || strings.TrimSpace(*s) == "" { + return nil, err + } + text := strings.TrimSpace(*s) + for _, layout := range []string{"2006-01-02 15:04:05.999999", "2006-01-02 15:04:05", time.RFC3339Nano} { + if t, err := time.ParseInLocation(layout, text, time.UTC); err == nil { + t = t.UTC() + return &t, nil + } + } + return nil, fmt.Errorf("want a date and time, got %s", raw) +} diff --git a/console/import_winter.go b/console/import_winter.go new file mode 100644 index 0000000..782980c --- /dev/null +++ b/console/import_winter.go @@ -0,0 +1,248 @@ +package console + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "git.golem15.com/golem15/sm-faq-plugin/models" + translate "git.golem15.com/golem15/sm-translate-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/bonfire" + "git.golem15.com/golem15/summercms/modules/lagoon" + "gorm.io/gorm" +) + +// WinterOptions are the inputs of faq:import-winter. File is the output of +// scripts/winter-export.sql: one JSON object per line with id, name, slug, +// locale_code, is_active, description, content (the PHP JSON text or a +// list), created_at and updated_at. Force replaces the FAQs a non-empty table +// holds; DryRun reports without writing. +type WinterOptions struct { + File string + Force bool + DryRun bool +} + +// WinterReport is what faq:import-winter copied (or would copy on a dry +// run): the number of FAQs and one line per FAQ. +type WinterReport struct { + FAQs int + Lines []string +} + +// ImportWinter is faq:import-winter: it copies a WinterCMS site's FAQs into +// golem15_faq_faqs, keeping their ids. Each row's locale is looked up by its +// code among the translate locales; an unknown code aborts the import before +// anything is written. A table that already holds FAQs is refused without +// --force, which replaces them in the same transaction. The id sequence is +// moved past the copied ids. +func ImportWinter(app *backpack.App) bonfire.Command { + return bonfire.Command{ + Name: "faq:import-winter", + Description: "Copy a WinterCMS FAQ export into this application", + Flags: []bonfire.Flag{ + {Name: "file", Description: "FAQ export (JSON lines from scripts/winter-export.sql)"}, + {Name: "force", Description: "Replace the FAQs the table already holds", Bare: true}, + {Name: "dry-run", Description: "Report what would be copied without writing", Bare: true}, + }, + Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { + opts := WinterOptions{ + File: flagValue(in, "file"), + Force: flagOn(in, "force"), + DryRun: flagOn(in, "dry-run"), + } + db, done, err := lookupDB(ctx, app) + if err != nil { + return err + } + defer done() + _, err = ImportWinterFile(ctx, db, opts, out) + return err + }, + } +} + +// ImportWinterFile runs faq:import-winter against db. +func ImportWinterFile(ctx context.Context, db *gorm.DB, opts WinterOptions, out bonfire.Output) (WinterReport, error) { + var report WinterReport + if opts.File == "" { + return report, errors.New("give --file") + } + rows, err := readJSONLines(opts.File) + if err != nil { + return report, err + } + if len(rows) == 0 { + out.Info("The export holds no FAQs; nothing to copy (0 FAQs).") + return report, nil + } + db = db.WithContext(ctx) + locales, err := localeIDs(db) + if err != nil { + return report, err + } + faqs := make([]models.FAQ, 0, len(rows)) + for i, row := range rows { + faq, code, err := decodeFAQ(row) + if err != nil { + return report, fmt.Errorf("%s row %d: %w", opts.File, i+1, err) + } + id, ok := locales[code] + if !ok { + return report, fmt.Errorf("FAQ %d '%s': locale '%s' is not a translate locale; add it first (nothing was copied)", faq.ID, faq.Slug, code) + } + faq.LocaleID = id + faqs = append(faqs, faq) + report.Lines = append(report.Lines, fmt.Sprintf("FAQ %d '%s' (%s): %d questions", faq.ID, faq.Slug, code, len(faq.Items()))) + } + report.FAQs = len(faqs) + for _, line := range report.Lines { + out.Info(line) + } + var existing int64 + if err := db.Model(&models.FAQ{}).Count(&existing).Error; err != nil { + return report, err + } + if existing > 0 && !opts.Force { + msg := fmt.Sprintf("golem15_faq_faqs already holds %d FAQs; rerun with --force to replace them.", existing) + if !opts.DryRun { + return report, errors.New(msg) + } + out.Warning(msg) + } + if opts.DryRun { + out.Info(fmt.Sprintf("Dry run: %d FAQs would be copied.", report.FAQs)) + out.Info("Dry run complete (no changes made).") + return report, nil + } + err = db.Transaction(func(tx *gorm.DB) error { + if existing > 0 { + if err := tx.Exec(`DELETE FROM golem15_faq_faqs`).Error; err != nil { + return err + } + } + for i := range faqs { + if err := tx.Create(&faqs[i]).Error; err != nil { + return fmt.Errorf("FAQ %d: %w", faqs[i].ID, err) + } + } + return tx.Exec(`SELECT setval(pg_get_serial_sequence('golem15_faq_faqs', 'id'), COALESCE(MAX(id), 1), MAX(id) IS NOT NULL) FROM golem15_faq_faqs`).Error + }) + if err != nil { + return report, err + } + out.Success(fmt.Sprintf("Copied %d FAQs.", report.FAQs)) + return report, nil +} + +// localeIDs maps every translate locale code to its id. +func localeIDs(db *gorm.DB) (map[string]uint, error) { + var locales []translate.Locale + if err := db.Find(&locales).Error; err != nil { + return nil, err + } + out := make(map[string]uint, len(locales)) + for _, l := range locales { + out[l.Code] = l.ID + } + return out, nil +} + +// decodeFAQ reads one exported row and returns the FAQ and its locale code. +func decodeFAQ(row map[string]json.RawMessage) (models.FAQ, string, error) { + var faq models.FAQ + id, err := jsonUint(row["id"]) + if err != nil || id == 0 { + return faq, "", fmt.Errorf("id: want a positive id, got %s", row["id"]) + } + faq.ID = id + text := func(key string) (string, error) { + s, err := jsonText(row[key]) + if err != nil { + return "", fmt.Errorf("%s: %w", key, err) + } + if s == nil || strings.TrimSpace(*s) == "" { + return "", fmt.Errorf("%s is missing", key) + } + return *s, nil + } + if faq.Name, err = text("name"); err != nil { + return faq, "", err + } + if faq.Slug, err = text("slug"); err != nil { + return faq, "", err + } + code, err := text("locale_code") + if err != nil { + return faq, "", err + } + if faq.IsActive, err = jsonBool(row["is_active"], false); err != nil { + return faq, "", fmt.Errorf("is_active: %w", err) + } + if faq.Description, err = jsonText(row["description"]); err != nil { + return faq, "", fmt.Errorf("description: %w", err) + } + content, err := decodeContent(row["content"]) + if err != nil { + return faq, "", fmt.Errorf("content: %w", err) + } + faq.Content = lagoon.Jsonable[models.Content]{Data: content, Valid: content != nil} + for key, dest := range map[string]**time.Time{"created_at": &faq.CreatedAt, "updated_at": &faq.UpdatedAt} { + if *dest, err = jsonTime(row[key]); err != nil { + return faq, "", fmt.Errorf("%s: %w", key, err) + } + } + return faq, code, nil +} + +// decodeContent reads the PHP content: a JSON list of rows, the same list as +// JSON text (a TEXT column selected as it is), or an object keyed by row +// number; nil for null. +func decodeContent(raw json.RawMessage) (models.Content, error) { + raw = bytes.TrimSpace(raw) + if isNull(raw) { + return nil, nil + } + if raw[0] == '"' { + var text string + if err := json.Unmarshal(raw, &text); err != nil { + return nil, err + } + raw = bytes.TrimSpace([]byte(text)) + if isNull(raw) { + return nil, nil + } + } + var list []map[string]any + if err := json.Unmarshal(raw, &list); err == nil { + return list, nil + } + var keyed map[string]map[string]any + if err := json.Unmarshal(raw, &keyed); err != nil { + return nil, errors.New("want a list of question and answer rows") + } + keys := make([]string, 0, len(keyed)) + for k := range keyed { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + a, errA := strconv.Atoi(keys[i]) + b, errB := strconv.Atoi(keys[j]) + if errA == nil && errB == nil { + return a < b + } + return keys[i] < keys[j] + }) + out := make(models.Content, 0, len(keys)) + for _, k := range keys { + out = append(out, keyed[k]) + } + return out, nil +} diff --git a/faq_smoke_test.go b/faq_smoke_test.go index 556d3a4..dac5213 100644 --- a/faq_smoke_test.go +++ b/faq_smoke_test.go @@ -15,9 +15,11 @@ import ( "time" _ "git.golem15.com/golem15/sm-faq-plugin" + "git.golem15.com/golem15/sm-faq-plugin/console" "git.golem15.com/golem15/sm-faq-plugin/models" _ "git.golem15.com/golem15/sm-translate-plugin" "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/bonfire" "git.golem15.com/golem15/summercms/modules/bouncer" "git.golem15.com/golem15/summercms/modules/cabana" "git.golem15.com/golem15/summercms/modules/compass" @@ -250,3 +252,115 @@ func TestFaqAdminSmoke(t *testing.T) { t.Fatalf("list without permission: %d %s", rec.Code, rec.Body.String()) } } + +func writeFile(t *testing.T, dir, name, body string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func testOutput() (bonfire.Output, *bytes.Buffer) { + var buf bytes.Buffer + return bonfire.NewOutput(strings.NewReader(""), &buf, &buf), &buf +} + +func (e *faqEnv) countFAQs(t *testing.T) int64 { + t.Helper() + var n int64 + if err := e.gdb.Model(&models.FAQ{}).Count(&n).Error; err != nil { + t.Fatal(err) + } + return n +} + +// TestFaqImportSmoke copies a Winter export through faq:import-winter: two +// locales by code, ids kept, the sequence moved past them, an unknown code +// aborting before any write, a second copy refused without --force and an +// empty file reporting zero. +func TestFaqImportSmoke(t *testing.T) { + e := bootFAQ(t) + ctx := t.Context() + dir := t.TempDir() + en, pl := e.localeID(t, "en"), e.localeID(t, "pl") + + content, _ := json.Marshal(`[{"question":"How long?","answer":"

Two days.

"},{"question":"Where?","answer":"

Everywhere.

"}]`) + good := writeFile(t, dir, "faqs.jsonl", + `{"id": 4, "name": "Shipping", "slug": "shipping", "locale_code": "en", "is_active": 1, "description": "

About shipping.

", "content": `+string(content)+`, "created_at": "2024-03-01 10:00:00.000000", "updated_at": "2024-03-02 11:30:00.000000"}`+"\n"+ + `{"id": 9, "name": "Wysyłka", "slug": "shipping", "locale_code": "pl", "is_active": 0, "description": null, "content": [{"question":"Jak długo?","answer":"

Dwa dni.

"}], "created_at": null, "updated_at": null}`+"\n\n") + unknown := writeFile(t, dir, "unknown.jsonl", + `{"id": 1, "name": "A", "slug": "a", "locale_code": "en", "is_active": 1, "content": null}`+"\n"+ + `{"id": 2, "name": "B", "slug": "b", "locale_code": "de", "is_active": 1, "content": null}`+"\n") + empty := writeFile(t, dir, "empty.jsonl", "\n") + + // An unknown locale code aborts before anything is written. + out, buf := testOutput() + if _, err := console.ImportWinterFile(ctx, e.gdb, console.WinterOptions{File: unknown}, out); err == nil || !strings.Contains(err.Error(), "'de'") { + t.Fatalf("unknown locale: %v\n%s", err, buf.String()) + } + if n := e.countFAQs(t); n != 0 { + t.Fatalf("unknown locale wrote %d rows", n) + } + + // A dry run reports and writes nothing. + out, buf = testOutput() + if report, err := console.ImportWinterFile(ctx, e.gdb, console.WinterOptions{File: good, DryRun: true}, out); err != nil || report.FAQs != 2 { + t.Fatalf("dry run: %+v %v\n%s", report, err, buf.String()) + } + if n := e.countFAQs(t); n != 0 || !strings.Contains(buf.String(), "Dry run complete") { + t.Fatalf("dry run wrote %d rows\n%s", n, buf.String()) + } + + out, buf = testOutput() + report, err := console.ImportWinterFile(ctx, e.gdb, console.WinterOptions{File: good}, out) + if err != nil || report.FAQs != 2 { + t.Fatalf("import: %+v %v\n%s", report, err, buf.String()) + } + var rows []models.FAQ + if err := e.gdb.Order("id").Find(&rows).Error; err != nil { + t.Fatal(err) + } + if len(rows) != 2 || rows[0].ID != 4 || rows[0].LocaleID != en || !rows[0].IsActive || rows[1].ID != 9 || rows[1].LocaleID != pl || rows[1].IsActive { + t.Fatalf("copied rows = %+v", rows) + } + if items := rows[0].Items(); len(items) != 2 || items[1]["question"] != "Where?" || rows[0].CreatedAt == nil || !rows[0].CreatedAt.Equal(time.Date(2024, 3, 1, 10, 0, 0, 0, time.UTC)) { + t.Fatalf("copied content = %v created %v", items, rows[0].CreatedAt) + } + if items := rows[1].Items(); len(items) != 1 || items[0]["answer"] != "

Dwa dni.

" { + t.Fatalf("copied pl content = %v", items) + } + + // The sequence moved past the copied ids: an admin create gets id 10. + token := e.admin(t, permFAQs) + rec := e.call(t, http.MethodPost, "/golem15/faq/faqs", token, map[string]any{"name": "Returns", "slug": "returns", "locale": en}) + if rec.Code != http.StatusCreated || !strings.Contains(rec.Body.String(), `"id":10`) { + t.Fatalf("create after import: %d %s", rec.Code, rec.Body.String()) + } + + // A second copy is refused without --force and replaces with it. + out, buf = testOutput() + if _, err := console.ImportWinterFile(ctx, e.gdb, console.WinterOptions{File: good}, out); err == nil || !strings.Contains(err.Error(), "--force") { + t.Fatalf("second import: %v\n%s", err, buf.String()) + } + if n := e.countFAQs(t); n != 3 { + t.Fatalf("refused import changed the table: %d rows", n) + } + out, buf = testOutput() + if _, err := console.ImportWinterFile(ctx, e.gdb, console.WinterOptions{File: good, Force: true}, out); err != nil { + t.Fatalf("forced import: %v\n%s", err, buf.String()) + } + if n := e.countFAQs(t); n != 2 { + t.Fatalf("forced import left %d rows", n) + } + + // An empty file writes nothing and reports zero. + out, buf = testOutput() + if report, err := console.ImportWinterFile(ctx, e.gdb, console.WinterOptions{File: empty}, out); err != nil || report.FAQs != 0 || !strings.Contains(buf.String(), "0 FAQs") { + t.Fatalf("empty file: %+v %v\n%s", report, err, buf.String()) + } + if n := e.countFAQs(t); n != 2 { + t.Fatalf("empty file changed the table: %d rows", n) + } +} diff --git a/plugin.go b/plugin.go index 8dafd85..a3f3cf9 100644 --- a/plugin.go +++ b/plugin.go @@ -4,9 +4,11 @@ package faq import ( + "git.golem15.com/golem15/sm-faq-plugin/console" "git.golem15.com/golem15/sm-faq-plugin/models" "git.golem15.com/golem15/sm-faq-plugin/updates" "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/bonfire" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/party" "github.com/go-gormigrate/gormigrate/v2" @@ -16,6 +18,7 @@ var ( _ party.Plugin = (*Plugin)(nil) _ pact.HasMigrations = (*Plugin)(nil) _ pact.HasModels = (*Plugin)(nil) + _ pact.HasCommands = (*Plugin)(nil) ) // Plugin is the golem15.faq plugin. @@ -38,6 +41,11 @@ func (p *Plugin) Boot(app *backpack.App) error { return nil } +// Commands registers faq:import-winter. +func (p *Plugin) Commands() []bonfire.Command { + return []bonfire.Command{console.ImportWinter(p.app)} +} + func (p *Plugin) Migrations() []*gormigrate.Migration { return updates.All() } func (p *Plugin) Models() []any { return models.All() } diff --git a/scripts/winter-export.sql b/scripts/winter-export.sql new file mode 100644 index 0000000..b6d131b --- /dev/null +++ b/scripts/winter-export.sql @@ -0,0 +1,39 @@ +-- FAQs of a WinterCMS site with the Golem15.FAQ plugin, one JSON object per +-- line: the --file input of faq:import-winter. +-- +-- mysql -N -B --raw < winter-export.sql > faqs.jsonl +-- +-- Read-only: it selects from the FAQ table and the translate locales table +-- and changes nothing. JSON_OBJECT needs MySQL 5.7.8+ or MariaDB 10.2.3+. +-- +-- Check two names on the server before running: +-- +-- 1. The FAQ table. This file assumes golem15_faq_faqs: +-- SHOW TABLES LIKE '%faq_faqs'; +-- +-- 2. The translate locales table that locale_id points at. This file +-- assumes golem15_translate_locales (Golem15.Translate). Check: +-- SHOW TABLES LIKE '%translate_locales'; +-- A site on Winter.Translate or RainLab.Translate uses +-- winter_translate_locales or rainlab_translate_locales; change the JOIN +-- line to match. +-- +-- locale_code is the locale's code: the command looks the code up among the +-- application's translate locales, so the ids may differ between the sites. +-- Every code must exist there before the import (an unknown code aborts it). +-- content is the stored question and answer list as text; the command +-- decodes it. +SELECT JSON_OBJECT( + 'id', f.id, + 'name', f.name, + 'slug', f.slug, + 'locale_code', l.code, + 'is_active', f.is_active, + 'description', f.description, + 'content', f.content, + 'created_at', f.created_at, + 'updated_at', f.updated_at +) +FROM golem15_faq_faqs f +LEFT JOIN golem15_translate_locales l ON l.id = f.locale_id +ORDER BY f.id;