package classes import ( "bytes" "compress/gzip" "context" "crypto/hmac" "crypto/rand" "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/json" "errors" "fmt" "io" "sort" "strconv" "strings" "sync" "time" "git.golem15.com/golem15/sm-seo-plugin/models" "git.golem15.com/golem15/summercms/modules/lagoon" "gorm.io/gorm" ) // The datasets of import and export (P-27, UI-SPEC C13). const ( DatasetRedirects = "redirects" DatasetRules = "rules" DatasetRecords = "records" ) // The size caps of import and export (RESEARCH Pitfall 14, T-32-29). const ( // AdminImportLimit is the largest file the import screen previews // (UI-SPEC Q-8 assumption: 60 KB, under the 64 KiB action payload). AdminImportLimit = 60 << 10 // MCPImportLimit is the largest file one MCP import takes; larger files // are imported in chunks. MCPImportLimit = 900 << 10 // ExportActionLimit is the largest export the screen downloads in one // action answer (the action data is capped at 256 KiB). ExportActionLimit = 250 << 10 // ExportPageRows is the number of rows of one export page through MCP. ExportPageRows = 1000 // PreviewRowCap is the number of preview rows the screen shows. PreviewRowCap = 200 ) // DatasetColumns are each dataset's columns in file order; the first one // is required. var DatasetColumns = map[string][]string{ DatasetRedirects: {"source", "match", "target", "status", "query", "ignore_case", "trailing_slash", "note"}, DatasetRules: {"url", "match", "title", "description", "canonical", "robots", "og_title", "og_description", "og_image", "note"}, DatasetRecords: {"url", "locale", "title", "description", "keyphrase", "canonical", "robots", "og_title", "og_description", "og_image"}, } // TransferError refuses an import or export as a whole: Code is // unreadable, missing_column, dataset_unknown, too_large or stale, Key the // phrase key of its message. type TransferError struct { Code string Key string Params map[string]string } func (e *TransferError) Error() string { return "golem15.seo: transfer " + e.Code } // ImportRefusedError refuses an apply while a row is refused; nothing was // written. type ImportRefusedError struct{ Result ImportResult } func (e *ImportRefusedError) Error() string { return "golem15.seo: import refused" } // Export is one export answer: the file name, the CSV text (UTF-8 with a // BOM) and the page of the rows it holds. type Export struct { Dataset string `json:"dataset"` Filename string `json:"filename"` CSV string `json:"csv"` Page int `json:"page"` Pages int `json:"pages"` Total int `json:"total"` } // boolCell writes a bool cell. func boolCell(b bool) string { if b { return "1" } return "0" } // parseBool reads a bool cell: 1, true, yes, on, tak and their opposites. func parseBool(s string) (bool, bool) { switch strings.ToLower(strings.TrimSpace(s)) { case "1", "true", "yes", "on", "tak", "y": return true, true case "0", "false", "no", "off", "nie", "n": return false, true } return false, false } // datasetRows reads every row of a dataset in file order. func datasetRows(db *gorm.DB, dataset string) ([][]string, error) { var out [][]string switch dataset { case DatasetRedirects: var rows []models.Redirect if err := db.Order("id ASC").Find(&rows).Error; err != nil { return nil, err } for _, r := range rows { out = append(out, []string{r.Source, r.Match, r.Target, strconv.Itoa(int(r.Status)), r.Query, boolCell(r.IgnoreCase), boolCell(r.TrailingSlash), r.Note}) } case DatasetRules: var rows []models.Rule if err := db.Order("url ASC, id ASC").Find(&rows).Error; err != nil { return nil, err } for _, r := range rows { out = append(out, []string{r.URL, r.Match, r.Title, r.Description, r.Canonical, r.Robots, r.OGTitle, r.OGDescription, r.OGImage, r.Note}) } case DatasetRecords: var rows []models.Entry if err := db.Where("url <> ''").Order("url ASC, locale ASC").Find(&rows).Error; err != nil { return nil, err } for _, e := range rows { out = append(out, []string{e.URL, e.Locale, e.Title, e.Description, e.Keyphrase, e.Canonical, e.Robots, e.OGTitle, e.OGDescription, e.OGImage}) } default: return nil, &TransferError{Code: "dataset_unknown", Key: "golem15.seo::lang.transfer.dataset_unknown"} } return out, nil } // Export writes one page of a dataset as CSV with the delimiter (D-13): // UTF-8 with a BOM and formula cells prefixed with an apostrophe. perPage // 0 exports every row. func (s *Service) Export(ctx context.Context, dataset, delimiter string, page, perPage int) (Export, error) { db, err := s.DB(ctx) if err != nil { return Export{}, err } rows, err := datasetRows(db, dataset) if err != nil { return Export{}, err } out := Export{Dataset: dataset, Total: len(rows), Page: 1, Pages: 1, Filename: "seo-" + dataset + "-" + time.Now().UTC().Format("2006-01-02") + ".csv"} if perPage > 0 { out.Pages = (len(rows) + perPage - 1) / perPage if out.Pages == 0 { out.Pages = 1 } if page < 1 { page = 1 } out.Page = page start := (page - 1) * perPage if start > len(rows) { start = len(rows) } end := min(start+perPage, len(rows)) rows = rows[start:end] } out.CSV, err = WriteCSV(DatasetColumns[dataset], rows, delimiter) return out, err } // ImportRequest is one import: the dataset, the file and its delimiter // (detected from the header when empty), whether to apply, the writer // history records, and for an apply from a preview token the fingerprint // the preview saw. type ImportRequest struct { Dataset string CSV string Delimiter string DryRun bool Fingerprint string } // PreviewRow is one row of the import preview (UI-SPEC C13): the file // line, the change (new, changed, unchanged or refused), the address, a // changed field with its before and after value, and the note; Warning // marks a row applied with a warning. type PreviewRow struct { Line int `json:"line"` Change string `json:"change"` Address string `json:"address"` Field string `json:"field"` Before string `json:"before"` After string `json:"after"` Note string `json:"note"` Warning bool `json:"warning,omitempty"` } // ImportSummary counts the rows of an import by change. type ImportSummary struct { New int `json:"new"` Changed int `json:"changed"` Unchanged int `json:"unchanged"` Refused int `json:"refused"` Warning int `json:"warning"` } // ImportResult is an import's answer: the summary, the preview rows // (refused first), the number of file rows, the number of changes an // apply writes, the fingerprint of the data the preview read, and after // an apply the batch it wrote. type ImportResult struct { Dataset string `json:"dataset"` Summary ImportSummary `json:"summary"` Rows []PreviewRow `json:"rows"` Total int `json:"total"` Count int `json:"count"` Applied bool `json:"applied"` BatchID string `json:"batch_id,omitempty"` Fingerprint string `json:"-"` } // planned is one file row mapped to a change. type planned struct { row CSVRow address string status string // new, changed, unchanged, refused change int // index in the batch, -1 for none diffs [][3]string notes []string state any } // Import maps a file's rows to changes and runs them as one batch with // via import (D-12, D-13, P-27): a preview is a dry run answering every // row's change, the notes (loops refused, chains, canonicals that // redirect, rows that create a URL rule) and the fingerprint of the data // it read; an apply runs the preview again, refuses with // *ImportRefusedError while a row is refused, with transfer.stale when // the fingerprint differs from req.Fingerprint, and applies every change // or none. func (s *Service) Import(ctx context.Context, req ImportRequest) (ImportResult, error) { header, rows, err := ReadCSV(req.CSV, req.Delimiter) if err != nil { return ImportResult{}, &TransferError{Code: "unreadable", Key: "golem15.seo::lang.transfer.unreadable"} } columns, ok := DatasetColumns[req.Dataset] if !ok { return ImportResult{}, &TransferError{Code: "dataset_unknown", Key: "golem15.seo::lang.transfer.dataset_unknown"} } if !contains(header, columns[0]) { return ImportResult{}, &TransferError{Code: "missing_column", Key: "golem15.seo::lang.transfer.missing_column", Params: map[string]string{"column": columns[0]}} } preview, err := s.importPass(ctx, req.Dataset, header, rows, true, req.Fingerprint) if err != nil || req.DryRun { return preview, err } if preview.Summary.Refused > 0 { return preview, &ImportRefusedError{Result: preview} } if preview.Count == 0 { preview.Applied = true return preview, nil } return s.importPass(ctx, req.Dataset, header, rows, false, preview.Fingerprint) } // contains reports whether list holds v. func contains(list []string, v string) bool { for _, x := range list { if x == v { return true } } return false } // importPass maps and runs the rows once. func (s *Service) importPass(ctx context.Context, dataset string, header []string, rows []CSVRow, dryRun bool, expect string) (ImportResult, error) { res := ImportResult{Dataset: dataset, Total: len(rows), Rows: []PreviewRow{}} var plan []planned batch, err := s.runBatch(ctx, BatchOptions{DryRun: dryRun, Via: models.ViaImport}, func(ctx context.Context, tx *gorm.DB) ([]BatchChange, error) { var changes []BatchChange var err error plan, changes, err = s.planImport(ctx, tx, dataset, header, rows) if err != nil { return nil, err } res.Fingerprint = fingerprint(dataset, plan) if expect != "" && res.Fingerprint != expect { return nil, ErrStale } return changes, nil }) var refused *BatchRefusedError switch { case errors.Is(err, ErrStale): return res, &TransferError{Code: "stale", Key: "golem15.seo::lang.transfer.stale"} case errors.As(err, &refused): batch = refused.Result case err != nil: return res, err } s.previewRows(ctx, &res, plan, batch) if refused != nil && !dryRun { return res, &ImportRefusedError{Result: res} } if !dryRun { res.Applied, res.BatchID = true, batch.BatchID } return res, nil } // fingerprint hashes the data an import read: each row's address and the // state of the object it changes, so an apply sees whether anything // changed since its preview (T-32-25). func fingerprint(dataset string, plan []planned) string { h := sha256.New() h.Write([]byte(dataset)) for _, p := range plan { raw, _ := json.Marshal([]any{p.row.Line, p.address, p.state}) h.Write(raw) } return hex.EncodeToString(h.Sum(nil)) } // cell returns a row's cell and whether the file has the column. func cell(row CSVRow, column string) (string, bool) { v, ok := row.Fields[column] return v, ok } // planImport maps rows to batch changes against the data in tx. func (s *Service) planImport(ctx context.Context, tx *gorm.DB, dataset string, header []string, rows []CSVRow) ([]planned, []BatchChange, error) { var plan []planned var changes []BatchChange add := func(p planned, ch *BatchChange) { p.change = -1 if ch != nil && p.status != "refused" { p.change = len(changes) changes = append(changes, *ch) } plan = append(plan, p) } for _, row := range rows { switch dataset { case DatasetRedirects: p, ch, err := s.planRedirect(ctx, tx, row) if err != nil { return nil, nil, err } add(p, ch) case DatasetRules: p, ch, err := s.planRule(tx, row) if err != nil { return nil, nil, err } add(p, ch) case DatasetRecords: p, ch, err := s.planRecord(ctx, tx, row) if err != nil { return nil, nil, err } add(p, ch) } } return plan, changes, nil } // planRedirect maps a redirects row: the redirect with the same source and // match is updated, else one is created. func (s *Service) planRedirect(ctx context.Context, tx *gorm.DB, row CSVRow) (planned, *BatchChange, error) { source, _ := cell(row, "source") match, _ := cell(row, "match") if match == "" { match = models.RedirectExact } p := planned{row: row, address: source} f := RedirectFields{Source: &source, Match: &match} for _, column := range []string{"target", "query", "note"} { if v, ok := cell(row, column); ok { v := v switch column { case "target": f.Target = &v case "query": if v != "" { f.Query = &v } case "note": f.Note = &v } } } if v, ok := cell(row, "status"); ok && v != "" { n, err := strconv.Atoi(v) if err != nil { p.status = "refused" p.notes = append(p.notes, s.PhraseCtx(ctx, "golem15.seo::lang.transfer.row_invalid", map[string]string{"field": "status", "message": v})) return p, nil, nil } f.Status = &n } for _, column := range []string{"ignore_case", "trailing_slash"} { v, ok := cell(row, column) if !ok || v == "" { continue } b, valid := parseBool(v) if !valid { p.status = "refused" p.notes = append(p.notes, s.PhraseCtx(ctx, "golem15.seo::lang.transfer.row_invalid", map[string]string{"field": column, "message": v})) return p, nil, nil } if column == "ignore_case" { f.IgnoreCase = &b } else { f.TrailingSlash = &b } } var rows []models.Redirect if err := tx.Where("source = ? AND match = ?", strings.TrimSpace(source), strings.TrimSpace(match)).Limit(1).Find(&rows).Error; err != nil { return p, nil, err } if len(rows) == 0 { p.status = "new" after := models.NewRedirect() f.Apply(&after) p.diffs = append(p.diffs, [3]string{"target", "", redirectSummary(after)}) return p, &BatchChange{Op: OpCreateRedirect, Redirect: &f}, nil } cur := rows[0] p.state = RedirectValues(cur) next := cur f.Apply(&next) next.Normalize() before, after := RedirectValues(cur), RedirectValues(next) for _, column := range DatasetColumns[DatasetRedirects] { if !sameValue(before[column], after[column]) { p.diffs = append(p.diffs, [3]string{column, text(jsonNorm(before[column])), text(jsonNorm(after[column]))}) } } if len(p.diffs) == 0 { p.status = "unchanged" return p, nil, nil } p.status = "changed" return p, &BatchChange{Op: OpUpdateRedirect, ID: cur.ID, Redirect: &f}, nil } // redirectSummary is a new redirect in the preview: "→ target (301)". func redirectSummary(r models.Redirect) string { if r.Status == models.Status410 { return "410" } return fmt.Sprintf("→ %s (%d)", r.Target, int(r.Status)) } // planRule maps a rules row: the rule with the same address and match is // updated, else one is created. func (s *Service) planRule(tx *gorm.DB, row CSVRow) (planned, *BatchChange, error) { url, _ := cell(row, "url") match, _ := cell(row, "match") match = models.NormalizeMatch(match) p := planned{row: row, address: url} f := RuleFields{URL: &url, Match: &match} values := map[string]*string{} for _, column := range DatasetColumns[DatasetRules][2:] { if v, ok := cell(row, column); ok { v := strings.TrimSpace(v) if column == "robots" && v != "" { if normal, err := NormalizeRobots(v); err == nil { v = normal } } values[column] = &v } } f.Title, f.Description, f.Canonical, f.Robots = values["title"], values["description"], values["canonical"], values["robots"] f.OGTitle, f.OGDescription, f.OGImage, f.Note = values["og_title"], values["og_description"], values["og_image"], values["note"] var rows []models.Rule if err := tx.Where("url = ? AND match = ?", strings.TrimSpace(url), match).Limit(1).Find(&rows).Error; err != nil { return p, nil, err } if len(rows) == 0 { p.status = "new" for _, column := range DatasetColumns[DatasetRules][2:] { if v := values[column]; v != nil && *v != "" { p.diffs = append(p.diffs, [3]string{column, "", *v}) } } return p, &BatchChange{Op: OpCreateRule, Rule: &f}, nil } cur := rows[0] p.state = RuleValues(cur) for column, v := range values { if old := ruleColumn(cur, column); old != *v { p.diffs = append(p.diffs, [3]string{column, old, *v}) } } sortDiffs(p.diffs, DatasetColumns[DatasetRules]) if len(p.diffs) == 0 { p.status = "unchanged" return p, nil, nil } p.status = "changed" return p, &BatchChange{Op: OpUpdateRule, ID: cur.ID, Rule: &f}, nil } // ruleColumn reads one column of a rule. func ruleColumn(r models.Rule, column string) string { switch column { case "url": return r.URL case "match": return r.Match case "note": return r.Note } return r.Value(column) } // sortDiffs orders diffs by the dataset's column order. func sortDiffs(diffs [][3]string, columns []string) { pos := map[string]int{} for i, c := range columns { pos[c] = i } sort.SliceStable(diffs, func(i, j int) bool { return pos[diffs[i][0]] < pos[diffs[j][0]] }) } // planRecord maps a records row by URL (D-11): the record behind the URL // in the row's locale, else an exact URL rule (transfer.row_rule). func (s *Service) planRecord(ctx context.Context, tx *gorm.DB, row CSVRow) (planned, *BatchChange, error) { url, _ := cell(row, "url") locale, _ := cell(row, "locale") p := planned{row: row, address: url} owner, _, err := s.FindOwner(ctx, tx, url, false) if errors.Is(err, ErrBadURL) { p.status = "refused" p.notes = append(p.notes, s.PhraseCtx(ctx, "golem15.seo::lang.transfer.row_invalid", map[string]string{"field": "url", "message": err.Error()})) return p, nil, nil } if err != nil { return p, nil, err } p.address = owner.URL values := map[string]string{} for _, column := range DatasetColumns[DatasetRecords][2:] { if v, ok := cell(row, column); ok { v = strings.TrimSpace(v) if column == "robots" && v != "" { if normal, err := NormalizeRobots(v); err == nil { v = normal } } values[column] = v } } current := map[string]string{} exists := false if owner.WrittenTo == WrittenToRecord { if locale == "" { locale = owner.Locale } typ, _ := s.Registry().ContentType(owner.Type) e, found, err := LoadEntry(tx, typ.Content.Model, owner.ID, locale) if err != nil { return p, nil, err } exists = true for _, column := range EntryColumns { if found { current[column] = EntryValue(e, column) } else { current[column] = "" } } p.state = []any{owner.Type, owner.ID, locale, current} } else { p.notes = append(p.notes, s.PhraseCtx(ctx, "golem15.seo::lang.transfer.row_rule", nil)) if owner.RuleID != 0 { r, err := LoadRule(tx, owner.RuleID) if err != nil { return p, nil, err } exists = true for _, column := range EntryColumns { current[column] = r.Value(column) } p.state = RuleValues(r) } if values["keyphrase"] == "" { delete(values, "keyphrase") } } changed := map[string]string{} for column, v := range values { if current[column] != v { changed[column] = v p.diffs = append(p.diffs, [3]string{column, current[column], v}) } } sortDiffs(p.diffs, DatasetColumns[DatasetRecords]) switch { case len(changed) == 0: p.status = "unchanged" return p, nil, nil case exists: p.status = "changed" default: p.status = "new" } return p, &BatchChange{Op: OpSetSEO, URL: url, Locale: locale, Values: changed}, nil } // previewRows fills the summary, the preview rows and the count from the // plan and the batch it ran. func (s *Service) previewRows(ctx context.Context, res *ImportResult, plan []planned, batch BatchResult) { var refused, others []PreviewRow for _, p := range plan { status, notes := p.status, append([]string{}, p.notes...) warning := false if p.change >= 0 && p.change < len(batch.Changes) { item := batch.Changes[p.change] if item.Error != "" { status = "refused" switch { case item.Error == ConflictLoop: notes = append(notes, s.PhraseCtx(ctx, "golem15.seo::lang.transfer.row_loop", map[string]string{"path": strings.Join(item.Path, " → ")})) case len(item.Errors) > 0: fields := make([]string, 0, len(item.Errors)) for f := range item.Errors { fields = append(fields, f) } sort.Strings(fields) for _, f := range fields { notes = append(notes, s.PhraseCtx(ctx, "golem15.seo::lang.transfer.row_invalid", map[string]string{"field": f, "message": strings.Join(item.Errors[f], " ")})) } default: notes = append(notes, item.Message) } } for _, c := range item.Conflicts { switch c.Kind { case ConflictLoop: case ConflictChain: warning = true notes = append(notes, s.PhraseCtx(ctx, "golem15.seo::lang.transfer.row_chain", map[string]string{"path": strings.Join(c.Path, " → ")})) default: warning = true notes = append(notes, c.Message) } } } switch status { case "new": res.Summary.New++ case "changed": res.Summary.Changed++ case "unchanged": res.Summary.Unchanged++ case "refused": res.Summary.Refused++ } if warning && status != "refused" { res.Summary.Warning++ } if status == "new" || status == "changed" { res.Count++ } note := strings.Join(notes, " ") base := PreviewRow{Line: p.row.Line, Change: status, Address: p.address, Note: note, Warning: warning && status != "refused"} var rows []PreviewRow if status == "changed" || status == "new" { for _, d := range p.diffs { r := base r.Field, r.Before, r.After = d[0], d[1], d[2] rows = append(rows, r) } } if len(rows) == 0 { rows = []PreviewRow{base} } if status == "refused" { refused = append(refused, rows...) } else { others = append(others, rows...) } } res.Rows = append(refused, others...) } // The transfer token (T-32-25): the preview's dataset, delimiter, file and // fingerprint, gzip-compressed and signed with HMAC-SHA256 under the app // key, so the apply needs no server-side state and a forged or edited // token is refused. type transferToken struct { Dataset string `json:"d"` Delimiter string `json:"l"` CSV string `json:"c"` Fingerprint string `json:"f"` } var ( fallbackKeyOnce sync.Once fallbackKey []byte ) // tokenKey is the HMAC key of transfer tokens: derived from the app key, // or a random key per process when the app has none (tokens then do not // survive a restart). func (s *Service) tokenKey() []byte { if cfg := s.Config(); cfg != nil { if primary, _, err := lagoon.LoadAppKey(cfg); err == nil && len(primary) > 0 { mac := hmac.New(sha256.New, primary) mac.Write([]byte("golem15.seo.transfer")) return mac.Sum(nil) } } fallbackKeyOnce.Do(func() { fallbackKey = make([]byte, 32) if _, err := rand.Read(fallbackKey); err != nil { panic(err) } }) return fallbackKey } // TransferToken signs a preview for its apply. func (s *Service) TransferToken(dataset, delimiter, csv, fp string) (string, error) { raw, err := json.Marshal(transferToken{Dataset: dataset, Delimiter: delimiter, CSV: csv, Fingerprint: fp}) if err != nil { return "", err } var b bytes.Buffer zw := gzip.NewWriter(&b) if _, err := zw.Write(raw); err != nil { return "", err } if err := zw.Close(); err != nil { return "", err } mac := hmac.New(sha256.New, s.tokenKey()) mac.Write(b.Bytes()) return base64.RawURLEncoding.EncodeToString(b.Bytes()) + "." + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)), nil } // ErrBadToken refuses a transfer token that is malformed or not signed by // this application. var ErrBadToken = errors.New("golem15.seo: the import token is not valid") // ReadTransferToken verifies a token and returns the import it applies: // the preview's file with the fingerprint the apply must find. func (s *Service) ReadTransferToken(token string) (ImportRequest, error) { body, sig, ok := strings.Cut(strings.TrimSpace(token), ".") if !ok { return ImportRequest{}, ErrBadToken } payload, err := base64.RawURLEncoding.DecodeString(body) if err != nil { return ImportRequest{}, ErrBadToken } got, err := base64.RawURLEncoding.DecodeString(sig) if err != nil { return ImportRequest{}, ErrBadToken } mac := hmac.New(sha256.New, s.tokenKey()) mac.Write(payload) if !hmac.Equal(got, mac.Sum(nil)) { return ImportRequest{}, ErrBadToken } zr, err := gzip.NewReader(bytes.NewReader(payload)) if err != nil { return ImportRequest{}, ErrBadToken } raw, err := io.ReadAll(io.LimitReader(zr, 4<<20)) if err != nil { return ImportRequest{}, ErrBadToken } var t transferToken if err := json.Unmarshal(raw, &t); err != nil { return ImportRequest{}, ErrBadToken } return ImportRequest{Dataset: t.Dataset, Delimiter: t.Delimiter, CSV: t.CSV, Fingerprint: t.Fingerprint}, nil } // ChoiceCtx translates a plural phrase for count in the request locale, // with :count set. func (s *Service) ChoiceCtx(ctx context.Context, key string, count int, params map[string]string) string { tr := s.translator() if tr == nil { return key } p := map[string]string{"count": strconv.Itoa(count)} for k, v := range params { p[k] = v } return tr.Choice(ctx, key, count, p) }