feat: batches with dry run, undo, and CSV import and export
- POST /_seo/api/v1/batch runs changes in one transaction: dry_run answers the diff and the conflicts (loops, chains, noindex on a published record, canonicals to redirected or noindex addresses) and rolls back; a refused change refuses the batch (D-12) - undo of a change or a whole batch from History and through MCP, with the undo_stale, undo_loop and undo_gone refusals (D-13) - CSV export and import of redirects, rules and records by URL: BOM, comma or semicolon, formula cells escaped, a dry-run preview, all or nothing with via import; the screen's apply token is HMAC-SHA256 over the file and a fingerprint of the data (transfer.stale) - Import and export singleton with the seo_transfer widget action, History record actions undo and undo_batch, navigation entry - MCP tools undo_change, undo_batch, apply_batch, export_csv, import_csv - TestBatch, TestUndo, TestCSVTransfer, TestLoopRefused mcp and import
This commit is contained in:
475
classes/batch.go
Normal file
475
classes/batch.go
Normal file
@@ -0,0 +1,475 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// MaxBatchChanges caps the changes of one batch.
|
||||
const MaxBatchChanges = 2000
|
||||
|
||||
// The operations of a batch change.
|
||||
const (
|
||||
OpSetSEO = "set_seo"
|
||||
OpCreateRedirect = "create_redirect"
|
||||
OpUpdateRedirect = "update_redirect"
|
||||
OpDeleteRedirect = "delete_redirect"
|
||||
OpCreateRule = "create_rule"
|
||||
OpUpdateRule = "update_rule"
|
||||
OpDeleteRule = "delete_rule"
|
||||
)
|
||||
|
||||
// The conflict kinds a batch reports (D-12).
|
||||
const (
|
||||
ConflictLoop = "redirect_loop"
|
||||
ConflictChain = "redirect_chain"
|
||||
ConflictNoindexPublished = "noindex_published"
|
||||
ConflictCanonicalRedirected = "canonical_redirected"
|
||||
ConflictCanonicalNoindex = "canonical_noindex"
|
||||
)
|
||||
|
||||
// RedirectFields are the fields of a redirect write; a nil field is left
|
||||
// as it is (or takes the default on a create).
|
||||
type RedirectFields struct {
|
||||
Source *string `json:"source,omitempty"`
|
||||
Target *string `json:"target,omitempty"`
|
||||
Status *int `json:"status,omitempty"`
|
||||
Match *string `json:"match,omitempty"`
|
||||
Query *string `json:"query,omitempty"`
|
||||
IgnoreCase *bool `json:"ignore_case,omitempty"`
|
||||
TrailingSlash *bool `json:"trailing_slash,omitempty"`
|
||||
Note *string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// Apply writes the fields that are set onto r.
|
||||
func (f RedirectFields) Apply(r *models.Redirect) {
|
||||
if f.Source != nil {
|
||||
r.Source = strings.TrimSpace(*f.Source)
|
||||
}
|
||||
if f.Target != nil {
|
||||
r.Target = strings.TrimSpace(*f.Target)
|
||||
}
|
||||
if f.Status != nil {
|
||||
r.Status = models.RedirectStatus(*f.Status)
|
||||
}
|
||||
if f.Match != nil {
|
||||
r.Match = strings.TrimSpace(*f.Match)
|
||||
}
|
||||
if f.Query != nil {
|
||||
r.Query = strings.TrimSpace(*f.Query)
|
||||
}
|
||||
if f.IgnoreCase != nil {
|
||||
r.IgnoreCase = *f.IgnoreCase
|
||||
}
|
||||
if f.TrailingSlash != nil {
|
||||
r.TrailingSlash = *f.TrailingSlash
|
||||
}
|
||||
if f.Note != nil {
|
||||
r.Note = strings.TrimSpace(*f.Note)
|
||||
}
|
||||
}
|
||||
|
||||
// RuleFields are the fields of a URL rule write; a nil field is left as
|
||||
// it is.
|
||||
type RuleFields struct {
|
||||
URL *string `json:"url,omitempty"`
|
||||
Match *string `json:"match,omitempty"`
|
||||
Title *string `json:"title,omitempty"`
|
||||
Description *string `json:"description,omitempty"`
|
||||
Canonical *string `json:"canonical,omitempty"`
|
||||
Robots *string `json:"robots,omitempty"`
|
||||
OGTitle *string `json:"og_title,omitempty"`
|
||||
OGDescription *string `json:"og_description,omitempty"`
|
||||
OGImage *string `json:"og_image,omitempty"`
|
||||
Note *string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// Apply writes the fields that are set onto r.
|
||||
func (f RuleFields) Apply(r *models.Rule) {
|
||||
for column, v := range map[string]*string{
|
||||
"url": f.URL, "match": f.Match, "title": f.Title, "description": f.Description,
|
||||
"canonical": f.Canonical, "robots": f.Robots, "og_title": f.OGTitle,
|
||||
"og_description": f.OGDescription, "og_image": f.OGImage, "note": f.Note,
|
||||
} {
|
||||
if v != nil {
|
||||
SetRuleValue(r, column, strings.TrimSpace(*v))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BatchChange is one change of a batch (D-12): set_seo addresses a page
|
||||
// by URL (D-11); the redirect and rule operations create, update (by id)
|
||||
// or delete (by id).
|
||||
type BatchChange struct {
|
||||
Op string `json:"op"`
|
||||
ID uint `json:"id,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Locale string `json:"locale,omitempty"`
|
||||
Values map[string]string `json:"values,omitempty"`
|
||||
ForceRule bool `json:"force_rule,omitempty"`
|
||||
Redirect *RedirectFields `json:"redirect,omitempty"`
|
||||
Rule *RuleFields `json:"rule,omitempty"`
|
||||
}
|
||||
|
||||
// DiffEntry is one changed object of a batch change: the history row the
|
||||
// change wrote, its before and after values.
|
||||
type DiffEntry struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
What string `json:"what"`
|
||||
Before map[string]any `json:"before"`
|
||||
After map[string]any `json:"after"`
|
||||
}
|
||||
|
||||
// Conflict is a finding of a batch (D-12): a loop refuses the batch, the
|
||||
// other kinds are warnings.
|
||||
type Conflict struct {
|
||||
Change int `json:"change"`
|
||||
Kind string `json:"kind"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Path []string `json:"path,omitempty"`
|
||||
Target string `json:"target,omitempty"`
|
||||
Message string `json:"message"`
|
||||
Refused bool `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
// BatchItem is the outcome of one change: what it wrote (the diff), where
|
||||
// a URL change went, its conflicts, and the refusal when it was refused.
|
||||
type BatchItem struct {
|
||||
Index int `json:"index"`
|
||||
Op string `json:"op"`
|
||||
Owner *Owner `json:"owner,omitempty"`
|
||||
Diff []DiffEntry `json:"diff"`
|
||||
Conflicts []Conflict `json:"conflicts,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Errors map[string][]string `json:"errors,omitempty"`
|
||||
Path []string `json:"path,omitempty"`
|
||||
}
|
||||
|
||||
// BatchResult is a batch's answer: the batch id its history rows carry,
|
||||
// whether it was a dry run and whether it was applied, every change's
|
||||
// outcome, all conflicts, and how many changes wrote something.
|
||||
type BatchResult struct {
|
||||
BatchID string `json:"batch_id"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
Applied bool `json:"applied"`
|
||||
Changes []BatchItem `json:"changes"`
|
||||
Conflicts []Conflict `json:"conflicts"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// Refused returns the first refused change, nil when none was.
|
||||
func (r BatchResult) Refused() *BatchItem {
|
||||
for i := range r.Changes {
|
||||
if r.Changes[i].Error != "" {
|
||||
return &r.Changes[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchRefusedError refuses a batch with a refused change: nothing was
|
||||
// written (D-12, all-or-nothing).
|
||||
type BatchRefusedError struct{ Result BatchResult }
|
||||
|
||||
func (e *BatchRefusedError) Error() string {
|
||||
if item := e.Result.Refused(); item != nil {
|
||||
return fmt.Sprintf("golem15.seo: batch change %d refused: %s", item.Index, item.Error)
|
||||
}
|
||||
return "golem15.seo: batch refused"
|
||||
}
|
||||
|
||||
// ErrStale refuses an apply whose data changed after its preview.
|
||||
var ErrStale = errors.New("golem15.seo: the data changed after the preview")
|
||||
|
||||
// errDryRun rolls a dry run back on purpose.
|
||||
var errDryRun = errors.New("golem15.seo: dry run")
|
||||
|
||||
// BatchOptions are a batch's mode: DryRun rolls everything back after the
|
||||
// diff and conflicts are collected; Via overrides the writer history
|
||||
// records (import); BatchID names the batch (a new id when empty).
|
||||
type BatchOptions struct {
|
||||
DryRun bool
|
||||
Via string
|
||||
BatchID string
|
||||
}
|
||||
|
||||
// NewBatchID returns a random batch id.
|
||||
func NewBatchID() string {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// RunBatch runs changes as one batch (D-12, RESEARCH Pattern 8): every
|
||||
// change goes through the service inside one transaction, each change's
|
||||
// diff (the history rows it wrote) and conflicts are collected, and the
|
||||
// conflicts that depend on the whole batch (a canonical pointing to a
|
||||
// redirect another change creates) are checked once every change is in
|
||||
// place. A dry run then rolls back on purpose; a refused change (a loop,
|
||||
// an invalid value) rolls everything back and answers a
|
||||
// *BatchRefusedError; otherwise the batch commits with one batch id on
|
||||
// every history row.
|
||||
func (s *Service) RunBatch(ctx context.Context, changes []BatchChange, opts BatchOptions) (BatchResult, error) {
|
||||
return s.runBatch(ctx, opts, func(context.Context, *gorm.DB) ([]BatchChange, error) { return changes, nil })
|
||||
}
|
||||
|
||||
// runBatch is RunBatch with the changes built inside the transaction
|
||||
// (the import maps its rows against the data it applies to).
|
||||
func (s *Service) runBatch(ctx context.Context, opts BatchOptions, build func(ctx context.Context, tx *gorm.DB) ([]BatchChange, error)) (BatchResult, error) {
|
||||
gdb, err := s.GormDB()
|
||||
if err != nil {
|
||||
return BatchResult{}, err
|
||||
}
|
||||
res := BatchResult{BatchID: opts.BatchID, DryRun: opts.DryRun, Changes: []BatchItem{}, Conflicts: []Conflict{}}
|
||||
if res.BatchID == "" {
|
||||
res.BatchID = NewBatchID()
|
||||
}
|
||||
err = lagoon.Transaction(ctx, gdb, func(ctx context.Context, tx *gorm.DB) error {
|
||||
ctx = WithTx(WithBatch(ctx, res.BatchID), tx)
|
||||
if opts.Via != "" {
|
||||
ctx = WithVia(ctx, opts.Via)
|
||||
}
|
||||
changes, err := build(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(changes) > MaxBatchChanges {
|
||||
return &ProblemsError{Problems: map[string][]models.Problem{"changes": {{Key: "golem15.seo::lang.api.batch_too_large", Params: map[string]string{"max": fmt.Sprint(MaxBatchChanges)}}}}}
|
||||
}
|
||||
for i, ch := range changes {
|
||||
item, err := s.batchChange(ctx, tx, i, ch, res.BatchID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res.Changes = append(res.Changes, item)
|
||||
}
|
||||
for i := range res.Changes {
|
||||
late, err := s.batchConflicts(ctx, tx, i, changes[i], res.Changes[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res.Changes[i].Conflicts = append(res.Changes[i].Conflicts, late...)
|
||||
}
|
||||
for _, item := range res.Changes {
|
||||
res.Conflicts = append(res.Conflicts, item.Conflicts...)
|
||||
if len(item.Diff) > 0 {
|
||||
res.Count++
|
||||
}
|
||||
}
|
||||
if res.Refused() != nil {
|
||||
return &BatchRefusedError{Result: res}
|
||||
}
|
||||
if opts.DryRun {
|
||||
return errDryRun
|
||||
}
|
||||
return nil
|
||||
})
|
||||
switch {
|
||||
case errors.Is(err, errDryRun):
|
||||
return res, nil
|
||||
case err != nil:
|
||||
var refused *BatchRefusedError
|
||||
if errors.As(err, &refused) {
|
||||
refused.Result.Applied = false
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
res.Applied = true
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// batchChange runs one change under a savepoint, so a refused change
|
||||
// leaves the transaction usable for the next one, and reads the history
|
||||
// rows it wrote as its diff.
|
||||
func (s *Service) batchChange(ctx context.Context, tx *gorm.DB, index int, ch BatchChange, batchID string) (BatchItem, error) {
|
||||
item := BatchItem{Index: index, Op: ch.Op, Diff: []DiffEntry{}}
|
||||
var last uint
|
||||
if err := tx.Raw(`SELECT COALESCE(MAX(id), 0) FROM golem15_seo_revisions`).Scan(&last).Error; err != nil {
|
||||
return item, err
|
||||
}
|
||||
point := fmt.Sprintf("seo_change_%d", index)
|
||||
if err := tx.SavePoint(point).Error; err != nil {
|
||||
return item, err
|
||||
}
|
||||
err := s.applyChange(ctx, tx, index, ch, &item)
|
||||
if err != nil {
|
||||
if rb := tx.RollbackTo(point).Error; rb != nil {
|
||||
return item, rb
|
||||
}
|
||||
var loop *LoopError
|
||||
var problems *ProblemsError
|
||||
switch {
|
||||
case errors.As(err, &loop):
|
||||
item.Error, item.Path = ConflictLoop, loop.Check.Path
|
||||
item.Message = s.LoopMessage(ctx, loop.Check)
|
||||
item.Conflicts = append(item.Conflicts, Conflict{Change: index, Kind: ConflictLoop, Path: loop.Check.Path, Message: item.Message, Refused: true})
|
||||
case errors.As(err, &problems):
|
||||
item.Error = "validation_failed"
|
||||
item.Errors = map[string][]string{}
|
||||
var msgs []string
|
||||
for field, list := range problems.Problems {
|
||||
for _, p := range list {
|
||||
msg := s.PhraseCtx(ctx, p.Key, p.Params)
|
||||
item.Errors[field] = append(item.Errors[field], msg)
|
||||
msgs = append(msgs, field+": "+msg)
|
||||
}
|
||||
}
|
||||
item.Message = strings.Join(msgs, "; ")
|
||||
case errors.Is(err, ErrNoRecord):
|
||||
item.Error, item.Message = "not_found", s.PhraseCtx(ctx, "golem15.seo::lang.api.change_not_found", nil)
|
||||
case errors.Is(err, ErrBadURL):
|
||||
item.Error, item.Message = "bad_url", err.Error()
|
||||
case errors.Is(err, errUnknownOp):
|
||||
item.Error, item.Message = "unknown_op", s.PhraseCtx(ctx, "golem15.seo::lang.api.op_unknown", map[string]string{"op": ch.Op})
|
||||
default:
|
||||
return item, err
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
if err := tx.Exec("RELEASE SAVEPOINT " + point).Error; err != nil {
|
||||
return item, err
|
||||
}
|
||||
var rows []models.Revision
|
||||
if err := tx.Where("id > ? AND batch_id = ?", last, batchID).Order("id ASC").Find(&rows).Error; err != nil {
|
||||
return item, err
|
||||
}
|
||||
for _, r := range rows {
|
||||
item.Diff = append(item.Diff, DiffEntry{Kind: r.SubjectKind, Subject: r.SubjectID, What: r.What, Before: r.Before, After: r.After})
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// errUnknownOp refuses a change with an unknown operation.
|
||||
var errUnknownOp = errors.New("golem15.seo: unknown batch operation")
|
||||
|
||||
// applyChange runs one change through the service.
|
||||
func (s *Service) applyChange(ctx context.Context, tx *gorm.DB, index int, ch BatchChange, item *BatchItem) error {
|
||||
switch ch.Op {
|
||||
case OpSetSEO:
|
||||
res, err := s.SetURLSEO(ctx, tx, URLSEOWrite{URL: ch.URL, Locale: ch.Locale, Values: ch.Values, ForceRule: ch.ForceRule})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
owner := res.Owner
|
||||
item.Owner = &owner
|
||||
return nil
|
||||
case OpCreateRedirect, OpUpdateRedirect:
|
||||
r := models.NewRedirect()
|
||||
r.Origin = RedirectOrigin(ctx)
|
||||
if ch.Op == OpUpdateRedirect {
|
||||
var err error
|
||||
if r, err = LoadRedirect(tx, ch.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if ch.Redirect != nil {
|
||||
ch.Redirect.Apply(&r)
|
||||
}
|
||||
check, err := s.SaveRedirect(ctx, tx, &r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if check.Chain {
|
||||
notice := ChainNotice(check)[0]
|
||||
item.Conflicts = append(item.Conflicts, Conflict{Change: index, Kind: ConflictChain, URL: r.Source, Path: check.Path, Target: check.Final, Message: s.PhraseCtx(ctx, notice.Message, notice.Params)})
|
||||
}
|
||||
return nil
|
||||
case OpDeleteRedirect:
|
||||
r, err := LoadRedirect(tx, ch.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.DeleteRedirect(ctx, tx, r)
|
||||
case OpCreateRule, OpUpdateRule:
|
||||
r := models.Rule{Match: models.MatchExact}
|
||||
if ch.Op == OpUpdateRule {
|
||||
var err error
|
||||
if r, err = LoadRule(tx, ch.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if ch.Rule != nil {
|
||||
ch.Rule.Apply(&r)
|
||||
}
|
||||
return s.SaveRule(ctx, tx, &r)
|
||||
case OpDeleteRule:
|
||||
r, err := LoadRule(tx, ch.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.DeleteRule(ctx, tx, r)
|
||||
}
|
||||
return errUnknownOp
|
||||
}
|
||||
|
||||
// batchConflicts are the warnings that depend on the whole batch, checked
|
||||
// once every change is in place: noindex on the address of a published
|
||||
// record (the records the sitemaps list; Phase 33's sitemap filter
|
||||
// refines it) and a canonical pointing to a redirected or noindex
|
||||
// address.
|
||||
func (s *Service) batchConflicts(ctx context.Context, tx *gorm.DB, index int, ch BatchChange, item BatchItem) ([]Conflict, error) {
|
||||
if item.Error != "" {
|
||||
return nil, nil
|
||||
}
|
||||
var out []Conflict
|
||||
var robots, canonical, url string
|
||||
switch ch.Op {
|
||||
case OpSetSEO:
|
||||
robots, canonical, url = ch.Values["robots"], ch.Values["canonical"], ch.URL
|
||||
if item.Owner != nil {
|
||||
url = item.Owner.URL
|
||||
}
|
||||
case OpCreateRule, OpUpdateRule:
|
||||
if ch.Rule == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if ch.Rule.Robots != nil {
|
||||
robots = *ch.Rule.Robots
|
||||
}
|
||||
if ch.Rule.Canonical != nil {
|
||||
canonical = *ch.Rule.Canonical
|
||||
}
|
||||
if ch.Rule.URL != nil {
|
||||
url = *ch.Rule.URL
|
||||
}
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
if strings.Contains(strings.ToLower(robots), "noindex") && url != "" && !strings.Contains(url, "*") {
|
||||
owner, _, err := s.FindOwner(ctx, tx, url, false)
|
||||
if err != nil && !errors.Is(err, ErrBadURL) {
|
||||
return nil, err
|
||||
}
|
||||
if err == nil && owner.WrittenTo == WrittenToRecord && owner.Published {
|
||||
out = append(out, Conflict{Change: index, Kind: ConflictNoindexPublished, URL: owner.URL,
|
||||
Message: s.PhraseCtx(ctx, "golem15.seo::lang.api.conflict_noindex_published", map[string]string{"url": owner.URL})})
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(canonical) != "" {
|
||||
kind, target, err := s.canonicalProblem(ctx, canonical)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch kind {
|
||||
case IssueCanonicalRedirected:
|
||||
out = append(out, Conflict{Change: index, Kind: ConflictCanonicalRedirected, URL: url, Target: target,
|
||||
Message: s.PhraseCtx(ctx, "golem15.seo::lang.transfer.row_canonical", map[string]string{"target": target})})
|
||||
case IssueCanonicalNoindex:
|
||||
out = append(out, Conflict{Change: index, Kind: ConflictCanonicalNoindex, URL: url,
|
||||
Message: s.PhraseCtx(ctx, "golem15.seo::lang.status.canonical_noindex", map[string]string{"url": canonical})})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user