package classes import ( "context" "crypto/rand" "encoding/hex" "errors" "fmt" "strings" "git.golem15.com/golem15/sm-seo-plugin/models" "git.golem15.com/golem15/summercms/modules/lagoon" "gorm.io/gorm" ) // MaxBatchChanges caps the changes of one batch. const MaxBatchChanges = 2000 // The operations of a batch change. const ( OpSetSEO = "set_seo" OpCreateRedirect = "create_redirect" OpUpdateRedirect = "update_redirect" OpDeleteRedirect = "delete_redirect" OpCreateRule = "create_rule" OpUpdateRule = "update_rule" OpDeleteRule = "delete_rule" ) // The conflict kinds a batch reports (D-12). const ( ConflictLoop = "redirect_loop" ConflictChain = "redirect_chain" ConflictNoindexPublished = "noindex_published" ConflictCanonicalRedirected = "canonical_redirected" ConflictCanonicalNoindex = "canonical_noindex" ) // RedirectFields are the fields of a redirect write; a nil field is left // as it is (or takes the default on a create). type RedirectFields struct { Source *string `json:"source,omitempty"` Target *string `json:"target,omitempty"` Status *int `json:"status,omitempty"` Match *string `json:"match,omitempty"` Query *string `json:"query,omitempty"` IgnoreCase *bool `json:"ignore_case,omitempty"` TrailingSlash *bool `json:"trailing_slash,omitempty"` Note *string `json:"note,omitempty"` } // Apply writes the fields that are set onto r. func (f RedirectFields) Apply(r *models.Redirect) { if f.Source != nil { r.Source = strings.TrimSpace(*f.Source) } if f.Target != nil { r.Target = strings.TrimSpace(*f.Target) } if f.Status != nil { r.Status = models.RedirectStatus(*f.Status) } if f.Match != nil { r.Match = strings.TrimSpace(*f.Match) } if f.Query != nil { r.Query = strings.TrimSpace(*f.Query) } if f.IgnoreCase != nil { r.IgnoreCase = *f.IgnoreCase } if f.TrailingSlash != nil { r.TrailingSlash = *f.TrailingSlash } if f.Note != nil { r.Note = strings.TrimSpace(*f.Note) } } // RuleFields are the fields of a URL rule write; a nil field is left as // it is. type RuleFields struct { URL *string `json:"url,omitempty"` Match *string `json:"match,omitempty"` Title *string `json:"title,omitempty"` Description *string `json:"description,omitempty"` Canonical *string `json:"canonical,omitempty"` Robots *string `json:"robots,omitempty"` OGTitle *string `json:"og_title,omitempty"` OGDescription *string `json:"og_description,omitempty"` OGImage *string `json:"og_image,omitempty"` Note *string `json:"note,omitempty"` } // Apply writes the fields that are set onto r. func (f RuleFields) Apply(r *models.Rule) { for column, v := range map[string]*string{ "url": f.URL, "match": f.Match, "title": f.Title, "description": f.Description, "canonical": f.Canonical, "robots": f.Robots, "og_title": f.OGTitle, "og_description": f.OGDescription, "og_image": f.OGImage, "note": f.Note, } { if v != nil { SetRuleValue(r, column, strings.TrimSpace(*v)) } } } // BatchChange is one change of a batch (D-12): set_seo addresses a page // by URL (D-11); the redirect and rule operations create, update (by id) // or delete (by id). type BatchChange struct { Op string `json:"op"` ID uint `json:"id,omitempty"` URL string `json:"url,omitempty"` Locale string `json:"locale,omitempty"` Values map[string]string `json:"values,omitempty"` ForceRule bool `json:"force_rule,omitempty"` Redirect *RedirectFields `json:"redirect,omitempty"` Rule *RuleFields `json:"rule,omitempty"` } // DiffEntry is one changed object of a batch change: the history row the // change wrote, its before and after values. type DiffEntry struct { Kind string `json:"kind"` Subject string `json:"subject"` What string `json:"what"` Before map[string]any `json:"before"` After map[string]any `json:"after"` } // Conflict is a finding of a batch (D-12): a loop refuses the batch, the // other kinds are warnings. type Conflict struct { Change int `json:"change"` Kind string `json:"kind"` URL string `json:"url,omitempty"` Path []string `json:"path,omitempty"` Target string `json:"target,omitempty"` Message string `json:"message"` Refused bool `json:"refused,omitempty"` } // BatchItem is the outcome of one change: what it wrote (the diff), where // a URL change went, its conflicts, and the refusal when it was refused. type BatchItem struct { Index int `json:"index"` Op string `json:"op"` Owner *Owner `json:"owner,omitempty"` Diff []DiffEntry `json:"diff"` Conflicts []Conflict `json:"conflicts,omitempty"` Error string `json:"error,omitempty"` Message string `json:"message,omitempty"` Errors map[string][]string `json:"errors,omitempty"` Path []string `json:"path,omitempty"` } // BatchResult is a batch's answer: the batch id its history rows carry, // whether it was a dry run and whether it was applied, every change's // outcome, all conflicts, and how many changes wrote something. type BatchResult struct { BatchID string `json:"batch_id"` DryRun bool `json:"dry_run"` Applied bool `json:"applied"` Changes []BatchItem `json:"changes"` Conflicts []Conflict `json:"conflicts"` Count int `json:"count"` } // Refused returns the first refused change, nil when none was. func (r BatchResult) Refused() *BatchItem { for i := range r.Changes { if r.Changes[i].Error != "" { return &r.Changes[i] } } return nil } // BatchRefusedError refuses a batch with a refused change: nothing was // written (D-12, all-or-nothing). type BatchRefusedError struct{ Result BatchResult } func (e *BatchRefusedError) Error() string { if item := e.Result.Refused(); item != nil { return fmt.Sprintf("golem15.seo: batch change %d refused: %s", item.Index, item.Error) } return "golem15.seo: batch refused" } // ErrStale refuses an apply whose data changed after its preview. var ErrStale = errors.New("golem15.seo: the data changed after the preview") // errDryRun rolls a dry run back on purpose. var errDryRun = errors.New("golem15.seo: dry run") // BatchOptions are a batch's mode: DryRun rolls everything back after the // diff and conflicts are collected; Via overrides the writer history // records (import); BatchID names the batch (a new id when empty). type BatchOptions struct { DryRun bool Via string BatchID string } // NewBatchID returns a random batch id. func NewBatchID() string { var b [16]byte if _, err := rand.Read(b[:]); err != nil { panic(err) } return hex.EncodeToString(b[:]) } // RunBatch runs changes as one batch (D-12, RESEARCH Pattern 8): every // change goes through the service inside one transaction, each change's // diff (the history rows it wrote) and conflicts are collected, and the // conflicts that depend on the whole batch (a canonical pointing to a // redirect another change creates) are checked once every change is in // place. A dry run then rolls back on purpose; a refused change (a loop, // an invalid value) rolls everything back and answers a // *BatchRefusedError; otherwise the batch commits with one batch id on // every history row. func (s *Service) RunBatch(ctx context.Context, changes []BatchChange, opts BatchOptions) (BatchResult, error) { return s.runBatch(ctx, opts, func(context.Context, *gorm.DB) ([]BatchChange, error) { return changes, nil }) } // runBatch is RunBatch with the changes built inside the transaction // (the import maps its rows against the data it applies to). func (s *Service) runBatch(ctx context.Context, opts BatchOptions, build func(ctx context.Context, tx *gorm.DB) ([]BatchChange, error)) (BatchResult, error) { gdb, err := s.GormDB() if err != nil { return BatchResult{}, err } res := BatchResult{BatchID: opts.BatchID, DryRun: opts.DryRun, Changes: []BatchItem{}, Conflicts: []Conflict{}} if res.BatchID == "" { res.BatchID = NewBatchID() } err = lagoon.Transaction(ctx, gdb, func(ctx context.Context, tx *gorm.DB) error { ctx = WithTx(WithBatch(ctx, res.BatchID), tx) if opts.Via != "" { ctx = WithVia(ctx, opts.Via) } // One lock for the whole batch, so a refused change rolled back to // its savepoint does not let another redirect writer in (WR-10). if err := lockRedirectWrites(tx); err != nil { return err } changes, err := build(ctx, tx) if err != nil { return err } if len(changes) > MaxBatchChanges { return &ProblemsError{Problems: map[string][]models.Problem{"changes": {{Key: "golem15.seo::lang.api.batch_too_large", Params: map[string]string{"max": fmt.Sprint(MaxBatchChanges)}}}}} } for i, ch := range changes { item, err := s.batchChange(ctx, tx, i, ch, res.BatchID) if err != nil { return err } res.Changes = append(res.Changes, item) } for i := range res.Changes { late, err := s.batchConflicts(ctx, tx, i, changes[i], res.Changes[i]) if err != nil { return err } res.Changes[i].Conflicts = append(res.Changes[i].Conflicts, late...) } for _, item := range res.Changes { res.Conflicts = append(res.Conflicts, item.Conflicts...) if len(item.Diff) > 0 { res.Count++ } } if res.Refused() != nil { return &BatchRefusedError{Result: res} } if opts.DryRun { return errDryRun } return nil }) switch { case errors.Is(err, errDryRun): return res, nil case err != nil: var refused *BatchRefusedError if errors.As(err, &refused) { refused.Result.Applied = false } return res, err } res.Applied = true return res, nil } // batchChange runs one change under a savepoint, so a refused change // leaves the transaction usable for the next one, and reads the history // rows it wrote as its diff. func (s *Service) batchChange(ctx context.Context, tx *gorm.DB, index int, ch BatchChange, batchID string) (BatchItem, error) { item := BatchItem{Index: index, Op: ch.Op, Diff: []DiffEntry{}} var last uint if err := tx.Raw(`SELECT COALESCE(MAX(id), 0) FROM golem15_seo_revisions`).Scan(&last).Error; err != nil { return item, err } point := fmt.Sprintf("seo_change_%d", index) if err := tx.SavePoint(point).Error; err != nil { return item, err } err := s.applyChange(ctx, tx, index, ch, &item) if err != nil { if rb := tx.RollbackTo(point).Error; rb != nil { return item, rb } var loop *LoopError var problems *ProblemsError switch { case errors.As(err, &loop): item.Error, item.Path = ConflictLoop, loop.Check.Path item.Message = s.LoopMessage(ctx, loop.Check) item.Conflicts = append(item.Conflicts, Conflict{Change: index, Kind: ConflictLoop, Path: loop.Check.Path, Message: item.Message, Refused: true}) case errors.As(err, &problems): item.Error = "validation_failed" item.Errors = map[string][]string{} var msgs []string for field, list := range problems.Problems { for _, p := range list { msg := s.PhraseCtx(ctx, p.Key, p.Params) item.Errors[field] = append(item.Errors[field], msg) msgs = append(msgs, field+": "+msg) } } item.Message = strings.Join(msgs, "; ") case errors.Is(err, ErrNoRecord): item.Error, item.Message = "not_found", s.PhraseCtx(ctx, "golem15.seo::lang.api.change_not_found", nil) case errors.Is(err, ErrBadURL): item.Error, item.Message = "bad_url", err.Error() case errors.Is(err, errUnknownOp): item.Error, item.Message = "unknown_op", s.PhraseCtx(ctx, "golem15.seo::lang.api.op_unknown", map[string]string{"op": ch.Op}) default: return item, err } return item, nil } if err := tx.Exec("RELEASE SAVEPOINT " + point).Error; err != nil { return item, err } var rows []models.Revision if err := tx.Where("id > ? AND batch_id = ?", last, batchID).Order("id ASC").Find(&rows).Error; err != nil { return item, err } for _, r := range rows { item.Diff = append(item.Diff, DiffEntry{Kind: r.SubjectKind, Subject: r.SubjectID, What: r.What, Before: r.Before, After: r.After}) } return item, nil } // errUnknownOp refuses a change with an unknown operation. var errUnknownOp = errors.New("golem15.seo: unknown batch operation") // applyChange runs one change through the service. func (s *Service) applyChange(ctx context.Context, tx *gorm.DB, index int, ch BatchChange, item *BatchItem) error { switch ch.Op { case OpSetSEO: res, err := s.SetURLSEO(ctx, tx, URLSEOWrite{URL: ch.URL, Locale: ch.Locale, Values: ch.Values, ForceRule: ch.ForceRule}) if err != nil { return err } owner := res.Owner item.Owner = &owner return nil case OpCreateRedirect, OpUpdateRedirect: r := models.NewRedirect() r.Origin = RedirectOrigin(ctx) if ch.Op == OpUpdateRedirect { var err error if r, err = LoadRedirect(tx, ch.ID); err != nil { return err } } if ch.Redirect != nil { ch.Redirect.Apply(&r) } check, err := s.SaveRedirect(ctx, tx, &r) if err != nil { return err } if check.Chain { notice := ChainNotice(check)[0] item.Conflicts = append(item.Conflicts, Conflict{Change: index, Kind: ConflictChain, URL: r.Source, Path: check.Path, Target: check.Final, Message: s.PhraseCtx(ctx, notice.Message, notice.Params)}) } return nil case OpDeleteRedirect: r, err := LoadRedirect(tx, ch.ID) if err != nil { return err } return s.DeleteRedirect(ctx, tx, r) case OpCreateRule, OpUpdateRule: r := models.Rule{Match: models.MatchExact} if ch.Op == OpUpdateRule { var err error if r, err = LoadRule(tx, ch.ID); err != nil { return err } } if ch.Rule != nil { ch.Rule.Apply(&r) } return s.SaveRule(ctx, tx, &r) case OpDeleteRule: r, err := LoadRule(tx, ch.ID) if err != nil { return err } return s.DeleteRule(ctx, tx, r) } return errUnknownOp } // batchConflicts are the warnings that depend on the whole batch, checked // once every change is in place: noindex on the address of a published // record (the records the sitemaps list; Phase 33's sitemap filter // refines it) and a canonical pointing to a redirected or noindex // address. func (s *Service) batchConflicts(ctx context.Context, tx *gorm.DB, index int, ch BatchChange, item BatchItem) ([]Conflict, error) { if item.Error != "" { return nil, nil } var out []Conflict var robots, canonical, url string switch ch.Op { case OpSetSEO: robots, canonical, url = ch.Values["robots"], ch.Values["canonical"], ch.URL if item.Owner != nil { url = item.Owner.URL } case OpCreateRule, OpUpdateRule: if ch.Rule == nil { return nil, nil } if ch.Rule.Robots != nil { robots = *ch.Rule.Robots } if ch.Rule.Canonical != nil { canonical = *ch.Rule.Canonical } if ch.Rule.URL != nil { url = *ch.Rule.URL } default: return nil, nil } if strings.Contains(strings.ToLower(robots), "noindex") && url != "" && !strings.Contains(url, "*") { owner, _, err := s.FindOwner(ctx, tx, url, false) if err != nil && !errors.Is(err, ErrBadURL) { return nil, err } if err == nil && owner.WrittenTo == WrittenToRecord && owner.Published { out = append(out, Conflict{Change: index, Kind: ConflictNoindexPublished, URL: owner.URL, Message: s.PhraseCtx(ctx, "golem15.seo::lang.api.conflict_noindex_published", map[string]string{"url": owner.URL})}) } } if strings.TrimSpace(canonical) != "" { kind, target, err := s.canonicalProblem(ctx, canonical) if err != nil { return nil, err } switch kind { case IssueCanonicalRedirected: out = append(out, Conflict{Change: index, Kind: ConflictCanonicalRedirected, URL: url, Target: target, Message: s.PhraseCtx(ctx, "golem15.seo::lang.transfer.row_canonical", map[string]string{"target": target})}) case IssueCanonicalNoindex: out = append(out, Conflict{Change: index, Kind: ConflictCanonicalNoindex, URL: url, Message: s.PhraseCtx(ctx, "golem15.seo::lang.status.canonical_noindex", map[string]string{"url": canonical})}) } } return out, nil }