sm-seo-plugin/classes/batch.go

package classes

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"errors"
	"fmt"
	"strings"

	"git.golem15.com/golem15/sm-seo-plugin/models"
	"git.golem15.com/golem15/summercms/modules/lagoon"
	"gorm.io/gorm"
)

// MaxBatchChanges caps the changes of one batch.
const MaxBatchChanges = 2000

// The operations of a batch change.
const (
	OpSetSEO         = "set_seo"
	OpCreateRedirect = "create_redirect"
	OpUpdateRedirect = "update_redirect"
	OpDeleteRedirect = "delete_redirect"
	OpCreateRule     = "create_rule"
	OpUpdateRule     = "update_rule"
	OpDeleteRule     = "delete_rule"
)

// The conflict kinds a batch reports (D-12).
const (
	ConflictLoop                = "redirect_loop"
	ConflictChain               = "redirect_chain"
	ConflictNoindexPublished    = "noindex_published"
	ConflictCanonicalRedirected = "canonical_redirected"
	ConflictCanonicalNoindex    = "canonical_noindex"
)

// RedirectFields are the fields of a redirect write; a nil field is left
// as it is (or takes the default on a create).
type RedirectFields struct {
	Source        *string `json:"source,omitempty"`
	Target        *string `json:"target,omitempty"`
	Status        *int    `json:"status,omitempty"`
	Match         *string `json:"match,omitempty"`
	Query         *string `json:"query,omitempty"`
	IgnoreCase    *bool   `json:"ignore_case,omitempty"`
	TrailingSlash *bool   `json:"trailing_slash,omitempty"`
	Note          *string `json:"note,omitempty"`
}

// Problems refuses a status, match or query option outside the allowed
// values (WR-17), so a caller learns its request was not taken rather than
// getting a default in its place; an empty match or query is the default.
func (f RedirectFields) Problems() map[string][]models.Problem {
	out := map[string][]models.Problem{}
	invalid := []models.Problem{{Key: "golem15.seo::lang.api.value_invalid"}}
	if f.Status != nil {
		switch models.RedirectStatus(*f.Status) {
		case models.Status301, models.Status302, models.Status410:
		default:
			out["status"] = invalid
		}
	}
	if f.Match != nil {
		switch strings.TrimSpace(*f.Match) {
		case "", models.RedirectExact, models.RedirectPattern, models.RedirectRegex:
		default:
			out["match"] = invalid
		}
	}
	if f.Query != nil {
		switch strings.TrimSpace(*f.Query) {
		case "", models.QueryIgnore, models.QueryPass:
		default:
			out["query"] = invalid
		}
	}
	return out
}

// Apply writes the fields that are set onto r.
func (f RedirectFields) Apply(r *models.Redirect) {
	if f.Source != nil {
		r.Source = strings.TrimSpace(*f.Source)
	}
	if f.Target != nil {
		r.Target = strings.TrimSpace(*f.Target)
	}
	if f.Status != nil {
		r.Status = models.RedirectStatus(*f.Status)
	}
	if f.Match != nil {
		r.Match = strings.TrimSpace(*f.Match)
	}
	if f.Query != nil {
		r.Query = strings.TrimSpace(*f.Query)
	}
	if f.IgnoreCase != nil {
		r.IgnoreCase = *f.IgnoreCase
	}
	if f.TrailingSlash != nil {
		r.TrailingSlash = *f.TrailingSlash
	}
	if f.Note != nil {
		r.Note = strings.TrimSpace(*f.Note)
	}
}

// RuleFields are the fields of a URL rule write; a nil field is left as
// it is.
type RuleFields struct {
	URL           *string `json:"url,omitempty"`
	Match         *string `json:"match,omitempty"`
	Title         *string `json:"title,omitempty"`
	Description   *string `json:"description,omitempty"`
	Canonical     *string `json:"canonical,omitempty"`
	Robots        *string `json:"robots,omitempty"`
	OGTitle       *string `json:"og_title,omitempty"`
	OGDescription *string `json:"og_description,omitempty"`
	OGImage       *string `json:"og_image,omitempty"`
	Note          *string `json:"note,omitempty"`
}

// Apply writes the fields that are set onto r.
func (f RuleFields) Apply(r *models.Rule) {
	for column, v := range map[string]*string{
		"url": f.URL, "match": f.Match, "title": f.Title, "description": f.Description,
		"canonical": f.Canonical, "robots": f.Robots, "og_title": f.OGTitle,
		"og_description": f.OGDescription, "og_image": f.OGImage, "note": f.Note,
	} {
		if v != nil {
			SetRuleValue(r, column, strings.TrimSpace(*v))
		}
	}
}

// BatchChange is one change of a batch (D-12): set_seo addresses a page
// by URL (D-11); the redirect and rule operations create, update (by id)
// or delete (by id).
type BatchChange struct {
	Op        string            `json:"op"`
	ID        uint              `json:"id,omitempty"`
	URL       string            `json:"url,omitempty"`
	Locale    string            `json:"locale,omitempty"`
	Values    map[string]string `json:"values,omitempty"`
	ForceRule bool              `json:"force_rule,omitempty"`
	Redirect  *RedirectFields   `json:"redirect,omitempty"`
	Rule      *RuleFields       `json:"rule,omitempty"`
}

// DiffEntry is one changed object of a batch change: the history row the
// change wrote, its before and after values.
type DiffEntry struct {
	Kind    string         `json:"kind"`
	Subject string         `json:"subject"`
	What    string         `json:"what"`
	Before  map[string]any `json:"before"`
	After   map[string]any `json:"after"`
}

// Conflict is a finding of a batch (D-12): a loop refuses the batch, the
// other kinds are warnings.
type Conflict struct {
	Change  int      `json:"change"`
	Kind    string   `json:"kind"`
	URL     string   `json:"url,omitempty"`
	Path    []string `json:"path,omitempty"`
	Target  string   `json:"target,omitempty"`
	Message string   `json:"message"`
	Refused bool     `json:"refused,omitempty"`
}

// BatchItem is the outcome of one change: what it wrote (the diff), where
// a URL change went, its conflicts, and the refusal when it was refused.
type BatchItem struct {
	Index     int                 `json:"index"`
	Op        string              `json:"op"`
	Owner     *Owner              `json:"owner,omitempty"`
	Diff      []DiffEntry         `json:"diff"`
	Conflicts []Conflict          `json:"conflicts,omitempty"`
	Error     string              `json:"error,omitempty"`
	Message   string              `json:"message,omitempty"`
	Errors    map[string][]string `json:"errors,omitempty"`
	Path      []string            `json:"path,omitempty"`
}

// BatchResult is a batch's answer: the batch id its history rows carry,
// whether it was a dry run and whether it was applied, every change's
// outcome, all conflicts, and how many changes wrote something.
type BatchResult struct {
	BatchID   string      `json:"batch_id"`
	DryRun    bool        `json:"dry_run"`
	Applied   bool        `json:"applied"`
	Changes   []BatchItem `json:"changes"`
	Conflicts []Conflict  `json:"conflicts"`
	Count     int         `json:"count"`
}

// Refused returns the first refused change, nil when none was.
func (r BatchResult) Refused() *BatchItem {
	for i := range r.Changes {
		if r.Changes[i].Error != "" {
			return &r.Changes[i]
		}
	}
	return nil
}

// BatchRefusedError refuses a batch with a refused change: nothing was
// written (D-12, all-or-nothing).
type BatchRefusedError struct{ Result BatchResult }

func (e *BatchRefusedError) Error() string {
	if item := e.Result.Refused(); item != nil {
		return fmt.Sprintf("golem15.seo: batch change %d refused: %s", item.Index, item.Error)
	}
	return "golem15.seo: batch refused"
}

// ErrStale refuses an apply whose data changed after its preview.
var ErrStale = errors.New("golem15.seo: the data changed after the preview")

// errDryRun rolls a dry run back on purpose.
var errDryRun = errors.New("golem15.seo: dry run")

// BatchOptions are a batch's mode: DryRun rolls everything back after the
// diff and conflicts are collected; Via overrides the writer history
// records (import); BatchID names the batch (a new id when empty).
type BatchOptions struct {
	DryRun  bool
	Via     string
	BatchID string
}

// NewBatchID returns a random batch id.
func NewBatchID() string {
	var b [16]byte
	if _, err := rand.Read(b[:]); err != nil {
		panic(err)
	}
	return hex.EncodeToString(b[:])
}

// RunBatch runs changes as one batch (D-12, RESEARCH Pattern 8): every
// change goes through the service inside one transaction, each change's
// diff (the history rows it wrote) and conflicts are collected, and the
// conflicts that depend on the whole batch (a canonical pointing to a
// redirect another change creates) are checked once every change is in
// place. A dry run then rolls back on purpose; a refused change (a loop,
// an invalid value) rolls everything back and answers a
// *BatchRefusedError; otherwise the batch commits with one batch id on
// every history row.
func (s *Service) RunBatch(ctx context.Context, changes []BatchChange, opts BatchOptions) (BatchResult, error) {
	return s.runBatch(ctx, opts, func(context.Context, *gorm.DB) ([]BatchChange, error) { return changes, nil })
}

// runBatch is RunBatch with the changes built inside the transaction
// (the import maps its rows against the data it applies to).
func (s *Service) runBatch(ctx context.Context, opts BatchOptions, build func(ctx context.Context, tx *gorm.DB) ([]BatchChange, error)) (BatchResult, error) {
	gdb, err := s.GormDB()
	if err != nil {
		return BatchResult{}, err
	}
	res := BatchResult{BatchID: opts.BatchID, DryRun: opts.DryRun, Changes: []BatchItem{}, Conflicts: []Conflict{}}
	if res.BatchID == "" {
		res.BatchID = NewBatchID()
	}
	err = lagoon.Transaction(ctx, gdb, func(ctx context.Context, tx *gorm.DB) error {
		ctx = WithTx(WithBatch(ctx, res.BatchID), tx)
		if opts.Via != "" {
			ctx = WithVia(ctx, opts.Via)
		}
		// One lock for the whole batch, so a refused change rolled back to
		// its savepoint does not let another redirect writer in (WR-10).
		if err := lockRedirectWrites(tx); err != nil {
			return err
		}
		changes, err := build(ctx, tx)
		if err != nil {
			return err
		}
		if len(changes) > MaxBatchChanges {
			return &ProblemsError{Problems: map[string][]models.Problem{"changes": {{Key: "golem15.seo::lang.api.batch_too_large", Params: map[string]string{"max": fmt.Sprint(MaxBatchChanges)}}}}}
		}
		for i, ch := range changes {
			item, err := s.batchChange(ctx, tx, i, ch, res.BatchID)
			if err != nil {
				return err
			}
			res.Changes = append(res.Changes, item)
		}
		for i := range res.Changes {
			late, err := s.batchConflicts(ctx, tx, i, changes[i], res.Changes[i])
			if err != nil {
				return err
			}
			res.Changes[i].Conflicts = append(res.Changes[i].Conflicts, late...)
		}
		for _, item := range res.Changes {
			res.Conflicts = append(res.Conflicts, item.Conflicts...)
			if len(item.Diff) > 0 {
				res.Count++
			}
		}
		if res.Refused() != nil {
			return &BatchRefusedError{Result: res}
		}
		if opts.DryRun {
			return errDryRun
		}
		return nil
	})
	switch {
	case errors.Is(err, errDryRun):
		return res, nil
	case err != nil:
		var refused *BatchRefusedError
		if errors.As(err, &refused) {
			refused.Result.Applied = false
		}
		return res, err
	}
	res.Applied = true
	return res, nil
}

// batchChange runs one change under a savepoint, so a refused change
// leaves the transaction usable for the next one, and reads the history
// rows it wrote as its diff.
func (s *Service) batchChange(ctx context.Context, tx *gorm.DB, index int, ch BatchChange, batchID string) (BatchItem, error) {
	item := BatchItem{Index: index, Op: ch.Op, Diff: []DiffEntry{}}
	var last uint
	if err := tx.Raw(`SELECT COALESCE(MAX(id), 0) FROM golem15_seo_revisions`).Scan(&last).Error; err != nil {
		return item, err
	}
	point := fmt.Sprintf("seo_change_%d", index)
	if err := tx.SavePoint(point).Error; err != nil {
		return item, err
	}
	err := s.applyChange(ctx, tx, index, ch, &item)
	if err != nil {
		if rb := tx.RollbackTo(point).Error; rb != nil {
			return item, rb
		}
		var loop *LoopError
		var problems *ProblemsError
		switch {
		case errors.As(err, &loop):
			item.Error, item.Path = ConflictLoop, loop.Check.Path
			item.Message = s.LoopMessage(ctx, loop.Check)
			item.Conflicts = append(item.Conflicts, Conflict{Change: index, Kind: ConflictLoop, Path: loop.Check.Path, Message: item.Message, Refused: true})
		case errors.As(err, &problems):
			item.Error = "validation_failed"
			item.Errors = map[string][]string{}
			var msgs []string
			for field, list := range problems.Problems {
				for _, p := range list {
					msg := s.PhraseCtx(ctx, p.Key, p.Params)
					item.Errors[field] = append(item.Errors[field], msg)
					msgs = append(msgs, field+": "+msg)
				}
			}
			item.Message = strings.Join(msgs, "; ")
		case errors.Is(err, ErrNoRecord):
			item.Error, item.Message = "not_found", s.PhraseCtx(ctx, "golem15.seo::lang.api.change_not_found", nil)
		case errors.Is(err, ErrBadURL):
			item.Error, item.Message = "bad_url", err.Error()
		case errors.Is(err, errUnknownOp):
			item.Error, item.Message = "unknown_op", s.PhraseCtx(ctx, "golem15.seo::lang.api.op_unknown", map[string]string{"op": ch.Op})
		default:
			return item, err
		}
		return item, nil
	}
	if err := tx.Exec("RELEASE SAVEPOINT " + point).Error; err != nil {
		return item, err
	}
	var rows []models.Revision
	if err := tx.Where("id > ? AND batch_id = ?", last, batchID).Order("id ASC").Find(&rows).Error; err != nil {
		return item, err
	}
	for _, r := range rows {
		item.Diff = append(item.Diff, DiffEntry{Kind: r.SubjectKind, Subject: r.SubjectID, What: r.What, Before: r.Before, After: r.After})
	}
	return item, nil
}

// errUnknownOp refuses a change with an unknown operation.
var errUnknownOp = errors.New("golem15.seo: unknown batch operation")

// applyChange runs one change through the service.
func (s *Service) applyChange(ctx context.Context, tx *gorm.DB, index int, ch BatchChange, item *BatchItem) error {
	switch ch.Op {
	case OpSetSEO:
		res, err := s.SetURLSEO(ctx, tx, URLSEOWrite{URL: ch.URL, Locale: ch.Locale, Values: ch.Values, ForceRule: ch.ForceRule})
		if err != nil {
			return err
		}
		owner := res.Owner
		item.Owner = &owner
		return nil
	case OpCreateRedirect, OpUpdateRedirect:
		r := models.NewRedirect()
		r.Origin = RedirectOrigin(ctx)
		if ch.Op == OpUpdateRedirect {
			var err error
			if r, err = LoadRedirect(tx, ch.ID); err != nil {
				return err
			}
		}
		if ch.Redirect != nil {
			if problems := ch.Redirect.Problems(); len(problems) > 0 {
				return &ProblemsError{Problems: problems}
			}
			ch.Redirect.Apply(&r)
		}
		check, err := s.SaveRedirect(ctx, tx, &r)
		if err != nil {
			return err
		}
		if check.Chain {
			notice := ChainNotice(check)[0]
			item.Conflicts = append(item.Conflicts, Conflict{Change: index, Kind: ConflictChain, URL: r.Source, Path: check.Path, Target: check.Final, Message: s.PhraseCtx(ctx, notice.Message, notice.Params)})
		}
		return nil
	case OpDeleteRedirect:
		r, err := LoadRedirect(tx, ch.ID)
		if err != nil {
			return err
		}
		return s.DeleteRedirect(ctx, tx, r)
	case OpCreateRule, OpUpdateRule:
		r := models.Rule{Match: models.MatchExact}
		if ch.Op == OpUpdateRule {
			var err error
			if r, err = LoadRule(tx, ch.ID); err != nil {
				return err
			}
		}
		if ch.Rule != nil {
			ch.Rule.Apply(&r)
		}
		return s.SaveRule(ctx, tx, &r)
	case OpDeleteRule:
		r, err := LoadRule(tx, ch.ID)
		if err != nil {
			return err
		}
		return s.DeleteRule(ctx, tx, r)
	}
	return errUnknownOp
}

// batchConflicts are the warnings that depend on the whole batch, checked
// once every change is in place: noindex on the address of a published
// record (the records the sitemaps list; Phase 33's sitemap filter
// refines it) and a canonical pointing to a redirected or noindex
// address.
func (s *Service) batchConflicts(ctx context.Context, tx *gorm.DB, index int, ch BatchChange, item BatchItem) ([]Conflict, error) {
	if item.Error != "" {
		return nil, nil
	}
	var out []Conflict
	var robots, canonical, url string
	switch ch.Op {
	case OpSetSEO:
		robots, canonical, url = ch.Values["robots"], ch.Values["canonical"], ch.URL
		if item.Owner != nil {
			url = item.Owner.URL
		}
	case OpCreateRule, OpUpdateRule:
		if ch.Rule == nil {
			return nil, nil
		}
		if ch.Rule.Robots != nil {
			robots = *ch.Rule.Robots
		}
		if ch.Rule.Canonical != nil {
			canonical = *ch.Rule.Canonical
		}
		if ch.Rule.URL != nil {
			url = *ch.Rule.URL
		}
	default:
		return nil, nil
	}
	if strings.Contains(strings.ToLower(robots), "noindex") && url != "" && !strings.Contains(url, "*") {
		owner, _, err := s.FindOwner(ctx, tx, url, false)
		if err != nil && !errors.Is(err, ErrBadURL) {
			return nil, err
		}
		if err == nil && owner.WrittenTo == WrittenToRecord && owner.Published {
			out = append(out, Conflict{Change: index, Kind: ConflictNoindexPublished, URL: owner.URL,
				Message: s.PhraseCtx(ctx, "golem15.seo::lang.api.conflict_noindex_published", map[string]string{"url": owner.URL})})
		}
	}
	if strings.TrimSpace(canonical) != "" {
		kind, target, err := s.canonicalProblem(ctx, canonical)
		if err != nil {
			return nil, err
		}
		switch kind {
		case IssueCanonicalRedirected:
			out = append(out, Conflict{Change: index, Kind: ConflictCanonicalRedirected, URL: url, Target: target,
				Message: s.PhraseCtx(ctx, "golem15.seo::lang.transfer.row_canonical", map[string]string{"target": target})})
		case IssueCanonicalNoindex:
			out = append(out, Conflict{Change: index, Kind: ConflictCanonicalNoindex, URL: url,
				Message: s.PhraseCtx(ctx, "golem15.seo::lang.status.canonical_noindex", map[string]string{"url": canonical})})
		}
	}
	return out, nil
}