sm-seo-plugin/classes/crawl/job.go

package crawl

import (
	"context"
	"errors"
	"strings"
	"time"

	"git.golem15.com/golem15/sm-seo-plugin/classes"
	"git.golem15.com/golem15/sm-seo-plugin/classes/schema"
	"git.golem15.com/golem15/sm-seo-plugin/models"
	"git.golem15.com/golem15/summercms/modules/conga"
	"git.golem15.com/golem15/summercms/modules/diary"
	"git.golem15.com/golem15/summercms/modules/lagoon"
	"git.golem15.com/golem15/summercms/modules/pact"
	"gorm.io/gorm"
)

// CrawlKind is the River kind of the crawl job.
const CrawlKind = "golem15.seo.crawl"

// CrawlArgs are the arguments of the crawl job: the crawl row it runs.
type CrawlArgs struct {
	CrawlID uint `json:"crawl_id"`
}

// Kind is CrawlKind.
func (CrawlArgs) Kind() string { return CrawlKind }

// The crawl limits used until the Site audit settings say otherwise.
const (
	DefaultMaxPages = 5000
	DefaultDelay    = 250 * time.Millisecond
)

// AlreadyRunningError refuses a start while a crawl is queued or running
// (P33-39): one crawl runs at a time.
type AlreadyRunningError struct {
	Crawl models.Crawl
}

func (e *AlreadyRunningError) Error() string { return "golem15.seo: a crawl is already running" }

// Since is the time the running crawl started, else its creation time.
func (e *AlreadyRunningError) Since() time.Time {
	if e.Crawl.StartedAt != nil {
		return *e.Crawl.StartedAt
	}
	if e.Crawl.CreatedAt != nil {
		return *e.Crawl.CreatedAt
	}
	return time.Time{}
}

// Start queues a crawl (D-14, P33-39) inside tx: the settings row is
// locked so two starts cannot both pass the check that no crawl is queued
// or running, then the crawl row is written and the conga job dispatched
// in the same transaction. A missing site address refuses with
// ErrNoSiteURL, a running crawl with *AlreadyRunningError.
func Start(ctx context.Context, svc *classes.Service, tx *gorm.DB, trigger, name string) (models.Crawl, error) {
	site := svc.SiteURL()
	if site == "" {
		return models.Crawl{}, ErrNoSiteURL
	}
	if _, err := classes.SettingsRow(tx); err != nil {
		return models.Crawl{}, err
	}
	if err := tx.Exec(`SELECT id FROM golem15_seo_settings WHERE id = 1 FOR UPDATE`).Error; err != nil {
		return models.Crawl{}, err
	}
	active, err := ActiveCrawl(tx)
	if err != nil {
		return models.Crawl{}, err
	}
	if active != nil {
		return models.Crawl{}, &AlreadyRunningError{Crawl: *active}
	}
	limits, err := LoadLimits(tx)
	if err != nil {
		return models.Crawl{}, err
	}
	now := time.Now().UTC()
	row := models.Crawl{
		Status: models.CrawlQueued, Trigger: trigger, TriggerName: name, Site: site,
		MaxPages: limits.MaxPages, RenderStatus: models.RenderSkipped, CreatedAt: &now,
	}
	if err := tx.Create(&row).Error; err != nil {
		return models.Crawl{}, err
	}
	m, err := conga.From(svc.App())
	if err != nil {
		return models.Crawl{}, err
	}
	id, err := m.Dispatch(ctx, tx, CrawlArgs{CrawlID: row.ID}, conga.DispatchOpts{Label: svc.Phrase("golem15.seo::lang.jobs.crawl", nil)})
	if err != nil {
		return models.Crawl{}, err
	}
	row.JobID = &id
	if err := tx.Model(&models.Crawl{}).Where("id = ?", row.ID).Update("job_id", id).Error; err != nil {
		return models.Crawl{}, err
	}
	return row, nil
}

// Limits are a crawl's politeness limits.
type Limits struct {
	MaxPages int
	Delay    time.Duration
}

// LoadLimits reads the crawl limits.
func LoadLimits(db *gorm.DB) (Limits, error) {
	return Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay}, nil
}

// LoadCrawl reads one crawl.
func LoadCrawl(db *gorm.DB, id uint) (models.Crawl, error) {
	var rows []models.Crawl
	if err := db.Where("id = ?", id).Limit(1).Find(&rows).Error; err != nil {
		return models.Crawl{}, err
	}
	if len(rows) == 0 {
		return models.Crawl{}, gorm.ErrRecordNotFound
	}
	return rows[0], nil
}

// Run runs one queued crawl: it fetches the site, checks every page,
// stores the pages, links and findings with the CMS-data issues, and marks
// the crawl finished, cancelled (pages and issues fetched so far are kept)
// or failed (with the error; what was fetched before the failure is kept).
// A crawl that is no longer queued is left alone.
func Run(ctx context.Context, svc *classes.Service, id uint) error {
	gdb, err := svc.GormDB()
	if err != nil {
		return err
	}
	db := gdb.WithContext(ctx)
	row, err := LoadCrawl(db, id)
	if err != nil {
		return err
	}
	if row.Status != models.CrawlQueued {
		return nil
	}
	now := time.Now().UTC()
	moved := db.Model(&models.Crawl{}).Where("id = ? AND status = ?", id, models.CrawlQueued).
		Updates(map[string]any{"status": models.CrawlRunning, "started_at": now})
	if moved.Error != nil {
		return moved.Error
	}
	if moved.RowsAffected == 0 {
		return nil
	}
	res, runErr := run(ctx, svc, db, row)
	return finish(ctx, svc, db, row, res, runErr)
}

// run fetches and checks the site of one crawl.
func run(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.Crawl) (*Result, error) {
	f, err := NewFetcher(row.Site, svc.Config())
	if err != nil {
		return nil, err
	}
	limits, err := LoadLimits(db)
	if err != nil {
		return nil, err
	}
	if row.MaxPages > 0 {
		limits.MaxPages = row.MaxPages
	}
	var m *conga.Manager
	if app := svc.App(); app != nil {
		m, _ = conga.From(app)
	}
	opt := Options{
		MaxPages: limits.MaxPages,
		Delay:    limits.Delay,
		Renderer: rendererFrom(ctx),
		Canceled: func(ctx context.Context) bool {
			var status string
			if err := db.Raw(`SELECT status FROM golem15_seo_crawls WHERE id = ?`, row.ID).Scan(&status).Error; err == nil && status == models.CrawlCancelled {
				return true
			}
			if m != nil && row.JobID != nil {
				if c, err := m.CheckIfCanceled(ctx, *row.JobID); err == nil && c {
					return true
				}
			}
			return false
		},
		Progress: func(ctx context.Context, n int) {
			_ = db.Model(&models.Crawl{}).Where("id = ?", row.ID).Update("pages", n).Error
			if m != nil && row.JobID != nil {
				_ = m.UpdateJobState(ctx, *row.JobID, n, nil)
			}
		},
	}
	if m != nil && row.JobID != nil {
		_ = m.StartJob(ctx, *row.JobID, limits.MaxPages)
	}
	return Crawl(ctx, f, opt)
}

// finish stores what a crawl found and writes its final state.
func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.Crawl, res *Result, runErr error) error {
	status := models.CrawlFinished
	errText := ""
	if runErr != nil {
		status, errText = models.CrawlFailed, runErr.Error()
	}
	var current string
	_ = db.Raw(`SELECT status FROM golem15_seo_crawls WHERE id = ?`, row.ID).Scan(&current).Error
	if current == models.CrawlCancelled || (res != nil && res.Canceled) {
		status = models.CrawlCancelled
	}
	store := func(ctx context.Context, tx *gorm.DB) error {
		if res == nil {
			return nil
		}
		if err := SavePages(tx, row.ID, res); err != nil {
			return err
		}
		keyphrases, err := Keyphrases(tx)
		if err != nil {
			return err
		}
		findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging(), GenericAnchors: DefaultGenericAnchors, Keyphrases: keyphrases})
		cms, err := svc.Issues(classes.WithTx(ctx, tx))
		if err != nil {
			return err
		}
		id := row.ID
		return SaveIssues(tx, &id, findings, cms)
	}
	if err := lagoon.Transaction(ctx, db, store); err != nil {
		status, errText = models.CrawlFailed, err.Error()
	}
	updates := map[string]any{"status": status, "error": errText, "finished_at": time.Now().UTC()}
	if res != nil {
		updates["pages"] = len(res.Order)
		updates["limit_hit"] = res.LimitHit
		updates["render_status"] = res.Render.Status
		updates["render_pages"] = res.Render.Pages
		updates["render_note"] = res.Render.Note
	}
	if err := db.Model(&models.Crawl{}).Where("id = ?", row.ID).Updates(updates).Error; err != nil {
		return err
	}
	if m, err := conga.From(svc.App()); err == nil && row.JobID != nil {
		meta := map[string]any{"crawl_id": row.ID, "status": status}
		switch status {
		case models.CrawlFailed:
			_ = m.FailJob(ctx, *row.JobID, meta)
		case models.CrawlCancelled:
			_ = m.StopJob(ctx, *row.JobID, meta)
		default:
			_ = m.CompleteJob(ctx, *row.JobID, meta)
		}
	}
	if status == models.CrawlFailed {
		pluginLog.Warn("golem15.seo: crawl failed", "crawl", row.ID, "error", errText)
	}
	return nil
}

// Lookup returns the CMS record behind a crawled address (cached per
// address): the content type whose Match answers the path exactly, its
// descriptor facts in the matched locale, its primary parent's address and
// whether its SEO tab stores custom JSON-LD.
func Lookup(ctx context.Context, svc *classes.Service, db *gorm.DB) func(key string) *Record {
	cache := map[string]*Record{}
	reg := svc.Registry()
	unprefixed := svc.UnprefixedLocale()
	return func(key string) *Record {
		if reg == nil {
			return nil
		}
		path, _, _ := strings.Cut(key, "?")
		if r, ok := cache[path]; ok {
			return r
		}
		cache[path] = nil
		typeKey, match, err := reg.MatchPath(ctx, path, unprefixed)
		if err != nil || match == nil || !match.Exact {
			return nil
		}
		typ, ok := reg.ContentType(typeKey)
		if !ok || typ.Content == nil {
			return nil
		}
		rec, err := reg.Describe(ctx, typeKey, match.ID, unprefixed)
		if err != nil || rec == nil {
			return nil
		}
		out := &Record{Type: typeKey, ID: match.ID, Locale: match.Locale, TopHeading: rec.TopHeading, Tools: typ.Content.Tools}
		var loc *pact.ContentLocale
		for i := range rec.Locales {
			if rec.Locales[i].Locale == match.Locale {
				loc = &rec.Locales[i]
				break
			}
		}
		if loc == nil && len(rec.Locales) > 0 {
			loc = &rec.Locales[0]
		}
		if loc != nil {
			out.Locale = loc.Locale
			out.Title, out.Words, out.Headings, out.Images = loc.Title, loc.Words, loc.Headings, loc.Images
		}
		if len(rec.Parents) > 0 {
			if parent, err := reg.Describe(ctx, rec.Parents[0].Type, rec.Parents[0].ID, unprefixed); err == nil && parent != nil {
				for _, pl := range parent.Locales {
					if pl.Locale == out.Locale && pl.URL != "" {
						out.Links = append(out.Links, pl.URL)
					}
				}
			}
		}
		if entry, found, err := classes.LoadEntry(db, typ.Content.Model, match.ID, out.Locale); err == nil && found && entry.Data != nil {
			if raw, _ := entry.Data["schema"].(string); raw != "" {
				if v, err := schema.ParseValue(raw); err == nil && strings.TrimSpace(v.Custom) != "" {
					out.CustomJSONLD = true
				}
			}
		}
		cache[path] = out
		return out
	}
}

// Keyphrases maps each address with an SEO entry to its focus keyphrase.
func Keyphrases(db *gorm.DB) (map[string]string, error) {
	var rows []struct {
		URL       string
		Keyphrase string
	}
	if err := db.Raw(`SELECT url, keyphrase FROM golem15_seo_entries WHERE url <> '' AND keyphrase <> ''`).Scan(&rows).Error; err != nil {
		return nil, err
	}
	out := make(map[string]string, len(rows))
	for _, r := range rows {
		out[r.URL] = r.Keyphrase
	}
	return out, nil
}

// errCancelledCrawl refuses cancelling a crawl that is not queued or
// running.
var errCancelledCrawl = errors.New("golem15.seo: the crawl is not running")

// Cancel stops a queued or running crawl inside tx: the row is marked
// cancelled by name (the crawl keeps what it fetched) and its conga job
// cancelled.
func Cancel(ctx context.Context, svc *classes.Service, tx *gorm.DB, row models.Crawl, name string) error {
	if !row.Active() {
		return errCancelledCrawl
	}
	now := time.Now().UTC()
	updates := map[string]any{"status": models.CrawlCancelled, "cancelled_by": name}
	if row.Status == models.CrawlQueued {
		updates["finished_at"] = now
	}
	if err := tx.Model(&models.Crawl{}).Where("id = ?", row.ID).Updates(updates).Error; err != nil {
		return err
	}
	if row.JobID != nil {
		if m, err := conga.From(svc.App()); err == nil {
			if err := m.CancelJob(ctx, *row.JobID); err != nil {
				pluginLog.Warn("golem15.seo: cancel crawl job", "crawl", row.ID, "error", err)
			}
		}
	}
	return nil
}

// ErrNotRunning is a cancel of a crawl that is not queued or running.
var ErrNotRunning = errCancelledCrawl

// pluginLog is the crawler's logger (component seo).
var pluginLog = diary.For("seo")