sm-seo-plugin/classes/crawl/store.go

package crawl

import (
	"context"
	"sort"
	"time"

	"git.golem15.com/golem15/sm-seo-plugin/classes"
	"git.golem15.com/golem15/sm-seo-plugin/models"
	"git.golem15.com/golem15/summercms/modules/lagoon"
	"gorm.io/gorm"
)

// insertBatch is the rows written per INSERT.
const insertBatch = 200

// SavePages writes the pages and links one crawl fetched. The crawler
// writes only its own tables (D-14).
func SavePages(tx *gorm.DB, crawlID uint, res *Result) error {
	if res == nil {
		return nil
	}
	in := map[string]int{}
	var links []models.CrawlLink
	for _, key := range res.Order {
		p := res.Pages[key]
		if p.Facts == nil {
			continue
		}
		seen := map[string]bool{}
		for _, l := range p.Facts.Links {
			to := ""
			internal := false
			if res.fetcher != nil {
				to = res.fetcher.Key(l.URL)
				internal = to != ""
			}
			if !internal {
				to = l.URL.String()
			}
			links = append(links, models.CrawlLink{CrawlID: crawlID, FromURL: key, ToURL: to, Text: truncateUTF8(l.Text, 500), Rel: l.Rel, Internal: internal})
			if internal && to != key && !seen[to] {
				seen[to] = true
				in[to]++
			}
		}
	}
	pages := make([]models.CrawlPage, 0, len(res.Order))
	for _, key := range res.Order {
		p := res.Pages[key]
		row := models.CrawlPage{CrawlID: crawlID, URL: key, Depth: p.Depth, InSitemap: p.InSitemap, SitemapFile: p.SitemapFile, LinksIn: in[key]}
		if !p.FetchedAt.IsZero() {
			t := p.FetchedAt
			row.FetchedAt = &t
		}
		if r := p.Resp; r != nil {
			row.Status, row.Location, row.ContentType, row.XRobots = r.Status, r.Location, truncateUTF8(r.ContentType, 200), truncateUTF8(r.XRobots, 500)
		}
		if f := p.Facts; f != nil {
			row.Title, row.Description, row.Canonical, row.Robots = truncateUTF8(f.Title, 1000), truncateUTF8(f.Description, 2000), truncateUTF8(f.Canonical, 2048), truncateUTF8(f.Robots, 500)
			row.H1 = lagoon.Jsonable[[]string]{Data: f.H1(), Valid: true}
			hs := make([]models.CrawlHeading, 0, len(f.Headings))
			for _, h := range f.Headings {
				hs = append(hs, models.CrawlHeading{Level: h.Level, Text: truncateUTF8(h.Text, 500)})
			}
			row.Headings = lagoon.Jsonable[[]models.CrawlHeading]{Data: hs, Valid: true}
			row.Words, row.Text = f.Words, truncateUTF8(f.Text, models.MaxPageText)
			row.LinksOut = len(f.Links)
		}
		pages = append(pages, row)
	}
	if len(pages) > 0 {
		if err := tx.CreateInBatches(pages, insertBatch).Error; err != nil {
			return err
		}
	}
	if len(links) > 0 {
		if err := tx.CreateInBatches(links, insertBatch).Error; err != nil {
			return err
		}
	}
	return nil
}

// issueRow turns a finding into its stored row.
func issueRow(crawlID *uint, source string, f Finding, now time.Time) models.Issue {
	fix, tool, what := Route(f)
	details := map[string]any{}
	for k, v := range f.Details {
		details[k] = v
	}
	if what != "" {
		details["what"] = what
	}
	if f.Check != "" {
		details["check"] = f.Check
	}
	sev := Severity(f.Kind)
	t := now
	return models.Issue{
		CrawlID:     crawlID,
		Source:      source,
		Kind:        f.Kind,
		Severity:    sev,
		Rank:        models.SeverityRank(sev),
		URL:         truncateUTF8(f.URL, 2048),
		Locale:      f.Locale,
		Details:     lagoon.Jsonable[map[string]any]{Data: details, Valid: true},
		Tool:        tool,
		Fix:         fix,
		Fingerprint: Fingerprint(f.Kind, f.URL, f.Locale, f.Check),
		FoundAt:     &t,
	}
}

// cmsFinding turns a CMS-data issue of the classes report into a finding:
// its tool is the one the report names; a CMS issue without one is fixed
// in the record's admin form.
func cmsFinding(issue classes.Issue) Finding {
	f := Finding{Kind: issue.Kind, URL: issue.URL, Locale: issue.Locale, Details: issue.Details, Tool: issue.Tool}
	if f.Tool == "" {
		f.What = "the record in its admin form"
	}
	return f
}

// SaveIssues stores a crawl's findings and the CMS-data issues with it
// (crawlID nil: the CMS issues of a site not crawled yet). A CMS issue
// with the fingerprint of a crawl finding is the same problem and is not
// stored twice.
func SaveIssues(tx *gorm.DB, crawlID *uint, findings []Finding, cms []classes.Issue) error {
	now := time.Now().UTC()
	seen := map[string]bool{}
	var rows []models.Issue
	for _, f := range findings {
		row := issueRow(crawlID, models.SourceCrawl, f, now)
		if seen[row.Fingerprint] {
			continue
		}
		seen[row.Fingerprint] = true
		rows = append(rows, row)
	}
	for _, issue := range cms {
		row := issueRow(crawlID, models.SourceCMS, cmsFinding(issue), now)
		if seen[row.Fingerprint] {
			continue
		}
		seen[row.Fingerprint] = true
		rows = append(rows, row)
	}
	sort.SliceStable(rows, func(i, j int) bool {
		if rows[i].Rank != rows[j].Rank {
			return rows[i].Rank < rows[j].Rank
		}
		if rows[i].Kind != rows[j].Kind {
			return rows[i].Kind < rows[j].Kind
		}
		return rows[i].URL < rows[j].URL
	})
	if len(rows) == 0 {
		return nil
	}
	return tx.CreateInBatches(rows, insertBatch).Error
}

// LatestCrawl returns the newest crawl that is no longer queued or
// running, nil when there is none: its issues are the audit's current
// ones.
func LatestCrawl(db *gorm.DB) (*models.Crawl, error) {
	var rows []models.Crawl
	if err := db.Where("status NOT IN ?", []string{models.CrawlQueued, models.CrawlRunning}).Order("id DESC").Limit(1).Find(&rows).Error; err != nil {
		return nil, err
	}
	if len(rows) == 0 {
		return nil, nil
	}
	return &rows[0], nil
}

// ActiveCrawl returns the queued or running crawl, nil when none.
func ActiveCrawl(db *gorm.DB) (*models.Crawl, error) {
	var rows []models.Crawl
	if err := db.Where("status IN ?", []string{models.CrawlQueued, models.CrawlRunning}).Order("id DESC").Limit(1).Find(&rows).Error; err != nil {
		return nil, err
	}
	if len(rows) == 0 {
		return nil, nil
	}
	return &rows[0], nil
}

// RecoverStale ends the running crawls whose job stopped writing its
// heartbeat (P33-60): a running row whose COALESCE(heartbeat_at,
// started_at, created_at) is older than StaleAfter before now becomes
// cancelled when its cancel was requested, else failed with the error
// "the crawl job stopped without finishing (no heartbeat since <time>)",
// with finished_at now. It answers the rows it ended, with their new
// status, so the caller can end their conga jobs.
func RecoverStale(tx *gorm.DB, now time.Time) ([]models.Crawl, error) {
	now = now.UTC()
	var rows []models.Crawl
	if err := tx.Where("status = ? AND COALESCE(heartbeat_at, started_at, created_at) < ?", models.CrawlRunning, now.Add(-StaleAfter)).
		Order("id ASC").Find(&rows).Error; err != nil {
		return nil, err
	}
	out := rows[:0]
	for _, r := range rows {
		last := r.HeartbeatAt
		if last == nil {
			last = r.StartedAt
		}
		if last == nil {
			last = r.CreatedAt
		}
		updates := map[string]any{"finished_at": now}
		if r.CancelRequestedAt != nil {
			updates["status"], updates["error"] = models.CrawlCancelled, ""
		} else {
			since := "an unknown time"
			if last != nil {
				since = last.UTC().Format(time.RFC3339)
			}
			updates["status"], updates["error"] = models.CrawlFailed, "the crawl job stopped without finishing (no heartbeat since "+since+")"
		}
		res := tx.Model(&models.Crawl{}).Where("id = ? AND status = ?", r.ID, models.CrawlRunning).Updates(updates)
		if res.Error != nil {
			return nil, res.Error
		}
		if res.RowsAffected == 0 {
			continue
		}
		r.Status = updates["status"].(string)
		r.Error = updates["error"].(string)
		r.FinishedAt = &now
		pluginLog.Warn("golem15.seo: ended a stale crawl", "crawl", r.ID, "status", r.Status)
		out = append(out, r)
	}
	return out, nil
}

// CurrentScope narrows an issues query to the audit's current issues: the
// newest finished, failed or cancelled crawl's, or before any crawl the
// stored CMS-data issues.
func CurrentScope(db *gorm.DB, latest *models.Crawl) *gorm.DB {
	if latest == nil {
		return db.Where("golem15_seo_issues.crawl_id IS NULL")
	}
	return db.Where("golem15_seo_issues.crawl_id = ?", latest.ID)
}

// IgnoredSet returns which of the fingerprints are ignored.
func IgnoredSet(db *gorm.DB, fingerprints []string) (map[string]bool, error) {
	out := map[string]bool{}
	if len(fingerprints) == 0 {
		return out, nil
	}
	var rows []models.IssueIgnore
	for start := 0; start < len(fingerprints); start += 1000 {
		end := min(start+1000, len(fingerprints))
		var part []models.IssueIgnore
		if err := db.Where("fingerprint IN ?", fingerprints[start:end]).Find(&part).Error; err != nil {
			return nil, err
		}
		rows = append(rows, part...)
	}
	for _, r := range rows {
		out[r.Fingerprint] = true
	}
	return out, nil
}

// Current returns the crawl whose issues are the audit's current ones (nil
// before any crawl). Before any crawl, the CMS-data checks run now and
// their issues replace the stored ones, so the list shows the CMS data as
// it is (UI-SPEC C23: when the list is opened with no crawl yet).
func Current(ctx context.Context, svc *classes.Service, db *gorm.DB) (*models.Crawl, error) {
	latest, err := LatestCrawl(db)
	if err != nil || latest != nil {
		return latest, err
	}
	return nil, RefreshCMS(ctx, svc, db)
}

// RefreshCMS brings the stored CMS-data issues of a site not crawled yet
// up to date with the report, in one transaction: an issue found again
// keeps its row (and id), the others are deleted and new ones stored.
func RefreshCMS(ctx context.Context, svc *classes.Service, db *gorm.DB) error {
	return lagoon.Transaction(ctx, db, func(ctx context.Context, tx *gorm.DB) error {
		cms, err := svc.Issues(classes.WithTx(ctx, tx))
		if err != nil {
			return err
		}
		found := map[string]classes.Issue{}
		for _, c := range cms {
			found[Fingerprint(c.Kind, c.URL, c.Locale, "")] = c
		}
		var existing []models.Issue
		if err := tx.Where("crawl_id IS NULL").Find(&existing).Error; err != nil {
			return err
		}
		have := map[string]bool{}
		var gone []uint
		now := time.Now().UTC()
		for _, e := range existing {
			c, ok := found[e.Fingerprint]
			if !ok || have[e.Fingerprint] {
				gone = append(gone, e.ID)
				continue
			}
			have[e.Fingerprint] = true
			row := issueRow(nil, models.SourceCMS, cmsFinding(c), now)
			if err := tx.Model(&models.Issue{}).Where("id = ?", e.ID).Updates(map[string]any{
				"details": row.Details, "tool": row.Tool, "fix": row.Fix, "severity": row.Severity, "rank": row.Rank,
			}).Error; err != nil {
				return err
			}
		}
		if len(gone) > 0 {
			if err := tx.Where("id IN ?", gone).Delete(&models.Issue{}).Error; err != nil {
				return err
			}
		}
		var fresh []classes.Issue
		for _, c := range cms {
			if !have[Fingerprint(c.Kind, c.URL, c.Locale, "")] {
				fresh = append(fresh, c)
			}
		}
		return SaveIssues(tx, nil, nil, fresh)
	})
}