sm-seo-plugin/classes/crawl/actions.go

package crawl

import (
	"context"
	"errors"
	"net/url"
	"time"

	"git.golem15.com/golem15/sm-seo-plugin/classes"
	"git.golem15.com/golem15/sm-seo-plugin/models"
	"gorm.io/gorm"
	"gorm.io/gorm/clause"
)

// ErrDeleteRunning refuses deleting a queued or running crawl
// (crawls.delete_running).
var ErrDeleteRunning = errors.New("golem15.seo: a running crawl cannot be deleted")

// LoadIssue reads one stored issue.
func LoadIssue(db *gorm.DB, id uint) (models.Issue, error) {
	var rows []models.Issue
	if err := db.Where("id = ?", id).Limit(1).Find(&rows).Error; err != nil {
		return models.Issue{}, err
	}
	if len(rows) == 0 {
		return models.Issue{}, gorm.ErrRecordNotFound
	}
	return rows[0], nil
}

// IsIgnored reports whether an issue's fingerprint is ignored.
func IsIgnored(db *gorm.DB, issue models.Issue) (bool, error) {
	set, err := IgnoredSet(db, []string{issue.Fingerprint})
	return set[issue.Fingerprint], err
}

// Ignore hides issues in this and later crawls (UI-SPEC C23): their
// fingerprints are stored with the administrator and the time. It returns
// the number of fingerprints newly ignored.
func Ignore(tx *gorm.DB, issues []models.Issue, adminID *uint) (int, error) {
	n := 0
	now := time.Now().UTC()
	seen := map[string]bool{}
	for _, i := range issues {
		if seen[i.Fingerprint] {
			continue
		}
		seen[i.Fingerprint] = true
		row := models.IssueIgnore{Fingerprint: i.Fingerprint, AdminID: adminID, IgnoredAt: &now}
		res := tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "fingerprint"}}, DoNothing: true}).Create(&row)
		if res.Error != nil {
			return n, res.Error
		}
		n += int(res.RowsAffected)
	}
	return n, nil
}

// Unignore shows issues in the audit again; it returns the number of
// fingerprints no longer ignored.
func Unignore(tx *gorm.DB, issues []models.Issue) (int, error) {
	var fps []string
	for _, i := range issues {
		fps = append(fps, i.Fingerprint)
	}
	if len(fps) == 0 {
		return 0, nil
	}
	res := tx.Where("fingerprint IN ?", fps).Delete(&models.IssueIgnore{})
	return int(res.RowsAffected), res.Error
}

// DeleteCrawls deletes crawls with their pages, links and issues; a
// queued or running one (also one whose cancel is requested) refuses the
// whole call with ErrDeleteRunning. A running crawl without a heartbeat is
// ended first (RecoverStale), so it can be deleted.
func DeleteCrawls(tx *gorm.DB, ids []uint) (int, error) {
	if len(ids) == 0 {
		return 0, nil
	}
	if _, err := RecoverStale(tx, time.Now()); err != nil {
		return 0, err
	}
	var active int64
	if err := tx.Model(&models.Crawl{}).Where("id IN ? AND status IN ?", ids, []string{models.CrawlQueued, models.CrawlRunning}).Count(&active).Error; err != nil {
		return 0, err
	}
	if active > 0 {
		return 0, ErrDeleteRunning
	}
	res := tx.Where("id IN ?", ids).Delete(&models.Crawl{})
	return int(res.RowsAffected), res.Error
}

// Prune deletes the finished, failed and cancelled crawls beyond the
// newest keep, with their pages, links and issues.
func Prune(db *gorm.DB, keep int) error {
	if keep < 1 {
		keep = 1
	}
	return db.Exec(`DELETE FROM golem15_seo_crawls WHERE status NOT IN ('queued', 'running') AND id NOT IN (
	SELECT id FROM golem15_seo_crawls WHERE status NOT IN ('queued', 'running') ORDER BY id DESC LIMIT ?)`, keep).Error
}

// crossPageKinds are the crawl kinds that depend on other pages of the
// crawl; a recheck of one address keeps them.
var crossPageKinds = map[string]bool{
	KindDuplicateTitle: true, KindDuplicateDesc: true, KindSitemapMissing: true,
	KindOrphanPage: true, KindBrokenLink: true, KindRedirectedLink: true, KindSitemapBlocked: true,
}

// RecheckResult is what a recheck left for the address.
type RecheckResult struct {
	URL       string
	Remaining int
}

// Recheck fetches one issue's address again through the crawler client
// (with its canonical and hreflang targets) and replaces the address's
// issues in the issue's crawl (UI-SPEC C23): the per-page crawl findings
// are checked again, broken and redirected link findings are dropped once
// the address answers 200, and the CMS-data issues of the address are
// checked again. An issue stored before any crawl refreshes the CMS-data
// issues. Remaining counts the address's issues afterwards.
func Recheck(ctx context.Context, svc *classes.Service, tx *gorm.DB, issue models.Issue) (RecheckResult, error) {
	out := RecheckResult{URL: issue.URL}
	ctx = classes.WithTx(ctx, tx)
	cms, err := svc.Issues(ctx)
	if err != nil {
		return out, err
	}
	var mine []classes.Issue
	for _, c := range cms {
		if c.URL == issue.URL {
			mine = append(mine, c)
		}
	}
	if issue.CrawlID == nil {
		found := map[string]bool{}
		for _, c := range mine {
			found[Fingerprint(c.Kind, c.URL, c.Locale, "")] = true
		}
		var existing []models.Issue
		if err := tx.Where("crawl_id IS NULL AND url = ?", issue.URL).Find(&existing).Error; err != nil {
			return out, err
		}
		have := map[string]bool{}
		for _, e := range existing {
			if found[e.Fingerprint] {
				have[e.Fingerprint] = true
			} else if err := tx.Delete(&models.Issue{}, e.ID).Error; err != nil {
				return out, err
			}
		}
		var fresh []classes.Issue
		for _, c := range mine {
			if !have[Fingerprint(c.Kind, c.URL, c.Locale, "")] {
				fresh = append(fresh, c)
			}
		}
		if err := SaveIssues(tx, nil, nil, fresh); err != nil {
			return out, err
		}
		return out, countRemaining(tx, nil, &out)
	}
	row, err := LoadCrawl(tx, *issue.CrawlID)
	if err != nil {
		return out, err
	}
	f, err := NewFetcher(row.Site, svc.Config())
	if err != nil {
		return out, err
	}
	u, err := url.Parse(f.Absolute(issue.URL))
	if err != nil || f.Key(u) == "" {
		return out, classes.ErrBadURL
	}
	res := &Result{Pages: map[string]*Page{}, Listed: map[string]string{}, Render: RenderState{Status: renderStatusSkipped}, fetcher: f}
	fetch := func(key string) *Page {
		p := &Page{Key: key, Resp: f.Get(ctx, key), FetchedAt: time.Now().UTC()}
		if p.Resp.Err == "" && p.Resp.Status == 200 && p.Resp.HTML() {
			base, _ := url.Parse(f.Absolute(key))
			p.Facts = Parse(base, p.Resp.Body)
		}
		p.Resp.Body = nil
		res.Pages[key] = p
		return p
	}
	var stored []models.CrawlPage
	if err := tx.Where("crawl_id = ? AND url = ?", row.ID, issue.URL).Limit(1).Find(&stored).Error; err != nil {
		return out, err
	}
	main := fetch(issue.URL)
	res.Order = []string{issue.URL}
	if len(stored) > 0 && stored[0].InSitemap {
		main.InSitemap, main.SitemapFile = true, stored[0].SitemapFile
		res.Listed[issue.URL] = stored[0].SitemapFile
	}
	if main.Facts != nil {
		var extra []string
		if c, err := url.Parse(main.Facts.Canonical); err == nil && main.Facts.Canonical != "" {
			extra = append(extra, f.Key(c))
		}
		for _, a := range main.Facts.Alternates {
			extra = append(extra, f.Key(a.URL))
		}
		for _, key := range extra {
			if key != "" && res.Pages[key] == nil {
				fetch(key)
			}
		}
	}
	limits, err := LoadLimits(tx)
	if err != nil {
		return out, err
	}
	env := &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging(), GenericAnchors: limits.GenericAnchors}
	rec := env.record(issue.URL)
	var findings []Finding
	findings = append(findings, checkStatus(res, main, rec)...)
	if main.OK() {
		findings = append(findings, checkHead(res, main, rec)...)
		findings = append(findings, checkHeadings(main, rec)...)
		findings = append(findings, checkJSONLD(main, rec)...)
		findings = append(findings, checkHreflang(res, main, rec)...)
		findings = append(findings, checkGenericAnchors(res, main, rec, env.GenericAnchors)...)
		findings = append(findings, checkImages(main, rec)...)
		findings = append(findings, checkContentCMS(res, main, rec)...)
	}
	for _, sf := range checkSitemaps(res, env) {
		if sf.URL == issue.URL && sf.Kind != KindSitemapMissing && sf.Kind != KindSitemapBlocked {
			findings = append(findings, sf)
		}
	}
	var keep []string
	for kind := range crossPageKinds {
		if main.Resp.Status == 200 && (kind == KindBrokenLink || kind == KindRedirectedLink) {
			continue
		}
		keep = append(keep, kind)
	}
	// The address's issues found again keep their row (and id); the
	// others are deleted, the new ones stored. Cross-page findings stay,
	// and so does the rendered comparison, which a recheck does not repeat.
	found := map[string]bool{}
	for _, f := range findings {
		found[Fingerprint(f.Kind, f.URL, f.Locale, f.Check)] = true
	}
	for _, c := range mine {
		found[Fingerprint(c.Kind, c.URL, c.Locale, "")] = true
	}
	id := row.ID
	var existing []models.Issue
	if err := tx.Where("crawl_id = ? AND url = ?", id, issue.URL).Find(&existing).Error; err != nil {
		return out, err
	}
	keepKind := map[string]bool{}
	for _, k := range keep {
		keepKind[k] = true
	}
	have := map[string]bool{}
	var gone []uint
	for _, e := range existing {
		stays := found[e.Fingerprint] ||
			(e.Source == models.SourceCrawl && keepKind[e.Kind]) ||
			(e.Source == models.SourceCrawl && e.Kind == KindContentJS && e.DetailsMap()["check"] == checkRendered)
		if stays {
			have[e.Fingerprint] = true
		} else {
			gone = append(gone, e.ID)
		}
	}
	if len(gone) > 0 {
		if err := tx.Where("id IN ?", gone).Delete(&models.Issue{}).Error; err != nil {
			return out, err
		}
	}
	var fresh []Finding
	for _, f := range findings {
		if !have[Fingerprint(f.Kind, f.URL, f.Locale, f.Check)] {
			fresh = append(fresh, f)
		}
	}
	var freshCMS []classes.Issue
	for _, c := range mine {
		if !have[Fingerprint(c.Kind, c.URL, c.Locale, "")] {
			freshCMS = append(freshCMS, c)
		}
	}
	if err := SaveIssues(tx, &id, fresh, freshCMS); err != nil {
		return out, err
	}
	return out, countRemaining(tx, &id, &out)
}

func countRemaining(tx *gorm.DB, crawlID *uint, out *RecheckResult) error {
	q := tx.Model(&models.Issue{}).Where("url = ?", out.URL)
	if crawlID == nil {
		q = q.Where("crawl_id IS NULL")
	} else {
		q = q.Where("crawl_id = ?", *crawlID)
	}
	var n int64
	if err := q.Count(&n).Error; err != nil {
		return err
	}
	out.Remaining = int(n)
	return nil
}