feat: Site audit and Crawls screens with their MCP tools

- golem15.seo.issues (Site audit): current issues errors first, labels,
  summary and fix columns, filters, P7 issue_details, Check this address
  again, Ignore, Stop ignoring, bulk Ignore, ignored rows badged
- golem15.seo.crawls (Crawls): newest first, P6 crawl_summary with one
  status callout, staging and rendered-comparison lines, stats strip and
  per-kind table; Cancel; a running crawl refuses delete
- Start crawl toolbar action on both, refusing already_running and
  no_site_url; navigation Site audit (gauge) and Crawls (globe)
- routes and MCP tools start/cancel/list/get/delete crawls, list_issues
  with filters (open by default), recheck_url, ignore and unignore
- AdminOperations maps both controllers; toolbar actions get their own
  operation kind
- TestCrawlAdmin, TestCrawlTools, TestIssueKindLabels
This commit is contained in:
Jakub Zych
2026-11-04 15:34:22 +01:00
parent edc03cae4b
commit c4f0b4f94c
32 changed files with 2778 additions and 20 deletions

304
classes/crawl/actions.go Normal file
View File

@@ -0,0 +1,304 @@
package crawl
import (
"context"
"errors"
"net/url"
"time"
"git.golem15.com/golem15/sm-seo-plugin/classes"
"git.golem15.com/golem15/sm-seo-plugin/models"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// ErrDeleteRunning refuses deleting a queued or running crawl
// (crawls.delete_running).
var ErrDeleteRunning = errors.New("golem15.seo: a running crawl cannot be deleted")
// LoadIssue reads one stored issue.
func LoadIssue(db *gorm.DB, id uint) (models.Issue, error) {
var rows []models.Issue
if err := db.Where("id = ?", id).Limit(1).Find(&rows).Error; err != nil {
return models.Issue{}, err
}
if len(rows) == 0 {
return models.Issue{}, gorm.ErrRecordNotFound
}
return rows[0], nil
}
// IsIgnored reports whether an issue's fingerprint is ignored.
func IsIgnored(db *gorm.DB, issue models.Issue) (bool, error) {
set, err := IgnoredSet(db, []string{issue.Fingerprint})
return set[issue.Fingerprint], err
}
// Ignore hides issues in this and later crawls (UI-SPEC C23): their
// fingerprints are stored with the administrator and the time. It returns
// the number of fingerprints newly ignored.
func Ignore(tx *gorm.DB, issues []models.Issue, adminID *uint) (int, error) {
n := 0
now := time.Now().UTC()
seen := map[string]bool{}
for _, i := range issues {
if seen[i.Fingerprint] {
continue
}
seen[i.Fingerprint] = true
row := models.IssueIgnore{Fingerprint: i.Fingerprint, AdminID: adminID, IgnoredAt: &now}
res := tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "fingerprint"}}, DoNothing: true}).Create(&row)
if res.Error != nil {
return n, res.Error
}
n += int(res.RowsAffected)
}
return n, nil
}
// Unignore shows issues in the audit again; it returns the number of
// fingerprints no longer ignored.
func Unignore(tx *gorm.DB, issues []models.Issue) (int, error) {
var fps []string
for _, i := range issues {
fps = append(fps, i.Fingerprint)
}
if len(fps) == 0 {
return 0, nil
}
res := tx.Where("fingerprint IN ?", fps).Delete(&models.IssueIgnore{})
return int(res.RowsAffected), res.Error
}
// DeleteCrawls deletes crawls with their pages, links and issues; a
// queued or running one refuses the whole call with ErrDeleteRunning.
func DeleteCrawls(tx *gorm.DB, ids []uint) (int, error) {
if len(ids) == 0 {
return 0, nil
}
var active int64
if err := tx.Model(&models.Crawl{}).Where("id IN ? AND status IN ?", ids, []string{models.CrawlQueued, models.CrawlRunning}).Count(&active).Error; err != nil {
return 0, err
}
if active > 0 {
return 0, ErrDeleteRunning
}
res := tx.Where("id IN ?", ids).Delete(&models.Crawl{})
return int(res.RowsAffected), res.Error
}
// Prune deletes the finished, failed and cancelled crawls beyond the
// newest keep, with their pages, links and issues.
func Prune(db *gorm.DB, keep int) error {
if keep < 1 {
keep = 1
}
return db.Exec(`DELETE FROM golem15_seo_crawls WHERE status NOT IN ('queued', 'running') AND id NOT IN (
SELECT id FROM golem15_seo_crawls WHERE status NOT IN ('queued', 'running') ORDER BY id DESC LIMIT ?)`, keep).Error
}
// crossPageKinds are the crawl kinds that depend on other pages of the
// crawl; a recheck of one address keeps them.
var crossPageKinds = map[string]bool{
KindDuplicateTitle: true, KindDuplicateDesc: true, KindSitemapMissing: true,
KindOrphanPage: true, KindBrokenLink: true, KindRedirectedLink: true, KindSitemapBlocked: true,
}
// RecheckResult is what a recheck left for the address.
type RecheckResult struct {
URL string
Remaining int
}
// Recheck fetches one issue's address again through the crawler client
// (with its canonical and hreflang targets) and replaces the address's
// issues in the issue's crawl (UI-SPEC C23): the per-page crawl findings
// are checked again, broken and redirected link findings are dropped once
// the address answers 200, and the CMS-data issues of the address are
// checked again. An issue stored before any crawl refreshes the CMS-data
// issues. Remaining counts the address's issues afterwards.
func Recheck(ctx context.Context, svc *classes.Service, tx *gorm.DB, issue models.Issue) (RecheckResult, error) {
out := RecheckResult{URL: issue.URL}
ctx = classes.WithTx(ctx, tx)
cms, err := svc.Issues(ctx)
if err != nil {
return out, err
}
var mine []classes.Issue
for _, c := range cms {
if c.URL == issue.URL {
mine = append(mine, c)
}
}
if issue.CrawlID == nil {
found := map[string]bool{}
for _, c := range mine {
found[Fingerprint(c.Kind, c.URL, c.Locale, "")] = true
}
var existing []models.Issue
if err := tx.Where("crawl_id IS NULL AND url = ?", issue.URL).Find(&existing).Error; err != nil {
return out, err
}
have := map[string]bool{}
for _, e := range existing {
if found[e.Fingerprint] {
have[e.Fingerprint] = true
} else if err := tx.Delete(&models.Issue{}, e.ID).Error; err != nil {
return out, err
}
}
var fresh []classes.Issue
for _, c := range mine {
if !have[Fingerprint(c.Kind, c.URL, c.Locale, "")] {
fresh = append(fresh, c)
}
}
if err := SaveIssues(tx, nil, nil, fresh); err != nil {
return out, err
}
return out, countRemaining(tx, nil, &out)
}
row, err := LoadCrawl(tx, *issue.CrawlID)
if err != nil {
return out, err
}
f, err := NewFetcher(row.Site, svc.Config())
if err != nil {
return out, err
}
u, err := url.Parse(f.Absolute(issue.URL))
if err != nil || f.Key(u) == "" {
return out, classes.ErrBadURL
}
res := &Result{Pages: map[string]*Page{}, Listed: map[string]string{}, Render: RenderState{Status: renderStatusSkipped}, fetcher: f}
fetch := func(key string) *Page {
p := &Page{Key: key, Resp: f.Get(ctx, key), FetchedAt: time.Now().UTC()}
if p.Resp.Err == "" && p.Resp.Status == 200 && p.Resp.HTML() {
base, _ := url.Parse(f.Absolute(key))
p.Facts = Parse(base, p.Resp.Body)
}
p.Resp.Body = nil
res.Pages[key] = p
return p
}
var stored []models.CrawlPage
if err := tx.Where("crawl_id = ? AND url = ?", row.ID, issue.URL).Limit(1).Find(&stored).Error; err != nil {
return out, err
}
main := fetch(issue.URL)
res.Order = []string{issue.URL}
if len(stored) > 0 && stored[0].InSitemap {
main.InSitemap, main.SitemapFile = true, stored[0].SitemapFile
res.Listed[issue.URL] = stored[0].SitemapFile
}
if main.Facts != nil {
var extra []string
if c, err := url.Parse(main.Facts.Canonical); err == nil && main.Facts.Canonical != "" {
extra = append(extra, f.Key(c))
}
for _, a := range main.Facts.Alternates {
extra = append(extra, f.Key(a.URL))
}
for _, key := range extra {
if key != "" && res.Pages[key] == nil {
fetch(key)
}
}
}
limits, err := LoadLimits(tx)
if err != nil {
return out, err
}
env := &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging(), GenericAnchors: limits.GenericAnchors}
rec := env.record(issue.URL)
var findings []Finding
findings = append(findings, checkStatus(res, main, rec)...)
if main.OK() {
findings = append(findings, checkHead(res, main, rec)...)
findings = append(findings, checkHeadings(main, rec)...)
findings = append(findings, checkJSONLD(main, rec)...)
findings = append(findings, checkHreflang(res, main, rec)...)
findings = append(findings, checkGenericAnchors(res, main, rec, env.GenericAnchors)...)
findings = append(findings, checkImages(main, rec)...)
findings = append(findings, checkContentCMS(res, main, rec)...)
}
for _, sf := range checkSitemaps(res, env) {
if sf.URL == issue.URL && sf.Kind != KindSitemapMissing && sf.Kind != KindSitemapBlocked {
findings = append(findings, sf)
}
}
var keep []string
for kind := range crossPageKinds {
if main.Resp.Status == 200 && (kind == KindBrokenLink || kind == KindRedirectedLink) {
continue
}
keep = append(keep, kind)
}
// The address's issues found again keep their row (and id); the
// others are deleted, the new ones stored. Cross-page findings stay,
// and so does the rendered comparison, which a recheck does not repeat.
found := map[string]bool{}
for _, f := range findings {
found[Fingerprint(f.Kind, f.URL, f.Locale, f.Check)] = true
}
for _, c := range mine {
found[Fingerprint(c.Kind, c.URL, c.Locale, "")] = true
}
id := row.ID
var existing []models.Issue
if err := tx.Where("crawl_id = ? AND url = ?", id, issue.URL).Find(&existing).Error; err != nil {
return out, err
}
keepKind := map[string]bool{}
for _, k := range keep {
keepKind[k] = true
}
have := map[string]bool{}
var gone []uint
for _, e := range existing {
stays := found[e.Fingerprint] ||
(e.Source == models.SourceCrawl && keepKind[e.Kind]) ||
(e.Source == models.SourceCrawl && e.Kind == KindContentJS && e.DetailsMap()["check"] == checkRendered)
if stays {
have[e.Fingerprint] = true
} else {
gone = append(gone, e.ID)
}
}
if len(gone) > 0 {
if err := tx.Where("id IN ?", gone).Delete(&models.Issue{}).Error; err != nil {
return out, err
}
}
var fresh []Finding
for _, f := range findings {
if !have[Fingerprint(f.Kind, f.URL, f.Locale, f.Check)] {
fresh = append(fresh, f)
}
}
var freshCMS []classes.Issue
for _, c := range mine {
if !have[Fingerprint(c.Kind, c.URL, c.Locale, "")] {
freshCMS = append(freshCMS, c)
}
}
if err := SaveIssues(tx, &id, fresh, freshCMS); err != nil {
return out, err
}
return out, countRemaining(tx, &id, &out)
}
func countRemaining(tx *gorm.DB, crawlID *uint, out *RecheckResult) error {
q := tx.Model(&models.Issue{}).Where("url = ?", out.URL)
if crawlID == nil {
q = q.Where("crawl_id IS NULL")
} else {
q = q.Where("crawl_id = ?", *crawlID)
}
var n int64
if err := q.Count(&n).Error; err != nil {
return err
}
out.Remaining = int(n)
return nil
}

View File

@@ -102,15 +102,22 @@ func Start(ctx context.Context, svc *classes.Service, tx *gorm.DB, trigger, name
return row, nil
}
// Limits are a crawl's politeness limits.
// Limits are a crawl's politeness limits and the Site audit settings it
// reads: pages per crawl, the pause between requests, the crawls kept and
// the generic link texts.
type Limits struct {
MaxPages int
Delay time.Duration
MaxPages int
Delay time.Duration
Keep int
GenericAnchors []string
}
// DefaultKeep is the number of crawls kept.
const DefaultKeep = 10
// LoadLimits reads the crawl limits.
func LoadLimits(db *gorm.DB) (Limits, error) {
return Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay}, nil
return Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay, Keep: DefaultKeep, GenericAnchors: DefaultGenericAnchors}, nil
}
// LoadCrawl reads one crawl.
@@ -214,6 +221,10 @@ func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.C
if current == models.CrawlCancelled || (res != nil && res.Canceled) {
status = models.CrawlCancelled
}
limits, err := LoadLimits(db)
if err != nil {
return err
}
store := func(ctx context.Context, tx *gorm.DB) error {
if res == nil {
return nil
@@ -225,7 +236,7 @@ func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.C
if err != nil {
return err
}
findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging(), GenericAnchors: DefaultGenericAnchors, Keyphrases: keyphrases})
findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging(), GenericAnchors: limits.GenericAnchors, Keyphrases: keyphrases})
cms, err := svc.Issues(classes.WithTx(ctx, tx))
if err != nil {
return err
@@ -261,6 +272,9 @@ func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.C
if status == models.CrawlFailed {
pluginLog.Warn("golem15.seo: crawl failed", "crawl", row.ID, "error", errText)
}
if err := Prune(db, limits.Keep); err != nil {
pluginLog.Warn("golem15.seo: prune crawls", "error", err)
}
return nil
}

View File

@@ -232,17 +232,51 @@ func Current(ctx context.Context, svc *classes.Service, db *gorm.DB) (*models.Cr
return nil, RefreshCMS(ctx, svc, db)
}
// RefreshCMS replaces the stored CMS-data issues of a site not crawled
// yet with the report's current ones, in one transaction.
// RefreshCMS brings the stored CMS-data issues of a site not crawled yet
// up to date with the report, in one transaction: an issue found again
// keeps its row (and id), the others are deleted and new ones stored.
func RefreshCMS(ctx context.Context, svc *classes.Service, db *gorm.DB) error {
return lagoon.Transaction(ctx, db, func(ctx context.Context, tx *gorm.DB) error {
cms, err := svc.Issues(classes.WithTx(ctx, tx))
if err != nil {
return err
}
if err := tx.Where("crawl_id IS NULL").Delete(&models.Issue{}).Error; err != nil {
found := map[string]classes.Issue{}
for _, c := range cms {
found[Fingerprint(c.Kind, c.URL, c.Locale, "")] = c
}
var existing []models.Issue
if err := tx.Where("crawl_id IS NULL").Find(&existing).Error; err != nil {
return err
}
return SaveIssues(tx, nil, nil, cms)
have := map[string]bool{}
var gone []uint
now := time.Now().UTC()
for _, e := range existing {
c, ok := found[e.Fingerprint]
if !ok || have[e.Fingerprint] {
gone = append(gone, e.ID)
continue
}
have[e.Fingerprint] = true
row := issueRow(nil, models.SourceCMS, cmsFinding(c), now)
if err := tx.Model(&models.Issue{}).Where("id = ?", e.ID).Updates(map[string]any{
"details": row.Details, "tool": row.Tool, "fix": row.Fix, "severity": row.Severity, "rank": row.Rank,
}).Error; err != nil {
return err
}
}
if len(gone) > 0 {
if err := tx.Where("id IN ?", gone).Delete(&models.Issue{}).Error; err != nil {
return err
}
}
var fresh []classes.Issue
for _, c := range cms {
if !have[Fingerprint(c.Kind, c.URL, c.Locale, "")] {
fresh = append(fresh, c)
}
}
return SaveIssues(tx, nil, nil, fresh)
})
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"net/url"
"reflect"
"strconv"
"strings"
"git.golem15.com/golem15/sm-seo-plugin/classes/schema"
@@ -427,3 +428,110 @@ func (s *Service) primaryGoneCallout(ctx context.Context, db *gorm.DB, key, id s
Title: s.PhraseCtx(ctx, "golem15.seo::lang.status.primary_gone", map[string]string{"name": name}),
}, true
}
// IssueParams turns an issue's details into the :name parameters of its
// issues.sum_<kind> phrase: scalars as text, lists by their length or
// joined (fields, also), with the aliases the Phase 32 kinds need (count
// from also or hops, url from canonical).
func IssueParams(details map[string]any) map[string]string {
out := map[string]string{}
for k, v := range details {
switch x := v.(type) {
case string:
out[k] = x
case float64:
out[k] = strconv.FormatFloat(x, 'f', -1, 64)
case int:
out[k] = strconv.Itoa(x)
case bool:
out[k] = strconv.FormatBool(x)
case []any:
parts := make([]string, 0, len(x))
for _, e := range x {
if s, ok := e.(string); ok {
parts = append(parts, s)
}
}
out[k] = strings.Join(parts, ", ")
case []string:
out[k] = strings.Join(x, ", ")
}
}
if _, ok := out["count"]; !ok {
if also, ok := details["also"].([]any); ok {
out["count"] = strconv.Itoa(len(also))
} else if hops, ok := out["hops"]; ok {
out["count"] = hops
}
}
if _, ok := out["url"]; !ok {
if c, ok := out["canonical"]; ok {
out["url"] = c
}
}
return out
}
// IssueSummary is the one-line Details cell of an issue (issues.sum_<kind>).
func (s *Service) IssueSummary(ctx context.Context, kind string, details map[string]any) string {
return s.PhraseCtx(ctx, "golem15.seo::lang.issues.sum_"+kind, IssueParams(details))
}
// IssueRow is one row of P7's "What was found" table.
type IssueRow struct {
Label string
Value string
Mono bool
}
// IssueLink is a page named in P7 (a linking page or a suggestion): its
// address and, when the page has an admin form, the admin path.
type IssueLink struct {
URL string
Admin string
Phrase string
}
// IssueDetailsView is the P7 issue_details view model (UI-SPEC C23).
type IssueDetailsView struct {
Rows []IssueRow
LinkedFrom []IssueLink
More string
Orphan bool
Suggestions []IssueLink
FixText string
PageURL string
RichResults bool
RichURL string
IgnoredNotes string
}
// CrawlStat is one item of P6's stats strip.
type CrawlStat struct {
Label string
Value string
}
// CrawlKindRow is one row of P6's per-kind table.
type CrawlKindRow struct {
Kind string
Label string
Severity string
Count int
Link string
}
// CrawlSummaryView is the P6 crawl_summary view model (UI-SPEC C24): one
// status callout (Tone "", warning or danger), the staging and rendered
// comparison lines, the stats strip and the per-kind table.
type CrawlSummaryView struct {
Tone string
Title string
Text string
Staging string
Render string
Stats []CrawlStat
Kinds []CrawlKindRow
NoIssues bool
HasCallout bool
}