From c4f0b4f94ca80a3d8bf73bec8223dde03ccf10f3 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Wed, 4 Nov 2026 15:34:22 +0100 Subject: [PATCH] feat: Site audit and Crawls screens with their MCP tools - golem15.seo.issues (Site audit): current issues errors first, labels, summary and fix columns, filters, P7 issue_details, Check this address again, Ignore, Stop ignoring, bulk Ignore, ignored rows badged - golem15.seo.crawls (Crawls): newest first, P6 crawl_summary with one status callout, staging and rendered-comparison lines, stats strip and per-kind table; Cancel; a running crawl refuses delete - Start crawl toolbar action on both, refusing already_running and no_site_url; navigation Site audit (gauge) and Crawls (globe) - routes and MCP tools start/cancel/list/get/delete crawls, list_issues with filters (open by default), recheck_url, ignore and unignore - AdminOperations maps both controllers; toolbar actions get their own operation kind - TestCrawlAdmin, TestCrawlTools, TestIssueKindLabels --- admin_navigation.go | 14 + classes/crawl/actions.go | 304 ++++++++++++++++ classes/crawl/job.go | 24 +- classes/crawl/store.go | 42 ++- classes/partials.go | 108 ++++++ controllers/admin_registry.go | 2 + controllers/api/crawls.go | 223 +++++++++++- controllers/api/issues.go | 142 +++++++- controllers/crawls.go | 350 ++++++++++++++++++ controllers/crawls/_crawl_summary.htm | 31 ++ controllers/crawls/config_filter.yaml | 11 + controllers/crawls/config_form.yaml | 14 + controllers/crawls/config_list.yaml | 22 ++ controllers/issues.go | 505 ++++++++++++++++++++++++++ controllers/issues/_issue_details.htm | 48 +++ controllers/issues/config_filter.yaml | 28 ++ controllers/issues/config_form.yaml | 14 + controllers/issues/config_list.yaml | 25 ++ crawl_admin_test.go | 325 +++++++++++++++++ crawl_site_test.go | 28 ++ crawler_test.go | 2 +- lang/en/lang.yaml | 134 +++++++ lang/pl/lang.yaml | 146 ++++++++ mcp.go | 85 ++++- models/crawl/columns.yaml | 27 ++ models/crawl/fields.yaml | 29 ++ models/issue/columns.yaml | 43 +++ models/issue/fields.yaml | 33 ++ operations.go | 19 + operations_test.go | 8 +- routes.go | 7 + seo_tab_test.go | 5 +- 32 files changed, 2778 insertions(+), 20 deletions(-) create mode 100644 classes/crawl/actions.go create mode 100644 controllers/crawls.go create mode 100644 controllers/crawls/_crawl_summary.htm create mode 100644 controllers/crawls/config_filter.yaml create mode 100644 controllers/crawls/config_form.yaml create mode 100644 controllers/crawls/config_list.yaml create mode 100644 controllers/issues.go create mode 100644 controllers/issues/_issue_details.htm create mode 100644 controllers/issues/config_filter.yaml create mode 100644 controllers/issues/config_form.yaml create mode 100644 controllers/issues/config_list.yaml create mode 100644 crawl_admin_test.go create mode 100644 models/crawl/columns.yaml create mode 100644 models/crawl/fields.yaml create mode 100644 models/issue/columns.yaml create mode 100644 models/issue/fields.yaml diff --git a/admin_navigation.go b/admin_navigation.go index 25db6fa..4dc9d40 100644 --- a/admin_navigation.go +++ b/admin_navigation.go @@ -45,6 +45,20 @@ func (*Plugin) Navigation() []pact.NavigationItem { Permissions: []string{controllers.PermissionManage}, Controller: controllers.PagesControllerID, }, + { + Code: "audit", + Label: "golem15.seo::lang.nav.audit", + Icon: "gauge", + Permissions: []string{controllers.PermissionManage}, + Controller: controllers.IssuesControllerID, + }, + { + Code: "crawls", + Label: "golem15.seo::lang.nav.crawls", + Icon: "globe", + Permissions: []string{controllers.PermissionManage}, + Controller: controllers.CrawlsControllerID, + }, { Code: "robots", Label: "golem15.seo::lang.nav.robots", diff --git a/classes/crawl/actions.go b/classes/crawl/actions.go new file mode 100644 index 0000000..fa6e74d --- /dev/null +++ b/classes/crawl/actions.go @@ -0,0 +1,304 @@ +package crawl + +import ( + "context" + "errors" + "net/url" + "time" + + "git.golem15.com/golem15/sm-seo-plugin/classes" + "git.golem15.com/golem15/sm-seo-plugin/models" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// ErrDeleteRunning refuses deleting a queued or running crawl +// (crawls.delete_running). +var ErrDeleteRunning = errors.New("golem15.seo: a running crawl cannot be deleted") + +// LoadIssue reads one stored issue. +func LoadIssue(db *gorm.DB, id uint) (models.Issue, error) { + var rows []models.Issue + if err := db.Where("id = ?", id).Limit(1).Find(&rows).Error; err != nil { + return models.Issue{}, err + } + if len(rows) == 0 { + return models.Issue{}, gorm.ErrRecordNotFound + } + return rows[0], nil +} + +// IsIgnored reports whether an issue's fingerprint is ignored. +func IsIgnored(db *gorm.DB, issue models.Issue) (bool, error) { + set, err := IgnoredSet(db, []string{issue.Fingerprint}) + return set[issue.Fingerprint], err +} + +// Ignore hides issues in this and later crawls (UI-SPEC C23): their +// fingerprints are stored with the administrator and the time. It returns +// the number of fingerprints newly ignored. +func Ignore(tx *gorm.DB, issues []models.Issue, adminID *uint) (int, error) { + n := 0 + now := time.Now().UTC() + seen := map[string]bool{} + for _, i := range issues { + if seen[i.Fingerprint] { + continue + } + seen[i.Fingerprint] = true + row := models.IssueIgnore{Fingerprint: i.Fingerprint, AdminID: adminID, IgnoredAt: &now} + res := tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "fingerprint"}}, DoNothing: true}).Create(&row) + if res.Error != nil { + return n, res.Error + } + n += int(res.RowsAffected) + } + return n, nil +} + +// Unignore shows issues in the audit again; it returns the number of +// fingerprints no longer ignored. +func Unignore(tx *gorm.DB, issues []models.Issue) (int, error) { + var fps []string + for _, i := range issues { + fps = append(fps, i.Fingerprint) + } + if len(fps) == 0 { + return 0, nil + } + res := tx.Where("fingerprint IN ?", fps).Delete(&models.IssueIgnore{}) + return int(res.RowsAffected), res.Error +} + +// DeleteCrawls deletes crawls with their pages, links and issues; a +// queued or running one refuses the whole call with ErrDeleteRunning. +func DeleteCrawls(tx *gorm.DB, ids []uint) (int, error) { + if len(ids) == 0 { + return 0, nil + } + var active int64 + if err := tx.Model(&models.Crawl{}).Where("id IN ? AND status IN ?", ids, []string{models.CrawlQueued, models.CrawlRunning}).Count(&active).Error; err != nil { + return 0, err + } + if active > 0 { + return 0, ErrDeleteRunning + } + res := tx.Where("id IN ?", ids).Delete(&models.Crawl{}) + return int(res.RowsAffected), res.Error +} + +// Prune deletes the finished, failed and cancelled crawls beyond the +// newest keep, with their pages, links and issues. +func Prune(db *gorm.DB, keep int) error { + if keep < 1 { + keep = 1 + } + return db.Exec(`DELETE FROM golem15_seo_crawls WHERE status NOT IN ('queued', 'running') AND id NOT IN ( + SELECT id FROM golem15_seo_crawls WHERE status NOT IN ('queued', 'running') ORDER BY id DESC LIMIT ?)`, keep).Error +} + +// crossPageKinds are the crawl kinds that depend on other pages of the +// crawl; a recheck of one address keeps them. +var crossPageKinds = map[string]bool{ + KindDuplicateTitle: true, KindDuplicateDesc: true, KindSitemapMissing: true, + KindOrphanPage: true, KindBrokenLink: true, KindRedirectedLink: true, KindSitemapBlocked: true, +} + +// RecheckResult is what a recheck left for the address. +type RecheckResult struct { + URL string + Remaining int +} + +// Recheck fetches one issue's address again through the crawler client +// (with its canonical and hreflang targets) and replaces the address's +// issues in the issue's crawl (UI-SPEC C23): the per-page crawl findings +// are checked again, broken and redirected link findings are dropped once +// the address answers 200, and the CMS-data issues of the address are +// checked again. An issue stored before any crawl refreshes the CMS-data +// issues. Remaining counts the address's issues afterwards. +func Recheck(ctx context.Context, svc *classes.Service, tx *gorm.DB, issue models.Issue) (RecheckResult, error) { + out := RecheckResult{URL: issue.URL} + ctx = classes.WithTx(ctx, tx) + cms, err := svc.Issues(ctx) + if err != nil { + return out, err + } + var mine []classes.Issue + for _, c := range cms { + if c.URL == issue.URL { + mine = append(mine, c) + } + } + if issue.CrawlID == nil { + found := map[string]bool{} + for _, c := range mine { + found[Fingerprint(c.Kind, c.URL, c.Locale, "")] = true + } + var existing []models.Issue + if err := tx.Where("crawl_id IS NULL AND url = ?", issue.URL).Find(&existing).Error; err != nil { + return out, err + } + have := map[string]bool{} + for _, e := range existing { + if found[e.Fingerprint] { + have[e.Fingerprint] = true + } else if err := tx.Delete(&models.Issue{}, e.ID).Error; err != nil { + return out, err + } + } + var fresh []classes.Issue + for _, c := range mine { + if !have[Fingerprint(c.Kind, c.URL, c.Locale, "")] { + fresh = append(fresh, c) + } + } + if err := SaveIssues(tx, nil, nil, fresh); err != nil { + return out, err + } + return out, countRemaining(tx, nil, &out) + } + row, err := LoadCrawl(tx, *issue.CrawlID) + if err != nil { + return out, err + } + f, err := NewFetcher(row.Site, svc.Config()) + if err != nil { + return out, err + } + u, err := url.Parse(f.Absolute(issue.URL)) + if err != nil || f.Key(u) == "" { + return out, classes.ErrBadURL + } + res := &Result{Pages: map[string]*Page{}, Listed: map[string]string{}, Render: RenderState{Status: renderStatusSkipped}, fetcher: f} + fetch := func(key string) *Page { + p := &Page{Key: key, Resp: f.Get(ctx, key), FetchedAt: time.Now().UTC()} + if p.Resp.Err == "" && p.Resp.Status == 200 && p.Resp.HTML() { + base, _ := url.Parse(f.Absolute(key)) + p.Facts = Parse(base, p.Resp.Body) + } + p.Resp.Body = nil + res.Pages[key] = p + return p + } + var stored []models.CrawlPage + if err := tx.Where("crawl_id = ? AND url = ?", row.ID, issue.URL).Limit(1).Find(&stored).Error; err != nil { + return out, err + } + main := fetch(issue.URL) + res.Order = []string{issue.URL} + if len(stored) > 0 && stored[0].InSitemap { + main.InSitemap, main.SitemapFile = true, stored[0].SitemapFile + res.Listed[issue.URL] = stored[0].SitemapFile + } + if main.Facts != nil { + var extra []string + if c, err := url.Parse(main.Facts.Canonical); err == nil && main.Facts.Canonical != "" { + extra = append(extra, f.Key(c)) + } + for _, a := range main.Facts.Alternates { + extra = append(extra, f.Key(a.URL)) + } + for _, key := range extra { + if key != "" && res.Pages[key] == nil { + fetch(key) + } + } + } + limits, err := LoadLimits(tx) + if err != nil { + return out, err + } + env := &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging(), GenericAnchors: limits.GenericAnchors} + rec := env.record(issue.URL) + var findings []Finding + findings = append(findings, checkStatus(res, main, rec)...) + if main.OK() { + findings = append(findings, checkHead(res, main, rec)...) + findings = append(findings, checkHeadings(main, rec)...) + findings = append(findings, checkJSONLD(main, rec)...) + findings = append(findings, checkHreflang(res, main, rec)...) + findings = append(findings, checkGenericAnchors(res, main, rec, env.GenericAnchors)...) + findings = append(findings, checkImages(main, rec)...) + findings = append(findings, checkContentCMS(res, main, rec)...) + } + for _, sf := range checkSitemaps(res, env) { + if sf.URL == issue.URL && sf.Kind != KindSitemapMissing && sf.Kind != KindSitemapBlocked { + findings = append(findings, sf) + } + } + var keep []string + for kind := range crossPageKinds { + if main.Resp.Status == 200 && (kind == KindBrokenLink || kind == KindRedirectedLink) { + continue + } + keep = append(keep, kind) + } + // The address's issues found again keep their row (and id); the + // others are deleted, the new ones stored. Cross-page findings stay, + // and so does the rendered comparison, which a recheck does not repeat. + found := map[string]bool{} + for _, f := range findings { + found[Fingerprint(f.Kind, f.URL, f.Locale, f.Check)] = true + } + for _, c := range mine { + found[Fingerprint(c.Kind, c.URL, c.Locale, "")] = true + } + id := row.ID + var existing []models.Issue + if err := tx.Where("crawl_id = ? AND url = ?", id, issue.URL).Find(&existing).Error; err != nil { + return out, err + } + keepKind := map[string]bool{} + for _, k := range keep { + keepKind[k] = true + } + have := map[string]bool{} + var gone []uint + for _, e := range existing { + stays := found[e.Fingerprint] || + (e.Source == models.SourceCrawl && keepKind[e.Kind]) || + (e.Source == models.SourceCrawl && e.Kind == KindContentJS && e.DetailsMap()["check"] == checkRendered) + if stays { + have[e.Fingerprint] = true + } else { + gone = append(gone, e.ID) + } + } + if len(gone) > 0 { + if err := tx.Where("id IN ?", gone).Delete(&models.Issue{}).Error; err != nil { + return out, err + } + } + var fresh []Finding + for _, f := range findings { + if !have[Fingerprint(f.Kind, f.URL, f.Locale, f.Check)] { + fresh = append(fresh, f) + } + } + var freshCMS []classes.Issue + for _, c := range mine { + if !have[Fingerprint(c.Kind, c.URL, c.Locale, "")] { + freshCMS = append(freshCMS, c) + } + } + if err := SaveIssues(tx, &id, fresh, freshCMS); err != nil { + return out, err + } + return out, countRemaining(tx, &id, &out) +} + +func countRemaining(tx *gorm.DB, crawlID *uint, out *RecheckResult) error { + q := tx.Model(&models.Issue{}).Where("url = ?", out.URL) + if crawlID == nil { + q = q.Where("crawl_id IS NULL") + } else { + q = q.Where("crawl_id = ?", *crawlID) + } + var n int64 + if err := q.Count(&n).Error; err != nil { + return err + } + out.Remaining = int(n) + return nil +} diff --git a/classes/crawl/job.go b/classes/crawl/job.go index 0f36431..3ca7af2 100644 --- a/classes/crawl/job.go +++ b/classes/crawl/job.go @@ -102,15 +102,22 @@ func Start(ctx context.Context, svc *classes.Service, tx *gorm.DB, trigger, name return row, nil } -// Limits are a crawl's politeness limits. +// Limits are a crawl's politeness limits and the Site audit settings it +// reads: pages per crawl, the pause between requests, the crawls kept and +// the generic link texts. type Limits struct { - MaxPages int - Delay time.Duration + MaxPages int + Delay time.Duration + Keep int + GenericAnchors []string } +// DefaultKeep is the number of crawls kept. +const DefaultKeep = 10 + // LoadLimits reads the crawl limits. func LoadLimits(db *gorm.DB) (Limits, error) { - return Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay}, nil + return Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay, Keep: DefaultKeep, GenericAnchors: DefaultGenericAnchors}, nil } // LoadCrawl reads one crawl. @@ -214,6 +221,10 @@ func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.C if current == models.CrawlCancelled || (res != nil && res.Canceled) { status = models.CrawlCancelled } + limits, err := LoadLimits(db) + if err != nil { + return err + } store := func(ctx context.Context, tx *gorm.DB) error { if res == nil { return nil @@ -225,7 +236,7 @@ func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.C if err != nil { return err } - findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging(), GenericAnchors: DefaultGenericAnchors, Keyphrases: keyphrases}) + findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging(), GenericAnchors: limits.GenericAnchors, Keyphrases: keyphrases}) cms, err := svc.Issues(classes.WithTx(ctx, tx)) if err != nil { return err @@ -261,6 +272,9 @@ func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.C if status == models.CrawlFailed { pluginLog.Warn("golem15.seo: crawl failed", "crawl", row.ID, "error", errText) } + if err := Prune(db, limits.Keep); err != nil { + pluginLog.Warn("golem15.seo: prune crawls", "error", err) + } return nil } diff --git a/classes/crawl/store.go b/classes/crawl/store.go index bca0bc5..da7e66a 100644 --- a/classes/crawl/store.go +++ b/classes/crawl/store.go @@ -232,17 +232,51 @@ func Current(ctx context.Context, svc *classes.Service, db *gorm.DB) (*models.Cr return nil, RefreshCMS(ctx, svc, db) } -// RefreshCMS replaces the stored CMS-data issues of a site not crawled -// yet with the report's current ones, in one transaction. +// RefreshCMS brings the stored CMS-data issues of a site not crawled yet +// up to date with the report, in one transaction: an issue found again +// keeps its row (and id), the others are deleted and new ones stored. func RefreshCMS(ctx context.Context, svc *classes.Service, db *gorm.DB) error { return lagoon.Transaction(ctx, db, func(ctx context.Context, tx *gorm.DB) error { cms, err := svc.Issues(classes.WithTx(ctx, tx)) if err != nil { return err } - if err := tx.Where("crawl_id IS NULL").Delete(&models.Issue{}).Error; err != nil { + found := map[string]classes.Issue{} + for _, c := range cms { + found[Fingerprint(c.Kind, c.URL, c.Locale, "")] = c + } + var existing []models.Issue + if err := tx.Where("crawl_id IS NULL").Find(&existing).Error; err != nil { return err } - return SaveIssues(tx, nil, nil, cms) + have := map[string]bool{} + var gone []uint + now := time.Now().UTC() + for _, e := range existing { + c, ok := found[e.Fingerprint] + if !ok || have[e.Fingerprint] { + gone = append(gone, e.ID) + continue + } + have[e.Fingerprint] = true + row := issueRow(nil, models.SourceCMS, cmsFinding(c), now) + if err := tx.Model(&models.Issue{}).Where("id = ?", e.ID).Updates(map[string]any{ + "details": row.Details, "tool": row.Tool, "fix": row.Fix, "severity": row.Severity, "rank": row.Rank, + }).Error; err != nil { + return err + } + } + if len(gone) > 0 { + if err := tx.Where("id IN ?", gone).Delete(&models.Issue{}).Error; err != nil { + return err + } + } + var fresh []classes.Issue + for _, c := range cms { + if !have[Fingerprint(c.Kind, c.URL, c.Locale, "")] { + fresh = append(fresh, c) + } + } + return SaveIssues(tx, nil, nil, fresh) }) } diff --git a/classes/partials.go b/classes/partials.go index 78f486e..6811edd 100644 --- a/classes/partials.go +++ b/classes/partials.go @@ -5,6 +5,7 @@ import ( "fmt" "net/url" "reflect" + "strconv" "strings" "git.golem15.com/golem15/sm-seo-plugin/classes/schema" @@ -427,3 +428,110 @@ func (s *Service) primaryGoneCallout(ctx context.Context, db *gorm.DB, key, id s Title: s.PhraseCtx(ctx, "golem15.seo::lang.status.primary_gone", map[string]string{"name": name}), }, true } + +// IssueParams turns an issue's details into the :name parameters of its +// issues.sum_ phrase: scalars as text, lists by their length or +// joined (fields, also), with the aliases the Phase 32 kinds need (count +// from also or hops, url from canonical). +func IssueParams(details map[string]any) map[string]string { + out := map[string]string{} + for k, v := range details { + switch x := v.(type) { + case string: + out[k] = x + case float64: + out[k] = strconv.FormatFloat(x, 'f', -1, 64) + case int: + out[k] = strconv.Itoa(x) + case bool: + out[k] = strconv.FormatBool(x) + case []any: + parts := make([]string, 0, len(x)) + for _, e := range x { + if s, ok := e.(string); ok { + parts = append(parts, s) + } + } + out[k] = strings.Join(parts, ", ") + case []string: + out[k] = strings.Join(x, ", ") + } + } + if _, ok := out["count"]; !ok { + if also, ok := details["also"].([]any); ok { + out["count"] = strconv.Itoa(len(also)) + } else if hops, ok := out["hops"]; ok { + out["count"] = hops + } + } + if _, ok := out["url"]; !ok { + if c, ok := out["canonical"]; ok { + out["url"] = c + } + } + return out +} + +// IssueSummary is the one-line Details cell of an issue (issues.sum_). +func (s *Service) IssueSummary(ctx context.Context, kind string, details map[string]any) string { + return s.PhraseCtx(ctx, "golem15.seo::lang.issues.sum_"+kind, IssueParams(details)) +} + +// IssueRow is one row of P7's "What was found" table. +type IssueRow struct { + Label string + Value string + Mono bool +} + +// IssueLink is a page named in P7 (a linking page or a suggestion): its +// address and, when the page has an admin form, the admin path. +type IssueLink struct { + URL string + Admin string + Phrase string +} + +// IssueDetailsView is the P7 issue_details view model (UI-SPEC C23). +type IssueDetailsView struct { + Rows []IssueRow + LinkedFrom []IssueLink + More string + Orphan bool + Suggestions []IssueLink + FixText string + PageURL string + RichResults bool + RichURL string + IgnoredNotes string +} + +// CrawlStat is one item of P6's stats strip. +type CrawlStat struct { + Label string + Value string +} + +// CrawlKindRow is one row of P6's per-kind table. +type CrawlKindRow struct { + Kind string + Label string + Severity string + Count int + Link string +} + +// CrawlSummaryView is the P6 crawl_summary view model (UI-SPEC C24): one +// status callout (Tone "", warning or danger), the staging and rendered +// comparison lines, the stats strip and the per-kind table. +type CrawlSummaryView struct { + Tone string + Title string + Text string + Staging string + Render string + Stats []CrawlStat + Kinds []CrawlKindRow + NoIssues bool + HasCallout bool +} diff --git a/controllers/admin_registry.go b/controllers/admin_registry.go index 132d921..b4d2392 100644 --- a/controllers/admin_registry.go +++ b/controllers/admin_registry.go @@ -33,6 +33,8 @@ func AdminControllers(app func() *backpack.App) []pact.AdminController { notFoundAdminController{app: app}, rulesAdminController{app: app}, pagesAdminController{app: app}, + issuesAdminController{app: app}, + crawlsAdminController{app: app}, robotsAdminController{app: app}, historyAdminController{app: app}, transferAdminController{app: app}, diff --git a/controllers/api/crawls.go b/controllers/api/crawls.go index d92f177..4a4372b 100644 --- a/controllers/api/crawls.go +++ b/controllers/api/crawls.go @@ -4,6 +4,8 @@ import ( "context" "errors" "net/http" + "strconv" + "strings" "time" "git.golem15.com/golem15/sm-seo-plugin/classes" @@ -61,7 +63,7 @@ func StartCrawl(service func() *classes.Service) http.HandlerFunc { if err != nil { return 0, nil, StartRefusal(ctx, svc, err) } - return http.StatusAccepted, map[string]any{"data": crawlView(row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil + return http.StatusAccepted, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil }) } @@ -88,3 +90,222 @@ func StartRefusal(ctx context.Context, svc *classes.Service, err error) error { func RunningMessage(ctx context.Context, svc *classes.Service, running *crawl.AlreadyRunningError) string { return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.already_running", map[string]string{"time": running.Since().UTC().Format("2006-01-02 15:04 UTC")}) } + +// RenderView is a crawl's rendered comparison in an answer (P33-55): +// skipped (no golem15.seo.crawl_chrome_path), done with the pages +// compared, or failed with the note; message says it in the request +// locale. +type RenderView struct { + Status string `json:"status"` + Pages int `json:"pages"` + Note string `json:"note,omitempty"` + Message string `json:"message"` +} + +// renderView is the answer form of a crawl's comparison. +func renderView(ctx context.Context, svc *classes.Service, c models.Crawl) RenderView { + out := RenderView{Status: c.RenderStatus, Pages: c.RenderPages, Note: c.RenderNote} + switch c.RenderStatus { + case models.RenderDone: + out.Message = svc.ChoiceCtx(ctx, "golem15.seo::lang.crawls.render_done", c.RenderPages, nil) + case models.RenderFailed: + out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_failed", map[string]string{"error": c.RenderNote}) + default: + out.Status = models.RenderSkipped + out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_skipped", nil) + } + return out +} + +// ListCrawlsInput is the query of GET /_seo/api/v1/crawls. +type ListCrawlsInput struct { + Page int `json:"page,omitempty" query:"page" jsonschema:"the page number, from 1"` + PerPage int `json:"per_page,omitempty" query:"per_page" jsonschema:"rows per page, at most 200 (50 by default)"` + Status string `json:"status,omitempty" query:"status" jsonschema:"queued, running, finished, failed or cancelled"` +} + +// CrawlIDInput names one crawl. +type CrawlIDInput struct { + ID uint `json:"id" path:"id" jsonschema:"the crawl id"` +} + +// DeleteCrawlsInput is the query of DELETE /_seo/api/v1/crawls. +type DeleteCrawlsInput struct { + IDs string `json:"ids" query:"ids" jsonschema:"the crawl ids to delete, comma-separated, e.g. 3,4; a queued or running crawl refuses the whole call"` +} + +// ListCrawls serves GET /_seo/api/v1/crawls: the crawls, newest first, +// with their issue counts and rendered comparison. +func ListCrawls(service func() *classes.Service) http.HandlerFunc { + return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { + db, err := read(ctx, svc) + if err != nil { + return 0, nil, err + } + in := listParams(r) + q := db.Model(&models.Crawl{}).Order("id DESC") + if s := strings.TrimSpace(r.URL.Query().Get("status")); s != "" { + q = q.Where("status = ?", s) + } + var rows []models.Crawl + total, err := paginate(q, in, &rows) + if err != nil { + return 0, nil, err + } + ids := make([]uint, 0, len(rows)) + for _, c := range rows { + ids = append(ids, c.ID) + } + counts := map[uint]int{} + if len(ids) > 0 { + var cs []struct { + CrawlID uint + N int + } + if err := db.Raw(`SELECT crawl_id, COUNT(*) AS n FROM golem15_seo_issues WHERE crawl_id IN ? GROUP BY crawl_id`, ids).Scan(&cs).Error; err != nil { + return 0, nil, err + } + for _, c := range cs { + counts[c.CrawlID] = c.N + } + } + out := make([]map[string]any, 0, len(rows)) + for _, c := range rows { + v := crawlMap(ctx, svc, c) + v["issues"] = counts[c.ID] + out = append(out, v) + } + return http.StatusOK, listBody(out, in, total), nil + }) +} + +// crawlMap is a crawl with its rendered comparison. +func crawlMap(ctx context.Context, svc *classes.Service, c models.Crawl) map[string]any { + v := crawlView(c) + return map[string]any{ + "id": v.ID, "status": v.Status, "trigger": v.Trigger, "trigger_name": v.TriggerName, "site": v.Site, + "max_pages": v.MaxPages, "pages": v.Pages, "limit_hit": v.LimitHit, "error": v.Error, + "started_at": v.StartedAt, "finished_at": v.FinishedAt, "created_at": v.CreatedAt, + "cancelled_by": c.CancelledBy, "render": renderView(ctx, svc, c), + } +} + +// ShowCrawl serves GET /_seo/api/v1/crawls/{id}: one crawl with its +// rendered comparison, whether the staging lock's noindex was left out, +// and its issues per kind and severity. +func ShowCrawl(service func() *classes.Service) http.HandlerFunc { + return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { + id, err := pathID(r, "id") + if err != nil { + return 0, nil, err + } + db, err := read(ctx, svc) + if err != nil { + return 0, nil, err + } + c, err := crawl.LoadCrawl(db, id) + if err != nil { + return 0, nil, err + } + var kinds []struct { + Kind string `json:"kind"` + Severity string `json:"severity"` + Count int `json:"count"` + } + if err := db.Raw(`SELECT kind, severity, COUNT(*) AS count FROM golem15_seo_issues WHERE crawl_id = ? GROUP BY kind, severity ORDER BY MIN(rank), COUNT(*) DESC, kind`, id).Scan(&kinds).Error; err != nil { + return 0, nil, err + } + out := crawlMap(ctx, svc, c) + out["staging"] = svc.Staging() + bySeverity := map[string]int{} + total := 0 + for _, k := range kinds { + bySeverity[k.Severity] += k.Count + total += k.Count + } + out["issues"] = total + out["severities"] = bySeverity + out["kinds"] = kinds + return http.StatusOK, map[string]any{"data": out}, nil + }) +} + +// CancelCrawl serves POST /_seo/api/v1/crawls/{id}/cancel: a queued or +// running crawl stops and keeps what it fetched; another answers 409 +// crawls.not_running. +func CancelCrawl(service func() *classes.Service) http.HandlerFunc { + return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { + id, err := pathID(r, "id") + if err != nil { + return 0, nil, err + } + var row models.Crawl + err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error { + c, err := crawl.LoadCrawl(tx, id) + if err != nil { + return err + } + _, name, _ := svc.Actor(ctx, tx) + if err := crawl.Cancel(ctx, svc, tx, c, name); err != nil { + return err + } + row, err = crawl.LoadCrawl(tx, id) + return err + }) + if errors.Is(err, crawl.ErrNotRunning) { + return 0, nil, Fail(http.StatusConflict, "crawls.not_running") + } + if err != nil { + return 0, nil, err + } + return http.StatusOK, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.cancelled", nil)}, nil + }) +} + +// DeleteCrawls serves DELETE /_seo/api/v1/crawls?ids=…: the crawls go with +// their pages and issues; a queued or running one refuses the whole call +// with 403 crawls.delete_running. +func DeleteCrawls(service func() *classes.Service) http.HandlerFunc { + return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { + ids, err := idList(r.URL.Query().Get("ids")) + if err != nil { + return 0, nil, err + } + n := 0 + err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error { + var err error + n, err = crawl.DeleteCrawls(tx, ids) + return err + }) + if errors.Is(err, crawl.ErrDeleteRunning) { + return 0, nil, &apiError{Status: http.StatusForbidden, Body: map[string]any{ + "error": "crawls.delete_running", + "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.delete_running", nil), + }} + } + if err != nil { + return 0, nil, err + } + return http.StatusOK, map[string]any{"deleted": n}, nil + }) +} + +// idList reads a comma-separated id list of at most 500 ids. +func idList(raw string) ([]uint, error) { + var out []uint + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + n, err := strconv.ParseUint(part, 10, 64) + if err != nil || n == 0 { + return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid") + } + out = append(out, uint(n)) + } + if len(out) == 0 || len(out) > 500 { + return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid") + } + return out, nil +} diff --git a/controllers/api/issues.go b/controllers/api/issues.go index 08ea0db..482c10d 100644 --- a/controllers/api/issues.go +++ b/controllers/api/issues.go @@ -2,6 +2,7 @@ package api import ( "context" + "errors" "net/http" "strconv" "strings" @@ -15,7 +16,23 @@ import ( // ListIssuesInput is the query of GET /_seo/api/v1/issues. type ListIssuesInput struct { - Kind string `json:"kind,omitempty" query:"kind" jsonschema:"one kind only, e.g. h1_multiple, broken_link, missing_title, redirect_chain or sitemap_blocked"` + Kind string `json:"kind,omitempty" query:"kind" jsonschema:"one kind only, e.g. h1_multiple, broken_link, missing_title, content_js, orphan_page, redirect_chain or sitemap_blocked"` + Severity string `json:"severity,omitempty" query:"severity" jsonschema:"error, warning or notice"` + Source string `json:"source,omitempty" query:"source" jsonschema:"crawl (found by the crawler) or cms (CMS data checks)"` + Fix string `json:"fix,omitempty" query:"fix" jsonschema:"tool (fixable through the MCP tool the issue names) or code (needs a change in the site's templates)"` + Status string `json:"status,omitempty" query:"status" jsonschema:"open (the default), ignored or all"` + URL string `json:"url,omitempty" query:"url" jsonschema:"only issues of this address (a path such as /blog/post)"` + Crawl uint `json:"crawl,omitempty" query:"crawl" jsonschema:"the issues of this crawl id instead of the current ones"` +} + +// IssueIDInput names one issue. +type IssueIDInput struct { + ID uint `json:"id" path:"id" jsonschema:"the issue id"` +} + +// IssueIDsInput is the body of the ignore and unignore calls. +type IssueIDsInput struct { + IDs []uint `json:"ids" jsonschema:"the issue ids"` } // IssueView is a stored issue in an answer: the Phase 32 keys (kind, url, @@ -61,8 +78,11 @@ func issueViews(db *gorm.DB, rows []models.Issue) ([]IssueView, error) { // ListIssues serves GET /_seo/api/v1/issues (D-13, D-14): the audit's // current issues, errors first: the newest crawl's findings with the // CMS-data issues stored with it, or before any crawl the CMS-data issues -// as they are now. Each issue names the tool that fixes it, or fix code. -// counts holds the number per kind. +// as they are now (crawl selects another crawl's). Each issue names the +// tool that fixes it, or fix code. The filters are severity, kind, +// source, fix, url and status (open by default: ignored issues are left +// out; ignored or all). counts holds the number per kind before the kind +// filter. func ListIssues(service func() *classes.Service) http.HandlerFunc { return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { db, err := read(ctx, svc) @@ -86,15 +106,127 @@ func ListIssues(service func() *classes.Service) http.HandlerFunc { if err != nil { return 0, nil, err } - kind := strings.TrimSpace(q.Get("kind")) + status := strings.TrimSpace(q.Get("status")) + if status == "" { + status = models.IssueOpen + } + match := map[string]string{ + "kind": strings.TrimSpace(q.Get("kind")), "severity": strings.TrimSpace(q.Get("severity")), + "source": strings.TrimSpace(q.Get("source")), "fix": strings.TrimSpace(q.Get("fix")), + "url": strings.TrimSpace(q.Get("url")), + } counts := map[string]int{} out := make([]IssueView, 0, len(views)) for _, v := range views { + if status == models.IssueOpen && v.Ignored || status == models.IssueIgnored && !v.Ignored { + continue + } + if !matches(match["severity"], v.Severity) || !matches(match["source"], v.Source) || !matches(match["fix"], v.Fix) || !matches(match["url"], v.URL) { + continue + } counts[v.Kind]++ - if kind == "" || v.Kind == kind { + if matches(match["kind"], v.Kind) { out = append(out, v) } } return http.StatusOK, map[string]any{"data": out, "counts": counts, "total": len(out)}, nil }) } + +// matches reports whether a filter value (empty: any) matches. +func matches(filter, value string) bool { return filter == "" || filter == value } + +// RecheckIssue serves POST /_seo/api/v1/issues/{id}/recheck: the issue's +// address is fetched again through the crawler client and its issues are +// replaced; the answer has the address, the issues left and the message. +func RecheckIssue(service func() *classes.Service) http.HandlerFunc { + return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { + id, err := pathID(r, "id") + if err != nil { + return 0, nil, err + } + var res crawl.RecheckResult + err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error { + issue, err := crawl.LoadIssue(tx, id) + if err != nil { + return err + } + res, err = crawl.Recheck(ctx, svc, tx, issue) + return err + }) + if errors.Is(err, crawl.ErrNoSiteURL) { + return 0, nil, StartRefusal(ctx, svc, err) + } + if err != nil { + return 0, nil, err + } + msg := svc.PhraseCtx(ctx, "golem15.seo::lang.issues.recheck_fixed", nil) + if res.Remaining > 0 { + msg = svc.ChoiceCtx(ctx, "golem15.seo::lang.issues.rechecked", res.Remaining, nil) + } + return http.StatusOK, map[string]any{"url": res.URL, "remaining": res.Remaining, "message": msg}, nil + }) +} + +// IgnoreIssues serves POST /_seo/api/v1/issues/ignore: the issues are +// hidden in this and later crawls until they are no longer ignored. +func IgnoreIssues(service func() *classes.Service) http.HandlerFunc { + return ignoreHandler(service, true) +} + +// UnignoreIssues serves POST /_seo/api/v1/issues/unignore: the issues are +// shown in the audit again. +func UnignoreIssues(service func() *classes.Service) http.HandlerFunc { + return ignoreHandler(service, false) +} + +func ignoreHandler(service func() *classes.Service, ignore bool) http.HandlerFunc { + return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { + var in IssueIDsInput + if err := decode(r, maxBody, &in); err != nil { + return 0, nil, err + } + if len(in.IDs) == 0 || len(in.IDs) > 1000 { + return 0, nil, Fail(http.StatusUnprocessableEntity, "ids_invalid") + } + n := 0 + err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error { + var rows []models.Issue + if err := tx.Where("id IN ?", in.IDs).Find(&rows).Error; err != nil { + return err + } + if len(rows) != len(uniqueIDs(in.IDs)) { + return gorm.ErrRecordNotFound + } + var err error + if ignore { + adminID, _, _ := svc.Actor(ctx, tx) + n, err = crawl.Ignore(tx, rows, adminID) + } else { + n, err = crawl.Unignore(tx, rows) + } + return err + }) + if err != nil { + return 0, nil, err + } + key, field := "golem15.seo::lang.issues.unignored", "unignored" + if ignore { + key, field = "golem15.seo::lang.issues.ignored", "ignored" + } + return http.StatusOK, map[string]any{field: n, "message": svc.ChoiceCtx(ctx, key, n, nil)}, nil + }) +} + +// uniqueIDs drops repeated ids. +func uniqueIDs(ids []uint) []uint { + seen := map[uint]bool{} + var out []uint + for _, id := range ids { + if !seen[id] { + seen[id] = true + out = append(out, id) + } + } + return out +} diff --git a/controllers/crawls.go b/controllers/crawls.go new file mode 100644 index 0000000..dd4e808 --- /dev/null +++ b/controllers/crawls.go @@ -0,0 +1,350 @@ +package controllers + +import ( + "context" + "errors" + "fmt" + "net/url" + "sort" + "strconv" + "time" + + "git.golem15.com/golem15/sm-seo-plugin/classes" + "git.golem15.com/golem15/sm-seo-plugin/classes/crawl" + "git.golem15.com/golem15/sm-seo-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/cabana" + "git.golem15.com/golem15/summercms/modules/pact" + "gorm.io/gorm" +) + +// CrawlsControllerID is the Crawls screen (UI-SPEC C24). +const CrawlsControllerID = "golem15.seo.crawls" + +// CrawlActionCancel is the preview's Cancel crawl. +const CrawlActionCancel = "cancel" + +// errCrawlReadOnly refuses creating or editing crawls. +var errCrawlReadOnly = errors.New("golem15.seo: crawls are read-only") + +// crawlsAdminController lists the crawls, newest first, and previews one +// with P6 and Cancel while it is queued or running; the toolbar starts a +// crawl and deletes the selected ones (a running crawl refuses). +type crawlsAdminController struct { + app func() *backpack.App +} + +func (crawlsAdminController) ID() string { return CrawlsControllerID } +func (crawlsAdminController) ModelName() string { return `Golem15\SEO\Models\Crawl` } +func (crawlsAdminController) ConfigDir() string { return "controllers/crawls" } +func (crawlsAdminController) NewRecord() any { return &models.Crawl{} } +func (crawlsAdminController) RequiredPermissions() []string { + return []string{PermissionManage} +} + +// FormBeforeCreate refuses: crawls are started with Start crawl. +func (crawlsAdminController) FormBeforeCreate(context.Context, any) error { return errCrawlReadOnly } + +// FormBeforeUpdate refuses: crawls are never edited. +func (crawlsAdminController) FormBeforeUpdate(context.Context, any) error { return errCrawlReadOnly } + +// FormBeforeDelete refuses a queued or running crawl with +// crawls.delete_running (403). +func (c crawlsAdminController) FormBeforeDelete(ctx context.Context, model any) error { + row, ok := model.(*models.Crawl) + if ok && row != nil && row.Active() { + return &cabana.ForbiddenError{Message: Phrase(ctx, translator(c.app), "golem15.seo::lang.crawls.delete_running", nil)} + } + return nil +} + +func (c crawlsAdminController) service() *classes.Service { + svc, err := classes.NewService(appOf(c.app)) + if err != nil { + return nil + } + return svc +} + +// AdminActions is the toolbar's Start crawl. +func (c crawlsAdminController) AdminActions() []pact.AdminAction { + return []pact.AdminAction{startCrawl(c.app)} +} + +// AdminRecordActions is Cancel crawl while queued or running; the pages +// and issues fetched so far are kept. +func (c crawlsAdminController) AdminRecordActions() []pact.AdminRecordAction { + return []pact.AdminRecordAction{{ + Name: CrawlActionCancel, + Label: "golem15.seo::lang.crawls.cancel", + Confirm: "golem15.seo::lang.crawls.cancel_confirm", + Permissions: []string{PermissionManage}, + Applies: func(_ context.Context, record any) (bool, error) { + row, ok := record.(*models.Crawl) + return ok && row != nil && row.Active(), nil + }, + Run: func(ctx context.Context, in pact.AdminRecordActionInput) (pact.AdminRecordActionResult, error) { + row, ok := in.Record.(*models.Crawl) + svc := c.service() + if !ok || row == nil || svc == nil { + return pact.AdminRecordActionResult{}, classes.ErrNotBooted + } + db, err := database(ctx, c.app) + if err != nil { + return pact.AdminRecordActionResult{}, err + } + _, name, _ := svc.Actor(ctx, db) + if err := crawl.Cancel(ctx, svc, db, *row, name); err != nil { + return pact.AdminRecordActionResult{}, err + } + return pact.AdminRecordActionResult{Message: "golem15.seo::lang.crawls.cancelled"}, nil + }, + }} +} + +// statusKeys are the phrase keys of the crawl states. +var statusKeys = map[string]string{ + models.CrawlQueued: "golem15.seo::lang.crawls.st_queued", + models.CrawlRunning: "golem15.seo::lang.crawls.st_running", + models.CrawlFinished: "golem15.seo::lang.crawls.st_finished", + models.CrawlFailed: "golem15.seo::lang.crawls.st_failed", + models.CrawlCancelled: "golem15.seo::lang.crawls.st_cancelled", +} + +// TriggerLabel is the Started by cell: Manual: name, Schedule or MCP: +// client. +func TriggerLabel(ctx context.Context, svc *classes.Service, row models.Crawl) string { + switch row.Trigger { + case models.TriggerSchedule: + return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.by_schedule", nil) + case models.TriggerMCP: + return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.by_mcp", map[string]string{"client": row.TriggerName}) + } + return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.by_admin", map[string]string{"name": row.TriggerName}) +} + +// Duration is a crawl's duration, e.g. "4 min 12 s"; "—" before it ends. +func Duration(row models.Crawl) string { + if row.StartedAt == nil { + return "—" + } + end := time.Now().UTC() + if row.FinishedAt != nil { + end = *row.FinishedAt + } + d := end.Sub(*row.StartedAt).Round(time.Second) + if d < 0 { + d = 0 + } + h, m, s := int(d.Hours()), int(d.Minutes())%60, int(d.Seconds())%60 + switch { + case h > 0: + return fmt.Sprintf("%d h %d min", h, m) + case m > 0: + return fmt.Sprintf("%d min %d s", m, s) + } + return fmt.Sprintf("%d s", s) +} + +// issueCounts counts the issues of crawls. +func issueCounts(db *gorm.DB, ids []uint) (map[uint]int, error) { + out := map[uint]int{} + if len(ids) == 0 { + return out, nil + } + var rows []struct { + CrawlID uint + N int + } + if err := db.Raw(`SELECT crawl_id, COUNT(*) AS n FROM golem15_seo_issues WHERE crawl_id IN ? GROUP BY crawl_id`, ids).Scan(&rows).Error; err != nil { + return nil, err + } + for _, r := range rows { + out[r.CrawlID] = r.N + } + return out, nil +} + +// ListExtendRecords shows the state, pages (with the limit note), issue +// count, trigger and duration of each crawl in the request locale. +func (c crawlsAdminController) ListExtendRecords(ctx context.Context, db *gorm.DB, records []any) error { + svc := c.service() + if svc == nil { + return nil + } + var ids []uint + for _, r := range records { + if row, ok := r.(*models.Crawl); ok && row != nil { + ids = append(ids, row.ID) + } + } + counts, err := issueCounts(db.WithContext(ctx), ids) + if err != nil { + return err + } + for _, r := range records { + row, ok := r.(*models.Crawl) + if !ok || row == nil { + continue + } + row.PagesCell = strconv.Itoa(row.Pages) + if row.LimitHit { + row.PagesCell = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.pages_limit", map[string]string{"count": strconv.Itoa(row.Pages)}) + } + row.IssuesCell = strconv.Itoa(counts[row.ID]) + row.DurationCell = Duration(*row) + row.TriggerCell = TriggerLabel(ctx, svc, *row) + if key, ok := statusKeys[row.Status]; ok { + row.Status = svc.PhraseCtx(ctx, key, nil) + } + } + return nil +} + +// DropdownOptions labels the preview's read-only status and trigger. +func (crawlsAdminController) DropdownOptions(field string) []pact.Option { + switch field { + case "status": + out := make([]pact.Option, 0, len(models.CrawlStatuses)) + for _, s := range models.CrawlStatuses { + out = append(out, pact.Option{Value: s, Label: statusKeys[s]}) + } + return out + case "trigger": + return []pact.Option{ + {Value: models.TriggerAdmin, Label: "golem15.seo::lang.crawls.trigger_admin"}, + {Value: models.TriggerSchedule, Label: "golem15.seo::lang.crawls.trigger_schedule"}, + {Value: models.TriggerMCP, Label: "golem15.seo::lang.crawls.trigger_mcp"}, + } + } + return nil +} + +// FilterOptions serves the status filter. +func (c crawlsAdminController) FilterOptions(scope string) []pact.Option { + if scope != models.FilterCrawlStatus { + return nil + } + return c.DropdownOptions("status") +} + +// PartialData serves P6 crawl_summary (UI-SPEC C24). +func (c crawlsAdminController) PartialData(ctx context.Context, name string, record any) (any, error) { + row, ok := record.(*models.Crawl) + svc := c.service() + if name != "crawl_summary" || !ok || row == nil || svc == nil { + return classes.CrawlSummaryView{}, nil + } + db, err := database(ctx, c.app) + if err != nil { + return nil, err + } + return CrawlSummary(ctx, svc, db, c.app, *row) +} + +// CrawlSummary builds P6: one status callout, the staging and rendered +// comparison lines, the stats strip and the per-kind table whose counts +// link to the Site audit list filtered by the kind. +func CrawlSummary(ctx context.Context, svc *classes.Service, db *gorm.DB, app func() *backpack.App, row models.Crawl) (classes.CrawlSummaryView, error) { + view := classes.CrawlSummaryView{} + p := func(key string, params map[string]string) string { + return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls."+key, params) + } + stamp := func(t *time.Time) string { + if t == nil { + return "—" + } + return t.UTC().Format("2006-01-02 15:04 UTC") + } + switch row.Status { + case models.CrawlRunning: + view.Title = p("running_title", nil) + view.Text = p("running_text", map[string]string{"done": strconv.Itoa(row.Pages), "limit": strconv.Itoa(row.MaxPages)}) + case models.CrawlQueued: + view.Text = p("queued_text", nil) + case models.CrawlFailed: + view.Tone = "danger" + view.Title = p("failed_title", nil) + view.Text = p("failed_text", map[string]string{"error": row.Error}) + case models.CrawlCancelled: + view.Text = p("cancelled_text", map[string]string{"name": row.CancelledBy, "time": stamp(row.FinishedAt)}) + case models.CrawlFinished: + if row.LimitHit { + view.Tone = "warning" + view.Text = p("limit_text", map[string]string{"limit": strconv.Itoa(row.MaxPages)}) + } + } + view.HasCallout = view.Text != "" + if svc.Staging() { + view.Staging = p("staging_text", nil) + } + switch row.RenderStatus { + case models.RenderDone: + view.Render = svc.ChoiceCtx(ctx, "golem15.seo::lang.crawls.render_done", row.RenderPages, nil) + case models.RenderFailed: + view.Render = p("render_failed", map[string]string{"error": row.RenderNote}) + default: + view.Render = p("render_skipped", nil) + } + var rows []struct { + Kind string + Severity string + N int + } + if err := db.Raw(`SELECT kind, severity, COUNT(*) AS n FROM golem15_seo_issues WHERE crawl_id = ? GROUP BY kind, severity`, row.ID).Scan(&rows).Error; err != nil { + return view, err + } + sevCount := map[string]int{} + prefix, err := cabana.AdminPrefix(appOf(app)) + if err != nil { + prefix = cabana.DefaultAdminPrefix + } + for _, r := range rows { + sevCount[r.Severity] += r.N + view.Kinds = append(view.Kinds, classes.CrawlKindRow{ + Kind: r.Kind, + Label: svc.PhraseCtx(ctx, "golem15.seo::lang.issues.kind_"+r.Kind, nil), + Severity: svc.PhraseCtx(ctx, labelKeys[r.Severity], nil), + Count: r.N, + Link: prefix + "/golem15/seo/issues?" + url.Values{"filter[kind]": {r.Kind}}.Encode(), + }) + } + rank := map[string]int{} + for _, r := range rows { + rank[r.Kind] = models.SeverityRank(r.Severity) + } + sort.SliceStable(view.Kinds, func(i, j int) bool { + a, b := view.Kinds[i], view.Kinds[j] + if rank[a.Kind] != rank[b.Kind] { + return rank[a.Kind] < rank[b.Kind] + } + if a.Count != b.Count { + return a.Count > b.Count + } + return a.Kind < b.Kind + }) + view.NoIssues = len(view.Kinds) == 0 && !row.Active() + view.Stats = []classes.CrawlStat{ + {Label: p("stat_pages", nil), Value: strconv.Itoa(row.Pages)}, + {Label: p("stat_errors", nil), Value: strconv.Itoa(sevCount[models.SeverityError])}, + {Label: p("stat_warnings", nil), Value: strconv.Itoa(sevCount[models.SeverityWarning])}, + {Label: p("stat_notices", nil), Value: strconv.Itoa(sevCount[models.SeverityNotice])}, + {Label: p("stat_duration", nil), Value: Duration(row)}, + } + return view, nil +} + +var ( + _ pact.AdminController = crawlsAdminController{} + _ pact.AdminPermissioned = crawlsAdminController{} + _ pact.AdminRecordSource = crawlsAdminController{} + _ pact.FormBeforeCreate = crawlsAdminController{} + _ pact.FormBeforeUpdate = crawlsAdminController{} + _ pact.FormBeforeDelete = crawlsAdminController{} + _ pact.HasAdminActions = crawlsAdminController{} + _ pact.HasAdminRecordActions = crawlsAdminController{} + _ pact.ListExtendRecords = crawlsAdminController{} + _ pact.DropdownOptionsProvider = crawlsAdminController{} + _ pact.FilterOptions = crawlsAdminController{} + _ pact.AdminPartialData = crawlsAdminController{} +) diff --git a/controllers/crawls/_crawl_summary.htm b/controllers/crawls/_crawl_summary.htm new file mode 100644 index 0000000..835e2d7 --- /dev/null +++ b/controllers/crawls/_crawl_summary.htm @@ -0,0 +1,31 @@ +{{- with .Data -}} +{{- if .HasCallout -}} +
+{{- if .Title -}}

{{ .Title }}

{{- end -}} +

{{ .Text }}

+
+{{- end -}} +{{- if .Staging -}}

{{ .Staging }}

{{- end -}} +{{- if .Render -}}

{{ .Render }}

{{- end -}} +{{- if .Stats -}} +
+{{- range .Stats -}} +
{{ .Label }}
{{ .Value }}
+{{- end -}} +
+{{- end -}} +{{- if .Kinds -}} +
+ + + +{{- range .Kinds -}} + +{{- end -}} + +
{{ trans "golem15.seo::lang.crawls.kinds_problem" }}{{ trans "golem15.seo::lang.crawls.kinds_severity" }}{{ trans "golem15.seo::lang.crawls.kinds_count" }}
{{ .Label }}{{ .Severity }}{{ .Count }}
+
+{{- else if .NoIssues -}} +

{{ trans "golem15.seo::lang.crawls.no_issues" }}

+{{- end -}} +{{- end -}} diff --git a/controllers/crawls/config_filter.yaml b/controllers/crawls/config_filter.yaml new file mode 100644 index 0000000..248c3d6 --- /dev/null +++ b/controllers/crawls/config_filter.yaml @@ -0,0 +1,11 @@ +# Crawls filters (UI-SPEC C24): status and start date. +scopes: + status: + label: golem15.seo::lang.crawls.status + modelClass: Golem15\SEO\Models\Crawl + nameFrom: status + scope: FilterCrawlStatus + started_at: + label: golem15.seo::lang.crawls.started_at + type: daterange + column: started_at diff --git a/controllers/crawls/config_form.yaml b/controllers/crawls/config_form.yaml new file mode 100644 index 0000000..8e44ef8 --- /dev/null +++ b/controllers/crawls/config_form.yaml @@ -0,0 +1,14 @@ +# Crawl preview (UI-SPEC C24): read-only, P6 above the fields and Cancel +# while the crawl is queued or running. +name: golem15.seo::lang.crawls.name +form: ~/plugins/golem15/seo/models/crawl/fields.yaml +modelClass: Golem15\SEO\Models\Crawl +defaultRedirect: golem15/seo/crawls + +preview: + headerPartial: crawl_summary + +recordActions: [cancel] + +messages: + preview: golem15.seo::lang.crawls.preview_title diff --git a/controllers/crawls/config_list.yaml b/controllers/crawls/config_list.yaml new file mode 100644 index 0000000..052ee8e --- /dev/null +++ b/controllers/crawls/config_list.yaml @@ -0,0 +1,22 @@ +# Crawls (UI-SPEC C24): newest first; the toolbar starts a crawl and +# deletes the selected ones (a running crawl refuses). +list: ~/plugins/golem15/seo/models/crawl/columns.yaml +modelClass: Golem15\SEO\Models\Crawl +title: golem15.seo::lang.crawls.list_title +recordUrl: golem15/seo/crawls/preview/:id +noRecordsMessage: golem15.seo::lang.crawls.list_empty +recordsPerPage: 20 +showCheckboxes: true +showSetup: true +filter: config_filter.yaml + +defaultSort: + column: started_at + direction: desc + +toolbar: + buttons: [start_crawl, delete] + +messages: + empty: golem15.seo::lang.crawls.list_empty + deleteConfirm: golem15.seo::lang.crawls.delete_selected_confirm diff --git a/controllers/issues.go b/controllers/issues.go new file mode 100644 index 0000000..db2b215 --- /dev/null +++ b/controllers/issues.go @@ -0,0 +1,505 @@ +package controllers + +import ( + "context" + "errors" + "fmt" + "strings" + + "git.golem15.com/golem15/sm-seo-plugin/classes" + "git.golem15.com/golem15/sm-seo-plugin/classes/crawl" + "git.golem15.com/golem15/sm-seo-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/cabana" + "git.golem15.com/golem15/summercms/modules/lagoon" + "git.golem15.com/golem15/summercms/modules/pact" + "gorm.io/gorm" +) + +// IssuesControllerID is the Site audit (UI-SPEC C23). +const IssuesControllerID = "golem15.seo.issues" + +// The Site audit's actions: the toolbar's start, the record actions and +// the bulk ignore. +const ( + ActionStartCrawl = "start_crawl" + IssueActionCheck = "recheck" + IssueActionIgnore = "ignore" + IssueActionShow = "unignore" +) + +// errIssueReadOnly refuses creating or editing issues: the audit writes +// them. +var errIssueReadOnly = errors.New("golem15.seo: issues are read-only") + +// issuesAdminController lists the audit's current issues, errors first, +// and previews one with P7 and its record actions (check again, ignore, +// stop ignoring); the toolbar starts a crawl and the bulk action ignores. +type issuesAdminController struct { + app func() *backpack.App +} + +func (issuesAdminController) ID() string { return IssuesControllerID } +func (issuesAdminController) ModelName() string { return `Golem15\SEO\Models\Issue` } +func (issuesAdminController) ConfigDir() string { return "controllers/issues" } +func (issuesAdminController) NewRecord() any { return &models.Issue{} } +func (issuesAdminController) RequiredPermissions() []string { + return []string{PermissionManage} +} + +// FormBeforeCreate refuses: the audit writes issues. +func (issuesAdminController) FormBeforeCreate(context.Context, any) error { return errIssueReadOnly } + +// FormBeforeUpdate refuses: issues are never edited. +func (issuesAdminController) FormBeforeUpdate(context.Context, any) error { return errIssueReadOnly } + +func (c issuesAdminController) service() *classes.Service { + svc, err := classes.NewService(appOf(c.app)) + if err != nil { + return nil + } + return svc +} + +// ListExtendQuery narrows the list to the current issues (the newest +// crawl's; before any crawl the CMS-data issues, checked now) and puts +// errors first, then warnings, then notices. +func (c issuesAdminController) ListExtendQuery(ctx context.Context, q *gorm.DB) *gorm.DB { + svc := c.service() + db, err := database(ctx, c.app) + if svc == nil || err != nil { + return q.Where("1 = 0") + } + latest, err := crawl.Current(ctx, svc, db) + if err != nil { + log.Warn("golem15.seo: site audit list", "error", err) + return q.Where("1 = 0") + } + return crawl.CurrentScope(q, latest).Order("golem15_seo_issues.rank ASC") +} + +// startCrawl is the toolbar's Start crawl (also on Crawls): it refuses with +// crawls.already_running or crawls.no_site_url as an action error and +// toasts crawls.started. +func startCrawl(app func() *backpack.App) pact.AdminAction { + return pact.AdminAction{ + Name: ActionStartCrawl, + Label: "golem15.seo::lang.crawls.start", + Permissions: []string{PermissionManage}, + Run: func(ctx context.Context, _ pact.AdminActionInput) (pact.AdminActionResult, error) { + svc, err := classes.NewService(appOf(app)) + if err != nil { + return pact.AdminActionResult{}, err + } + db, err := database(ctx, app) + if err != nil { + return pact.AdminActionResult{}, err + } + err = lagoon.Transaction(ctx, db, func(ctx context.Context, tx *gorm.DB) error { + _, name, _ := svc.Actor(ctx, tx) + _, err := crawl.Start(ctx, svc, tx, models.TriggerAdmin, name) + return err + }) + if err != nil { + return pact.AdminActionResult{}, startRefusal(ctx, app, err) + } + return pact.AdminActionResult{Message: "golem15.seo::lang.crawls.started"}, nil + }, + } +} + +// startRefusal turns a refused start into the admin's action error. +func startRefusal(ctx context.Context, app func() *backpack.App, err error) error { + tr := translator(app) + var running *crawl.AlreadyRunningError + switch { + case errors.As(err, &running): + return &cabana.ForbiddenError{Message: Phrase(ctx, tr, "golem15.seo::lang.crawls.already_running", map[string]string{"time": running.Since().UTC().Format("2006-01-02 15:04 UTC")})} + case errors.Is(err, crawl.ErrNoSiteURL): + return &cabana.ForbiddenError{Message: Phrase(ctx, tr, "golem15.seo::lang.crawls.no_site_url", nil)} + } + return err +} + +// AdminActions is the toolbar's Start crawl. +func (c issuesAdminController) AdminActions() []pact.AdminAction { + return []pact.AdminAction{startCrawl(c.app)} +} + +// ignored reports whether a record is an ignored issue. +func (c issuesAdminController) ignored(ctx context.Context, record any) (bool, error) { + issue, ok := record.(*models.Issue) + if !ok || issue == nil { + return false, nil + } + db, err := database(ctx, c.app) + if err != nil { + return false, err + } + return crawl.IsIgnored(db, *issue) +} + +// AdminRecordActions are the preview's Check this address again and Ignore +// (while open) and Stop ignoring (while ignored). +func (c issuesAdminController) AdminRecordActions() []pact.AdminRecordAction { + open := func(ctx context.Context, record any) (bool, error) { + ig, err := c.ignored(ctx, record) + return !ig, err + } + return []pact.AdminRecordAction{ + { + Name: IssueActionCheck, + Label: "golem15.seo::lang.issues.recheck", + Confirm: "golem15.seo::lang.issues.recheck_confirm", + Permissions: []string{PermissionManage}, + Applies: open, + Run: func(ctx context.Context, in pact.AdminRecordActionInput) (pact.AdminRecordActionResult, error) { + issue, ok := in.Record.(*models.Issue) + svc := c.service() + if !ok || issue == nil || svc == nil { + return pact.AdminRecordActionResult{}, classes.ErrNotBooted + } + db, err := database(ctx, c.app) + if err != nil { + return pact.AdminRecordActionResult{}, err + } + res, err := crawl.Recheck(ctx, svc, db, *issue) + if err != nil { + if errors.Is(err, crawl.ErrNoSiteURL) { + return pact.AdminRecordActionResult{}, &cabana.ForbiddenError{Message: Phrase(ctx, translator(c.app), "golem15.seo::lang.crawls.no_site_url", nil)} + } + return pact.AdminRecordActionResult{}, err + } + return pact.AdminRecordActionResult{Message: RecheckMessage(ctx, svc, res)}, nil + }, + }, + { + Name: IssueActionIgnore, + Label: "golem15.seo::lang.issues.ignore", + Confirm: "golem15.seo::lang.issues.ignore_confirm", + Permissions: []string{PermissionManage}, + Applies: open, + Run: func(ctx context.Context, in pact.AdminRecordActionInput) (pact.AdminRecordActionResult, error) { + return c.ignore(ctx, []any{in.Record}) + }, + }, + { + Name: IssueActionShow, + Label: "golem15.seo::lang.issues.unignore", + Confirm: "golem15.seo::lang.issues.unignore_confirm", + Permissions: []string{PermissionManage}, + Applies: c.ignored, + Run: func(ctx context.Context, in pact.AdminRecordActionInput) (pact.AdminRecordActionResult, error) { + issue, ok := in.Record.(*models.Issue) + if !ok || issue == nil { + return pact.AdminRecordActionResult{}, classes.ErrNotBooted + } + db, err := database(ctx, c.app) + if err != nil { + return pact.AdminRecordActionResult{}, err + } + n, err := crawl.Unignore(db, []models.Issue{*issue}) + if err != nil { + return pact.AdminRecordActionResult{}, err + } + svc := c.service() + return pact.AdminRecordActionResult{Message: svc.ChoiceCtx(ctx, "golem15.seo::lang.issues.unignored", n, nil)}, nil + }, + }, + } +} + +// RecheckMessage is the toast of a recheck: issues.rechecked with the +// remaining count, or issues.recheck_fixed. +func RecheckMessage(ctx context.Context, svc *classes.Service, res crawl.RecheckResult) string { + if res.Remaining == 0 { + return svc.PhraseCtx(ctx, "golem15.seo::lang.issues.recheck_fixed", nil) + } + return svc.ChoiceCtx(ctx, "golem15.seo::lang.issues.rechecked", res.Remaining, nil) +} + +// ignore stores the fingerprints of records as ignored. +func (c issuesAdminController) ignore(ctx context.Context, records []any) (pact.AdminRecordActionResult, error) { + svc := c.service() + db, err := database(ctx, c.app) + if err != nil || svc == nil { + return pact.AdminRecordActionResult{}, errors.Join(err, classes.ErrNotBooted) + } + var list []models.Issue + for _, r := range records { + if issue, ok := r.(*models.Issue); ok && issue != nil { + list = append(list, *issue) + } + } + adminID, _, _ := svc.Actor(ctx, db) + n, err := crawl.Ignore(db, list, adminID) + if err != nil { + return pact.AdminRecordActionResult{}, err + } + return pact.AdminRecordActionResult{Message: svc.ChoiceCtx(ctx, "golem15.seo::lang.issues.ignored", n, nil)}, nil +} + +// AdminBulkActions is the list's bulk Ignore issues. +func (c issuesAdminController) AdminBulkActions() []pact.AdminBulkAction { + return []pact.AdminBulkAction{{ + Name: IssueActionIgnore, + Label: "golem15.seo::lang.issues.ignore_selected", + Confirm: "golem15.seo::lang.issues.ignore_selected_confirm", + Permissions: []string{PermissionManage}, + Run: func(ctx context.Context, in pact.AdminBulkActionInput) (pact.AdminBulkActionResult, error) { + res, err := c.ignore(ctx, in.Records) + if err != nil { + return pact.AdminBulkActionResult{}, err + } + return pact.AdminBulkActionResult{Message: res.Message, Affected: len(in.Records)}, nil + }, + }} +} + +// ListRowStates marks ignored issues disabled (issues.state_ignored). +func (issuesAdminController) ListRowStates(ctx context.Context, db *gorm.DB, records []any) ([][]pact.RowState, error) { + var fps []string + for _, r := range records { + if issue, ok := r.(*models.Issue); ok && issue != nil { + fps = append(fps, issue.Fingerprint) + } + } + set, err := crawl.IgnoredSet(db.WithContext(ctx), fps) + if err != nil { + return nil, err + } + out := make([][]pact.RowState, len(records)) + for i, r := range records { + if issue, ok := r.(*models.Issue); ok && issue != nil && set[issue.Fingerprint] { + out[i] = []pact.RowState{pact.RowStateDisabled} + } + } + return out, nil +} + +// labelKeys are the phrase keys of the stored severity and source values. +var labelKeys = map[string]string{ + models.SeverityError: "golem15.seo::lang.issues.sev_error", + models.SeverityWarning: "golem15.seo::lang.issues.sev_warning", + models.SeverityNotice: "golem15.seo::lang.issues.sev_notice", + models.SourceCrawl: "golem15.seo::lang.issues.source_crawl", + models.SourceCMS: "golem15.seo::lang.issues.source_cms", +} + +// FixCell is the Fix column: issues.fix_tool with the tool, or +// issues.fix_code. +func FixCell(ctx context.Context, svc *classes.Service, issue models.Issue) string { + if issue.Fix == models.FixTool { + return svc.PhraseCtx(ctx, "golem15.seo::lang.issues.fix_tool", map[string]string{"tool": issue.Tool}) + } + return svc.PhraseCtx(ctx, "golem15.seo::lang.issues.fix_code", nil) +} + +// ListExtendRecords shows the severity, kind and source labels, the +// one-line summary and the fix route in the request locale. +func (c issuesAdminController) ListExtendRecords(ctx context.Context, _ *gorm.DB, records []any) error { + svc := c.service() + if svc == nil { + return nil + } + for _, r := range records { + issue, ok := r.(*models.Issue) + if !ok || issue == nil { + continue + } + issue.Summary = svc.IssueSummary(ctx, issue.Kind, issue.DetailsMap()) + issue.FixCell = FixCell(ctx, svc, *issue) + issue.KindCell = svc.PhraseCtx(ctx, "golem15.seo::lang.issues.kind_"+issue.Kind, nil) + if key, ok := labelKeys[issue.Severity]; ok { + issue.Severity = svc.PhraseCtx(ctx, key, nil) + } + if key, ok := labelKeys[issue.Source]; ok { + issue.Source = svc.PhraseCtx(ctx, key, nil) + } + } + return nil +} + +// DropdownOptions labels the preview's read-only kind, severity and +// source fields. +func (issuesAdminController) DropdownOptions(field string) []pact.Option { + switch field { + case "kind": + out := make([]pact.Option, 0, len(crawl.Kinds())) + for _, k := range crawl.Kinds() { + out = append(out, pact.Option{Value: k, Label: "golem15.seo::lang.issues.kind_" + k}) + } + return out + case "severity": + return []pact.Option{ + {Value: models.SeverityError, Label: labelKeys[models.SeverityError]}, + {Value: models.SeverityWarning, Label: labelKeys[models.SeverityWarning]}, + {Value: models.SeverityNotice, Label: labelKeys[models.SeverityNotice]}, + } + case "source": + return []pact.Option{ + {Value: models.SourceCrawl, Label: labelKeys[models.SourceCrawl]}, + {Value: models.SourceCMS, Label: labelKeys[models.SourceCMS]}, + } + } + return nil +} + +// FilterOptions serves the severity, kind, source, fix and status filters. +func (c issuesAdminController) FilterOptions(scope string) []pact.Option { + switch scope { + case models.FilterIssueSeverity: + return c.DropdownOptions("severity") + case models.FilterIssueKind: + return c.DropdownOptions("kind") + case models.FilterIssueSource: + return c.DropdownOptions("source") + case models.FilterIssueFix: + return []pact.Option{ + {Value: models.FixTool, Label: "golem15.seo::lang.issues.filter_fix_tool"}, + {Value: models.FixCode, Label: "golem15.seo::lang.issues.filter_fix_code"}, + } + case models.FilterIssueStatus: + return []pact.Option{ + {Value: models.IssueOpen, Label: "golem15.seo::lang.issues.status_open"}, + {Value: models.IssueIgnored, Label: "golem15.seo::lang.issues.status_ignored"}, + } + } + return nil +} + +// maxLinkedShown caps P7's linking pages before issues.and_more. +const maxLinkedShown = 10 + +// PartialData serves P7 issue_details (UI-SPEC C23). +func (c issuesAdminController) PartialData(ctx context.Context, name string, record any) (any, error) { + issue, ok := record.(*models.Issue) + svc := c.service() + if name != "issue_details" || !ok || issue == nil || svc == nil { + return classes.IssueDetailsView{}, nil + } + return IssueDetails(ctx, svc, c.app, *issue), nil +} + +// IssueDetails builds P7: the kind's found rows, the linking pages, the +// orphan suggestions, how to fix it and the page address. +func IssueDetails(ctx context.Context, svc *classes.Service, app func() *backpack.App, issue models.Issue) classes.IssueDetailsView { + d := issue.DetailsMap() + view := classes.IssueDetailsView{} + label := func(key string) string { return svc.PhraseCtx(ctx, "golem15.seo::lang.issues."+key, nil) } + view.Rows = append(view.Rows, classes.IssueRow{Label: label("summary"), Value: svc.IssueSummary(ctx, issue.Kind, d)}) + add := func(labelKey string, value any, mono bool) { + if value == nil { + return + } + text := fmt.Sprint(value) + if list, ok := value.([]any); ok { + parts := make([]string, 0, len(list)) + for _, e := range list { + switch x := e.(type) { + case map[string]any: + var kv []string + for _, k := range []string{"type", "property", "text", "target", "locale", "url", "status", "from", "to"} { + if v, ok := x[k]; ok { + kv = append(kv, fmt.Sprint(v)) + } + } + parts = append(parts, strings.Join(kv, " · ")) + default: + parts = append(parts, fmt.Sprint(x)) + } + } + text = strings.Join(parts, "; ") + } + if strings.TrimSpace(text) == "" { + return + } + view.Rows = append(view.Rows, classes.IssueRow{Label: label(labelKey), Value: text, Mono: mono}) + } + add("detail_h1", d["h1"], false) + add("detail_skips", d["skips"], false) + add("detail_missing", d["missing"], false) + add("detail_error", d["error"], true) + add("detail_anchors", d["anchors"], false) + add("detail_alternates", d["alternates"], true) + add("detail_images", d["images"], true) + add("detail_target", d["target"], true) + add("detail_what", d["what"], false) + if from, ok := d["from"].([]any); ok && issue.Kind != classes.IssueImageAlt { + if _, isLevel := d["to"]; !isLevel { + for i, f := range from { + if i >= maxLinkedShown { + break + } + view.LinkedFrom = append(view.LinkedFrom, classes.IssueLink{URL: fmt.Sprint(f)}) + } + total := len(from) + if n, ok := d["count"].(float64); ok && int(n) > total { + total = int(n) + } + if total > len(view.LinkedFrom) { + view.More = svc.PhraseCtx(ctx, "golem15.seo::lang.issues.and_more", map[string]string{"count": fmt.Sprint(total - len(view.LinkedFrom))}) + } + } + } + if issue.Kind == crawl.KindOrphanPage { + view.Orphan = true + if list, ok := d["suggestions"].([]any); ok { + for _, e := range list { + s, _ := e.(map[string]any) + u := fmt.Sprint(s["url"]) + view.Suggestions = append(view.Suggestions, classes.IssueLink{URL: u, Phrase: fmt.Sprint(s["phrase"])}) + } + } + } + if issue.Fix == models.FixTool { + view.FixText = svc.PhraseCtx(ctx, "golem15.seo::lang.issues.fix_tool_text", map[string]string{"tool": issue.Tool, "where": fixWhere(ctx, svc, issue)}) + } else { + what, _ := d["what"].(string) + view.FixText = svc.PhraseCtx(ctx, "golem15.seo::lang.issues.fix_code_text", map[string]string{"what": what}) + } + site := svc.SiteURL() + if strings.HasPrefix(issue.URL, "/") && site != "" { + view.PageURL = site + issue.URL + } + if strings.HasPrefix(issue.Kind, "jsonld_") && view.PageURL != "" { + view.RichResults = true + view.RichURL = "https://search.google.com/test/rich-results?url=" + view.PageURL + } + return view +} + +// fixWhere names the admin screen where a CMS tool's value is changed. +func fixWhere(ctx context.Context, svc *classes.Service, issue models.Issue) string { + key := "golem15.seo::lang.issues.where_record" + switch issue.Tool { + case crawl.ToolUpdateRobots: + key = "golem15.seo::lang.issues.where_robots" + case crawl.ToolUpdateSettings: + key = "golem15.seo::lang.issues.where_settings" + case crawl.ToolCreateRedirect, "seo_update_redirect": + key = "golem15.seo::lang.issues.where_redirects" + case crawl.ToolCreatePage, crawl.ToolUpdatePage: + key = "golem15.seo::lang.issues.where_pages" + case "seo_update_rule": + key = "golem15.seo::lang.issues.where_rules" + } + return svc.PhraseCtx(ctx, key, nil) +} + +var ( + _ pact.AdminController = issuesAdminController{} + _ pact.AdminPermissioned = issuesAdminController{} + _ pact.AdminRecordSource = issuesAdminController{} + _ pact.FormBeforeCreate = issuesAdminController{} + _ pact.FormBeforeUpdate = issuesAdminController{} + _ pact.ListExtendQuery = issuesAdminController{} + _ pact.HasAdminActions = issuesAdminController{} + _ pact.HasAdminRecordActions = issuesAdminController{} + _ pact.HasAdminBulkActions = issuesAdminController{} + _ pact.ListRowStates = issuesAdminController{} + _ pact.ListExtendRecords = issuesAdminController{} + _ pact.DropdownOptionsProvider = issuesAdminController{} + _ pact.FilterOptions = issuesAdminController{} + _ pact.AdminPartialData = issuesAdminController{} +) diff --git a/controllers/issues/_issue_details.htm b/controllers/issues/_issue_details.htm new file mode 100644 index 0000000..b0fc909 --- /dev/null +++ b/controllers/issues/_issue_details.htm @@ -0,0 +1,48 @@ +{{- with .Data -}} +{{- if .Rows -}} +

{{ trans "golem15.seo::lang.issues.found_heading" }}

+
+ + +{{- range .Rows -}} + +{{- end -}} + +
{{ .Label }}{{ if .Mono }}{{ .Value }}{{ else }}{{ .Value }}{{ end }}
+
+{{- end -}} +{{- if .LinkedFrom -}} +

{{ trans "golem15.seo::lang.issues.linked_from" }}

+
    +{{- range .LinkedFrom -}}
  • {{ .URL }}
    • {{- end -}} +
+{{- if .More -}}

{{ .More }}

{{- end -}} +{{- end -}} +{{- if .Orphan -}} +

{{ trans "golem15.seo::lang.issues.orphan_suggestions" }}

+{{- if .Suggestions -}} +
+ + + +{{- range .Suggestions -}} + +{{- end -}} + +
{{ trans "golem15.seo::lang.issues.orphan_suggestions_page" }}{{ trans "golem15.seo::lang.issues.orphan_suggestions_phrase" }}
{{ .URL }}{{ .Phrase }}
+
+{{- else -}} +

{{ trans "golem15.seo::lang.issues.orphan_none" }}

+{{- end -}} +{{- end -}} +
+

{{ trans "golem15.seo::lang.issues.fix_heading" }}

+

{{ .FixText }}

+
+{{- if .PageURL -}} +

{{ trans "golem15.seo::lang.issues.open_page" }}: {{ .PageURL }}

+{{- end -}} +{{- if .RichResults -}} +

{{ trans "golem15.seo::lang.issues.rich_results" }}: {{ .RichURL }}

+{{- end -}} +{{- end -}} diff --git a/controllers/issues/config_filter.yaml b/controllers/issues/config_filter.yaml new file mode 100644 index 0000000..640c060 --- /dev/null +++ b/controllers/issues/config_filter.yaml @@ -0,0 +1,28 @@ +# Site audit filters (UI-SPEC C23): severity, kind, source, fix and status +# (Open or Ignored). +scopes: + severity: + label: golem15.seo::lang.issues.filter_severity + modelClass: Golem15\SEO\Models\Issue + nameFrom: severity + scope: FilterIssueSeverity + kind: + label: golem15.seo::lang.issues.filter_kind + modelClass: Golem15\SEO\Models\Issue + nameFrom: kind + scope: FilterIssueKind + source: + label: golem15.seo::lang.issues.filter_source + modelClass: Golem15\SEO\Models\Issue + nameFrom: source + scope: FilterIssueSource + fix: + label: golem15.seo::lang.issues.filter_fix + modelClass: Golem15\SEO\Models\Issue + nameFrom: fix + scope: FilterIssueFix + status: + label: golem15.seo::lang.issues.filter_status + modelClass: Golem15\SEO\Models\Issue + nameFrom: url + scope: FilterIssueStatus diff --git a/controllers/issues/config_form.yaml b/controllers/issues/config_form.yaml new file mode 100644 index 0000000..ba0078f --- /dev/null +++ b/controllers/issues/config_form.yaml @@ -0,0 +1,14 @@ +# Issue preview (UI-SPEC C23): read-only, P7 above the fields and the +# record actions Check this address again, Ignore and Stop ignoring. +name: golem15.seo::lang.issues.name +form: ~/plugins/golem15/seo/models/issue/fields.yaml +modelClass: Golem15\SEO\Models\Issue +defaultRedirect: golem15/seo/issues + +preview: + headerPartial: issue_details + +recordActions: [recheck, ignore, unignore] + +messages: + preview: golem15.seo::lang.issues.preview diff --git a/controllers/issues/config_list.yaml b/controllers/issues/config_list.yaml new file mode 100644 index 0000000..37b7897 --- /dev/null +++ b/controllers/issues/config_list.yaml @@ -0,0 +1,25 @@ +# Site audit (UI-SPEC C23): the current issues, errors first (the +# controller's ListExtendQuery), then by kind; ignored issues badged. +list: ~/plugins/golem15/seo/models/issue/columns.yaml +modelClass: Golem15\SEO\Models\Issue +title: golem15.seo::lang.issues.list_title +recordUrl: golem15/seo/issues/preview/:id +noRecordsMessage: golem15.seo::lang.issues.list_empty +recordsPerPage: 50 +showCheckboxes: true +showSetup: true +filter: config_filter.yaml +bulkActions: [ignore] + +defaultSort: + column: kind + direction: asc + +toolbar: + buttons: [start_crawl] + search: + prompt: backend::lang.list.search_prompt + +messages: + empty: golem15.seo::lang.issues.list_empty + rowStateDisabled: golem15.seo::lang.issues.state_ignored diff --git a/crawl_admin_test.go b/crawl_admin_test.go new file mode 100644 index 0000000..ffda2f7 --- /dev/null +++ b/crawl_admin_test.go @@ -0,0 +1,325 @@ +package seo_test + +import ( + "encoding/json" + "fmt" + "io/fs" + "net/http" + "os" + "strings" + "testing" + + seo "git.golem15.com/golem15/sm-seo-plugin" + "git.golem15.com/golem15/sm-seo-plugin/classes/crawl" + "github.com/goccy/go-yaml" +) + +var ( + issuesAPI = adminAPI("/golem15/seo/issues") + crawlsAPI = adminAPI("/golem15/seo/crawls") +) + +// adminList is a list answer of the admin. +type adminList struct { + Data []map[string]any `json:"data"` + Meta struct { + Total int `json:"total"` + RowStates map[string][]string `json:"row_states"` + } `json:"meta"` +} + +func (e *testEnv) adminList(t *testing.T, path, token string) adminList { + t.Helper() + var out adminList + if err := json.Unmarshal(e.expect(t, http.StatusOK, http.MethodGet, path, token, nil).Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + return out +} + +// TestCrawlAdmin walks Site audit and Crawls (UI-SPEC C23, C24): the +// toolbar start and its refusals, the issue list (errors first, labels, +// fix column, ignored rows badged, the status filter), P7, check again, +// ignore and stop ignoring, the crawl list and P6 with its status callout +// and the rendered comparison line, the crawl API's render answer, cancel +// and the refused delete of a running crawl. +func TestCrawlAdmin(t *testing.T) { + env := bootCrawlEnv(t) + token := env.admin(t, true) + site := newFakeSite() + site.sitemap("/", "/p/two-h1", "/p/no-desc", "/p/lonely") + site.plant( + doc{path: "/", title: "Home", links: []string{"/p/two-h1", "/p/no-desc", "/p/gone"}}, + doc{path: "/p/two-h1", title: "Two", h1: []string{"Two", "Again"}}, + doc{path: "/p/no-desc", title: "No description", noDesc: true}, + doc{path: "/p/lonely", title: "Lonely"}, + ) + site.set("/p/gone", 404, "gone") + ctx := siteCtx(t, site) + + // Start crawl from the toolbar, then a second start refuses. + rec := env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/toolbar/start_crawl", token, map[string]any{}) + if !strings.Contains(rec.Body.String(), "Crawl started") { + t.Fatalf("start = %s", rec.Body.String()) + } + rec = env.call(t, http.MethodPost, crawlsAPI+"/toolbar/start_crawl", token, map[string]any{}) + if rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "A crawl is already running") { + t.Fatalf("second start = %d %s", rec.Code, rec.Body.String()) + } + if rec := env.call(t, http.MethodPost, "/_seo/api/v1/crawls", token, map[string]any{}); rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "crawls.already_running") { + t.Fatalf("API second start = %d %s", rec.Code, rec.Body.String()) + } + + // The queued crawl: P6 queued, render skipped line; deleting it refuses. + crawls := env.adminList(t, crawlsAPI, token) + if len(crawls.Data) != 1 { + t.Fatalf("crawls = %v", crawls.Data) + } + queuedID := fmt.Sprint(crawls.Data[0]["id"]) + p6 := env.expect(t, http.StatusOK, http.MethodGet, crawlsAPI+"/partials/crawl_summary?id="+queuedID, token, nil).Body.String() + if !strings.Contains(p6, "Waiting for a free worker") { + t.Fatalf("queued P6 = %s", p6) + } + if rec := env.call(t, http.MethodDelete, crawlsAPI+"/"+queuedID, token, nil); rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "A running crawl cannot be deleted") { + t.Fatalf("delete queued = %d %s", rec.Code, rec.Body.String()) + } + if rec := env.call(t, http.MethodDelete, "/_seo/api/v1/crawls?ids="+queuedID, token, nil); rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "crawls.delete_running") { + t.Fatalf("API delete queued = %d %s", rec.Code, rec.Body.String()) + } + + var id uint + fmt.Sscan(queuedID, &id) + env.runCrawl(t, ctx, id) + + // Site audit: errors first, labels, fix column. + issues := env.adminList(t, issuesAPI, token) + if len(issues.Data) == 0 { + t.Fatal("no issues in the Site audit list") + } + if first := issues.Data[0]; first["severity"] != "Error" { + t.Fatalf("first row = %v, want an error", first) + } + var lastRank int + for _, row := range issues.Data { + rank := map[string]int{"Error": 1, "Warning": 2, "Notice": 3}[fmt.Sprint(row["severity"])] + if rank < lastRank { + t.Fatalf("rows not errors first: %v", issues.Data) + } + lastRank = rank + } + var h1ID, descID string + for _, row := range issues.Data { + switch { + case row["kind_cell"] == "More than one H1": + h1ID = fmt.Sprint(row["id"]) + if row["fix_cell"] != "Site templates (code)" || row["summary"] != "2 H1 headings" || row["source"] != "Crawl" { + t.Fatalf("h1 row = %v", row) + } + case row["kind_cell"] == "Missing meta description": + descID = fmt.Sprint(row["id"]) + if row["fix_cell"] != "MCP: seo_set_url_seo" { + t.Fatalf("description row = %v", row) + } + } + } + if h1ID == "" || descID == "" { + t.Fatalf("rows = %v", issues.Data) + } + + // P7 of the broken link and of the orphan page. + var goneID, orphanID string + for _, row := range issues.Data { + if row["url"] == "/p/gone" { + goneID = fmt.Sprint(row["id"]) + } + if row["url"] == "/p/lonely" && row["kind_cell"] == "Orphan page (no internal links)" { + orphanID = fmt.Sprint(row["id"]) + } + } + if orphanID == "" { + t.Fatalf("no orphan row: %v", issues.Data) + } + p7 := env.expect(t, http.StatusOK, http.MethodGet, issuesAPI+"/partials/issue_details?id="+goneID, token, nil).Body.String() + for _, want := range []string{"What was found", "Linked from", "How to fix", "seo_create_redirect", "Open the page", "summer-callout"} { + if !strings.Contains(p7, want) { + t.Fatalf("P7 broken link lacks %q: %s", want, p7) + } + } + if p7 := env.expect(t, http.StatusOK, http.MethodGet, issuesAPI+"/partials/issue_details?id="+orphanID, token, nil).Body.String(); !strings.Contains(p7, "No suggestions") { + t.Fatalf("P7 orphan = %s", p7) + } + + // Ignore, row state, status filter, stop ignoring. + env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/"+h1ID+"/actions/ignore", token, map[string]any{}) + issues = env.adminList(t, issuesAPI, token) + if st := issues.Meta.RowStates[h1ID]; len(st) != 1 || st[0] != "disabled" { + t.Fatalf("ignored row states = %v", issues.Meta.RowStates) + } + open := env.adminList(t, issuesAPI+"?filter[status]=open", token) + for _, row := range open.Data { + if fmt.Sprint(row["id"]) == h1ID { + t.Fatal("an ignored issue in the open filter") + } + } + if rec := env.call(t, http.MethodPost, issuesAPI+"/"+h1ID+"/actions/ignore", token, map[string]any{}); rec.Code != http.StatusConflict { + t.Fatalf("ignore twice = %d", rec.Code) + } + env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/"+h1ID+"/actions/unignore", token, map[string]any{}) + rec = env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/bulk/ignore", token, map[string]any{"ids": []string{h1ID, descID}}) + if !strings.Contains(rec.Body.String(), `"affected":2`) { + t.Fatalf("bulk ignore = %s", rec.Body.String()) + } + env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/"+descID+"/actions/unignore", token, map[string]any{}) + + // Check again: the description is fixed on the site. + site.plant(doc{path: "/p/no-desc", title: "No description"}) + rec = env.callCtx(t, ctx, http.MethodPost, issuesAPI+"/"+descID+"/actions/recheck", token, map[string]any{}) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "no issues remain") { + t.Fatalf("recheck = %d %s", rec.Code, rec.Body.String()) + } + rec = env.callCtx(t, ctx, http.MethodPost, "/_seo/api/v1/issues/"+goneID+"/recheck", token, map[string]any{}) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"remaining":1`) || !strings.Contains(rec.Body.String(), "1 issue remains") { + t.Fatalf("API recheck of a still broken link = %d %s", rec.Code, rec.Body.String()) + } + + // Crawls: the finished crawl's P6 and the API's render answer. + crawls = env.adminList(t, crawlsAPI, token) + row := crawls.Data[0] + if row["status"] != "Finished" || !strings.HasPrefix(fmt.Sprint(row["trigger_cell"]), "Manual: ") || row["pages_cell"] == nil { + t.Fatalf("crawl row = %v", row) + } + p6 = env.expect(t, http.StatusOK, http.MethodGet, crawlsAPI+"/partials/crawl_summary?id="+queuedID, token, nil).Body.String() + for _, want := range []string{"summer-stats", "Pages fetched", "More than one H1", "filter%5Bkind%5D=h1_multiple", "comparison with the JavaScript-rendered pages was skipped", "golem15.seo.crawl_chrome_path"} { + if !strings.Contains(p6, want) { + t.Fatalf("P6 lacks %q: %s", want, p6) + } + } + got := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/crawls/"+queuedID, token, nil).Body.String() + if !jsonContains(got, `"render":{"status":"skipped","pages":0,"message":"The comparison with the JavaScript-rendered pages was skipped`) || !jsonContains(got, `"kinds":[`) { + t.Fatalf("GET crawl = %s", got) + } + + // Cancel a queued crawl: it keeps its row, P6 says who cancelled it. + env.expect(t, http.StatusOK, http.MethodPost, issuesAPI+"/toolbar/start_crawl", token, map[string]any{}) + crawls = env.adminList(t, crawlsAPI, token) + nextID := fmt.Sprint(crawls.Data[0]["id"]) + if nextID == queuedID { + t.Fatalf("crawls not newest first: %v", crawls.Data) + } + env.expect(t, http.StatusOK, http.MethodPost, crawlsAPI+"/"+nextID+"/actions/cancel", token, map[string]any{}) + p6 = env.expect(t, http.StatusOK, http.MethodGet, crawlsAPI+"/partials/crawl_summary?id="+nextID, token, nil).Body.String() + if !strings.Contains(p6, "Cancelled by") { + t.Fatalf("cancelled P6 = %s", p6) + } + var next uint + fmt.Sscan(nextID, &next) + env.runCrawl(t, ctx, next) + if c := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/crawls/"+nextID, token, nil).Body.String(); !jsonContains(c, `"status":"cancelled"`) { + t.Fatalf("cancelled crawl after its job = %s", c) + } + // The cancelled crawl is the newest finished one; the audit shows its + // (empty) result, and the earlier crawl can be deleted. + env.expect(t, http.StatusOK, http.MethodDelete, "/_seo/api/v1/crawls?ids="+queuedID, token, nil) + if list := env.adminList(t, crawlsAPI, token); len(list.Data) != 1 { + t.Fatalf("crawls after delete = %v", list.Data) + } +} + +// TestCrawlTools checks the MCP parity of the audit (D-10): the nine +// tools exist, bind to their routes and methods, the reads are GET, the +// delete is destructive, and MCP can start, follow and read a crawl and +// ignore an issue. +func TestCrawlTools(t *testing.T) { + p := &seo.Plugin{} + want := map[string]string{ + "start_crawl": "POST /_seo/api/v1/crawls", + "cancel_crawl": "POST /_seo/api/v1/crawls/{id}/cancel", + "list_crawls": "GET /_seo/api/v1/crawls", + "get_crawl": "GET /_seo/api/v1/crawls/{id}", + "delete_crawls": "DELETE /_seo/api/v1/crawls", + "list_issues": "GET /_seo/api/v1/issues", + "recheck_url": "POST /_seo/api/v1/issues/{id}/recheck", + "ignore_issues": "POST /_seo/api/v1/issues/ignore", + "unignore_issues": "POST /_seo/api/v1/issues/unignore", + } + found := 0 + for _, tool := range p.MCP().Tools { + route, ok := want[tool.Name] + if !ok { + continue + } + found++ + if got := tool.Method + " " + tool.Path; got != route { + t.Errorf("tool %s binds %s, want %s", tool.Name, got, route) + } + if tool.Name == "delete_crawls" && (tool.Destructive == nil || !*tool.Destructive) { + t.Error("delete_crawls is not destructive") + } + } + if found != len(want) { + t.Fatalf("found %d of the %d audit tools", found, len(want)) + } + if testing.Short() { + return + } + + env := bootCrawlEnv(t) + agent := env.principal(t, env.admin(t, false, "golem15.seo.manage")) + site := newFakeSite() + site.sitemap("/") + site.plant(doc{path: "/", title: "Home", h1: []string{"Home", "Twice"}}) + started := env.mcpExpect(t, http.StatusAccepted, http.MethodPost, "/_seo/api/v1/crawls", agent, map[string]any{}) + data, _ := started["data"].(map[string]any) + if data["trigger"] != "mcp" { + t.Fatalf("MCP start = %v", started) + } + var id uint + fmt.Sscan(fmt.Sprint(data["id"]), &id) + env.runCrawl(t, siteCtx(t, site), id) + list := env.mcpExpect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/crawls", agent, nil) + if rows, _ := list["data"].([]any); len(rows) != 1 { + t.Fatalf("list_crawls = %v", list) + } + issues := env.mcpExpect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/issues?kind=h1_multiple", agent, nil) + rows, _ := issues["data"].([]any) + if len(rows) != 1 { + t.Fatalf("list_issues = %v", issues) + } + issueID := rows[0].(map[string]any)["id"] + env.mcpExpect(t, http.StatusOK, http.MethodPost, "/_seo/api/v1/issues/ignore", agent, map[string]any{"ids": []any{issueID}}) + if open := env.mcpExpect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/issues?kind=h1_multiple", agent, nil); len(open["data"].([]any)) != 0 { + t.Fatalf("an ignored issue is listed as open: %v", open) + } + if ignored := env.mcpExpect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/issues?kind=h1_multiple&status=ignored", agent, nil); len(ignored["data"].([]any)) != 1 { + t.Fatalf("status=ignored = %v", ignored) + } + env.mcpExpect(t, http.StatusOK, http.MethodPost, "/_seo/api/v1/issues/unignore", agent, map[string]any{"ids": []any{issueID}}) + if rec := env.mcp(t, http.MethodPost, "/_seo/api/v1/crawls/"+fmt.Sprint(id)+"/cancel", agent, map[string]any{}); rec.Code != http.StatusConflict { + t.Fatalf("cancel a finished crawl = %d", rec.Code) + } +} + +// TestIssueKindLabels checks that every kind the audit stores has its +// issues.kind_ label and issues.sum_ Details cell in both +// phrasebooks (UI-SPEC C23). +func TestIssueKindLabels(t *testing.T) { + for _, locale := range []string{"en", "pl"} { + raw, err := fs.ReadFile(os.DirFS("lang/"+locale), "lang.yaml") + if err != nil { + t.Fatal(err) + } + var book struct { + Issues map[string]any `yaml:"issues"` + } + if err := yaml.Unmarshal(raw, &book); err != nil { + t.Fatal(err) + } + for _, kind := range crawl.Kinds() { + for _, key := range []string{"kind_" + kind, "sum_" + kind} { + if v, ok := book.Issues[key].(string); !ok || v == "" { + t.Errorf("lang/%s: issues.%s is missing", locale, key) + } + } + } + } +} diff --git a/crawl_site_test.go b/crawl_site_test.go index 3e5d68a..0544186 100644 --- a/crawl_site_test.go +++ b/crawl_site_test.go @@ -1,11 +1,14 @@ package seo_test import ( + "bytes" "context" "encoding/json" + "fmt" "html" "io" "net/http" + "net/http/httptest" "net/url" "slices" "sort" @@ -581,3 +584,28 @@ func auditRenderer(site *fakeSite) *fakeRenderer { "/p/render-h1": doc{path: "/p/render-h1"}.render(), }} } + +// callCtx runs one request with ctx (a crawler transport for a recheck). +func (e *testEnv) callCtx(t *testing.T, ctx context.Context, method, path, token string, body any) *httptest.ResponseRecorder { + t.Helper() + var reader io.Reader + if body != nil { + raw, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + reader = bytes.NewReader(raw) + } + req := httptest.NewRequest(method, path, reader).WithContext(ctx) + req.RemoteAddr = fmt.Sprintf("198.51.100.%d:4000", seq.Add(1)%250+1) + req.Header.Set("Accept-Language", "en") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + e.h.ServeHTTP(rec, req) + return rec +} diff --git a/crawler_test.go b/crawler_test.go index 84d80f6..b0aae1f 100644 --- a/crawler_test.go +++ b/crawler_test.go @@ -323,7 +323,7 @@ func TestIssueMerge(t *testing.T) { } site.plant(doc{path: "/p/fixme", title: "Fix me"}) second := env.crawlOnce(t, token, ctx) - list = env.issues(t, token, "") + list = env.issues(t, token, "status=all") again := findIssue(t, list, "heading_skipped", "/cat/news") if !again.Ignored || again.CrawlID == nil || *again.CrawlID != second { t.Fatalf("ignored issue in the next crawl = %+v", again) diff --git a/lang/en/lang.yaml b/lang/en/lang.yaml index 56c782c..d824fe8 100644 --- a/lang/en/lang.yaml +++ b/lang/en/lang.yaml @@ -2,6 +2,8 @@ plugin: name: SEO description: Search engine metadata for every content type, URL rules and global SEO settings. nav: + audit: "Site audit" + crawls: "Crawls" pages: Pages robots: robots.txt notfound: 404 log @@ -522,10 +524,142 @@ schema: currency_invalid: "Use a three-letter currency code, e.g. PLN." merged: "Saved. Your custom JSON-LD replaces :props in the generated :node node." crawls: + start: "Start crawl" + list_title: "Crawls" + name: "Crawl" + started_at: "Started" + finished_at: "Finished" + status: "Status" + pages: "Pages" + issues: "Issues" + trigger: "Started by" + duration: "Duration" + site: "Site address" + st_queued: "Queued" + st_running: "Running" + st_finished: "Finished" + st_failed: "Failed" + st_cancelled: "Cancelled" + pages_limit: ":count (limit reached)" + by_admin: "Manual: :name" + by_schedule: "Schedule" + by_mcp: "MCP: :client" + trigger_admin: "Manual" + trigger_schedule: "Schedule" + trigger_mcp: "MCP" + preview_title: "Crawl" + preview: "Crawl of :url" + running_title: "In progress" + running_text: ":done pages fetched so far, up to :limit. Reload the page to see the progress." + queued_text: "Waiting for a free worker. It starts within a minute." + failed_title: "The crawl failed" + failed_text: ":error. Pages fetched before the failure and their issues are kept." + cancelled_text: "Cancelled by :name at :time." + limit_text: "The crawl stopped at the limit of :limit pages. Raise the limit in SEO settings › Site audit to crawl the whole site." + staging_text: "On this environment every page carries noindex from the non-production lock; the crawl does not report it." + render_skipped: "The comparison with the JavaScript-rendered pages was skipped: no Chrome path is configured (golem15.seo.crawl_chrome_path). The raw-HTML checks ran." + render_done: + one: ":count page was compared with its JavaScript-rendered version." + other: ":count pages were compared with their JavaScript-rendered versions." + render_failed: "The comparison with the JavaScript-rendered pages stopped: :error. The raw-HTML checks ran." + stat_pages: "Pages fetched" + stat_errors: "Errors" + stat_warnings: "Warnings" + stat_notices: "Notices" + stat_duration: "Duration" + kinds_problem: "Problem" + kinds_severity: "Severity" + kinds_count: "Addresses" + no_issues: "This crawl found no issues." + cancel: "Cancel crawl" + cancel_confirm: "Cancel this crawl? Pages fetched so far and their issues are kept." + cancelled: "Crawl cancelled." + delete_selected_confirm: + one: "Delete :count crawl? Its pages and issues are deleted with it." + other: "Delete :count crawls? Their pages and issues are deleted with them." + delete_running: "A running crawl cannot be deleted. Cancel it first." + list_empty: "No crawls yet. A crawl fetches the site like a search engine and lists what it finds in SEO › Site audit." started: Crawl started. Follow it in SEO › Crawls; the issues appear in Site audit when it finishes. already_running: A crawl is already running (started :time). Wait for it, or cancel it in SEO › Crawls. no_site_url: The site address for crawls is not configured (golem15.seo.site_url). Ask the administrator to set it. issues: + list_title: "Site audit" + name: "Issue" + severity: "Severity" + kind: "Problem" + kind_code: "Problem code" + url: "Address" + summary: "Details" + source: "Found by" + fix: "Fix" + found: "Found" + locale: "Language" + sev_error: "Error" + sev_warning: "Warning" + sev_notice: "Notice" + source_crawl: "Crawl" + source_cms: "CMS data check" + fix_tool: "MCP: :tool" + fix_code: "Site templates (code)" + state_ignored: "Ignored" + filter_severity: "Severity" + filter_kind: "Problem" + filter_source: "Found by" + filter_fix: "Fix" + filter_status: "Status" + filter_fix_tool: "Fixable through MCP" + filter_fix_code: "Needs a code change" + status_open: "Open" + status_ignored: "Ignored" + preview: "Issue details" + found_heading: "What was found" + fix_heading: "How to fix" + fix_tool_text: "Ask your MCP client to fix it with the tool “:tool”, or change it in the admin: :where." + fix_code_text: "This comes from the site's templates, not from CMS data. Ask the site's developer to change :what." + open_page: "Open the page" + rich_results: "Check in Google's Rich Results Test" + linked_from: "Linked from" + orphan_suggestions: "Pages that could link here" + orphan_suggestions_page: "Page" + orphan_suggestions_phrase: "Shared phrase" + orphan_none: "No suggestions: no other page mentions this page's title or focus keyphrase." + and_more: "and :count more" + recheck: "Check this address again" + recheck_confirm: "Fetch “:url” again now and update its issues?" + rechecked: + one: "Checked again: :count issue remains for this address." + other: "Checked again: :count issues remain for this address." + recheck_fixed: "Checked again: no issues remain for this address." + ignore: "Ignore issue" + ignore_confirm: "Ignore this issue? It stays hidden in later crawls until you stop ignoring it." + ignore_selected: "Ignore issues" + ignore_selected_confirm: + one: "Ignore :count issue? It stays hidden in later crawls." + other: "Ignore :count issues? They stay hidden in later crawls." + ignored: + one: ":count issue is ignored." + other: ":count issues are ignored." + unignore: "Stop ignoring" + unignore_confirm: "Show this issue in the audit again?" + unignored: + one: ":count issue is shown again." + other: ":count issues are shown again." + list_empty: "No issues. Problems found by the last crawl and by the CMS data checks appear here. Start a crawl to check the site again." + detail_h1: "H1 headings" + detail_skips: "Skipped levels" + detail_missing: "Missing properties" + detail_error: "Error" + detail_anchors: "Links" + detail_alternates: "Alternates" + detail_images: "Images" + detail_target: "Target" + detail_what: "Missing or to change" + where_record: "the SEO tab of the page" + where_robots: "SEO › robots.txt" + where_settings: "SEO settings" + where_redirects: "SEO › Redirects" + where_pages: "SEO › Pages" + where_rules: "SEO › URL rules" kind_orphan_page: "Orphan page (no internal links)" sum_orphan_page: ":count suggestions" kind_h1_in_content: "H1 in the content" diff --git a/lang/pl/lang.yaml b/lang/pl/lang.yaml index fdb160e..19c793a 100644 --- a/lang/pl/lang.yaml +++ b/lang/pl/lang.yaml @@ -2,6 +2,8 @@ plugin: name: SEO description: Metadane dla wyszukiwarek we wszystkich typach treści, reguły adresów i globalne ustawienia SEO. nav: + audit: "Audyt witryny" + crawls: "Skanowania" pages: Strony robots: robots.txt notfound: Dziennik 404 @@ -536,10 +538,154 @@ schema: currency_invalid: "Użyj trzyliterowego kodu waluty, np. PLN." merged: "Zapisano. Twój JSON-LD zastępuje :props w wygenerowanym węźle :node." crawls: + start: "Uruchom skanowanie" + list_title: "Skanowania" + name: "Skanowanie" + started_at: "Rozpoczęto" + finished_at: "Zakończono" + status: "Stan" + pages: "Strony" + issues: "Problemy" + trigger: "Uruchomienie" + duration: "Czas trwania" + site: "Adres witryny" + st_queued: "W kolejce" + st_running: "W toku" + st_finished: "Zakończone" + st_failed: "Nieudane" + st_cancelled: "Anulowane" + pages_limit: ":count (osiągnięto limit)" + by_admin: "Ręcznie: :name" + by_schedule: "Harmonogram" + by_mcp: "MCP: :client" + trigger_admin: "Ręcznie" + trigger_schedule: "Harmonogram" + trigger_mcp: "MCP" + preview_title: "Skanowanie" + preview: "Skanowanie :url" + running_title: "W toku" + running_text: "Dotąd pobrano stron: :done, najwyżej :limit. Odśwież stronę, aby zobaczyć postęp." + queued_text: "Czeka na wolny proces. Rozpocznie się w ciągu minuty." + failed_title: "Skanowanie nie powiodło się" + failed_text: ":error. Strony pobrane przed błędem i ich problemy zostają zachowane." + cancelled_text: "Anulowane przez: :name, :time." + limit_text: "Skanowanie zatrzymało się na limicie :limit stron. Zwiększ limit w Ustawieniach SEO › Audyt witryny, aby przeskanować całą witrynę." + staging_text: "W tym środowisku każda strona ma noindex z blokady nieprodukcyjnej; skanowanie tego nie zgłasza." + render_skipped: "Porównanie ze stronami wyrenderowanymi przez JavaScript pominięto: nie skonfigurowano ścieżki do Chrome (golem15.seo.crawl_chrome_path). Sprawdzenia surowego HTML zostały wykonane." + render_done: + one: "Porównano :count stronę z wersją wyrenderowaną przez JavaScript." + few: "Porównano :count strony z wersją wyrenderowaną przez JavaScript." + many: "Porównano :count stron z wersją wyrenderowaną przez JavaScript." + other: "Porównano :count strony z wersją wyrenderowaną przez JavaScript." + render_failed: "Porównanie ze stronami wyrenderowanymi przez JavaScript zostało przerwane: :error. Sprawdzenia surowego HTML zostały wykonane." + stat_pages: "Pobrane strony" + stat_errors: "Błędy" + stat_warnings: "Ostrzeżenia" + stat_notices: "Uwagi" + stat_duration: "Czas trwania" + kinds_problem: "Problem" + kinds_severity: "Waga" + kinds_count: "Adresy" + no_issues: "To skanowanie nie wykryło problemów." + cancel: "Anuluj skanowanie" + cancel_confirm: "Anulować to skanowanie? Pobrane dotąd strony i ich problemy zostaną zachowane." + cancelled: "Anulowano skanowanie." + delete_selected_confirm: + one: "Usunąć :count skanowanie? Jego strony i problemy zostaną usunięte." + few: "Usunąć :count skanowania? Ich strony i problemy zostaną usunięte." + many: "Usunąć :count skanowań? Ich strony i problemy zostaną usunięte." + other: "Usunąć :count skanowania? Ich strony i problemy zostaną usunięte." + delete_running: "Nie można usunąć trwającego skanowania. Najpierw je anuluj." + list_empty: "Nie było jeszcze skanowań. Skanowanie pobiera witrynę jak wyszukiwarka i wypisuje znalezione problemy w SEO › Audyt witryny." started: Rozpoczęto skanowanie. Postęp znajdziesz w SEO › Skanowania; problemy pojawią się w Audycie witryny po jego zakończeniu. already_running: Skanowanie już trwa (rozpoczęte :time). Poczekaj na nie lub anuluj je w SEO › Skanowania. no_site_url: Adres witryny do skanowania nie jest skonfigurowany (golem15.seo.site_url). Poproś administratora o jego ustawienie. issues: + list_title: "Audyt witryny" + name: "Problem" + severity: "Waga" + kind: "Problem" + kind_code: "Kod problemu" + url: "Adres" + summary: "Szczegóły" + source: "Wykryte przez" + fix: "Naprawa" + found: "Wykryto" + locale: "Język" + sev_error: "Błąd" + sev_warning: "Ostrzeżenie" + sev_notice: "Uwaga" + source_crawl: "Skanowanie" + source_cms: "Kontrola danych CMS" + fix_tool: "MCP: :tool" + fix_code: "Szablony witryny (kod)" + state_ignored: "Zignorowany" + filter_severity: "Waga" + filter_kind: "Problem" + filter_source: "Wykryte przez" + filter_fix: "Naprawa" + filter_status: "Stan" + filter_fix_tool: "Do naprawy przez MCP" + filter_fix_code: "Wymaga zmiany w kodzie" + status_open: "Otwarte" + status_ignored: "Zignorowane" + preview: "Szczegóły problemu" + found_heading: "Co znaleziono" + fix_heading: "Jak naprawić" + fix_tool_text: "Poproś klienta MCP o naprawę narzędziem „:tool” lub zmień to w panelu: :where." + fix_code_text: "To wynika z szablonów witryny, a nie z danych CMS. Poproś dewelopera witryny o zmianę: :what." + open_page: "Otwórz stronę" + rich_results: "Sprawdź w Teście wyników z elementami rozszerzonymi Google" + linked_from: "Linkowane z" + orphan_suggestions: "Strony, które mogą tu linkować" + orphan_suggestions_page: "Strona" + orphan_suggestions_phrase: "Wspólna fraza" + orphan_none: "Brak propozycji: żadna inna strona nie wspomina tytułu ani frazy kluczowej tej strony." + and_more: "i :count więcej" + recheck: "Sprawdź ten adres ponownie" + recheck_confirm: "Pobrać teraz ponownie „:url” i zaktualizować jego problemy?" + rechecked: + one: "Sprawdzono ponownie: dla tego adresu został :count problem." + few: "Sprawdzono ponownie: dla tego adresu zostały :count problemy." + many: "Sprawdzono ponownie: dla tego adresu zostało :count problemów." + other: "Sprawdzono ponownie: dla tego adresu zostało :count problemu." + recheck_fixed: "Sprawdzono ponownie: dla tego adresu nie ma już problemów." + ignore: "Ignoruj problem" + ignore_confirm: "Zignorować ten problem? Pozostanie ukryty w kolejnych skanowaniach, dopóki nie przestaniesz go ignorować." + ignore_selected: "Ignoruj problemy" + ignore_selected_confirm: + one: "Zignorować :count problem? Pozostanie ukryty w kolejnych skanowaniach." + few: "Zignorować :count problemy? Pozostaną ukryte w kolejnych skanowaniach." + many: "Zignorować :count problemów? Pozostaną ukryte w kolejnych skanowaniach." + other: "Zignorować :count problemu? Pozostaną ukryte w kolejnych skanowaniach." + ignored: + one: "Zignorowano :count problem." + few: "Zignorowano :count problemy." + many: "Zignorowano :count problemów." + other: "Zignorowano :count problemu." + unignore: "Przestań ignorować" + unignore_confirm: "Ponownie pokazywać ten problem w audycie?" + unignored: + one: "Ponownie pokazywany jest :count problem." + few: "Ponownie pokazywane są :count problemy." + many: "Ponownie pokazywanych jest :count problemów." + other: "Ponownie pokazywane jest :count problemu." + list_empty: "Brak problemów. Tutaj pojawiają się problemy z ostatniego skanowania i z kontroli danych CMS. Uruchom skanowanie, aby ponownie sprawdzić witrynę." + detail_h1: "Nagłówki H1" + detail_skips: "Pominięte poziomy" + detail_missing: "Brakujące właściwości" + detail_error: "Błąd" + detail_anchors: "Linki" + detail_alternates: "Wersje językowe" + detail_images: "Obrazy" + detail_target: "Cel" + detail_what: "Brakuje lub do zmiany" + where_record: "zakładka SEO strony" + where_robots: "SEO › robots.txt" + where_settings: "Ustawienia SEO" + where_redirects: "SEO › Przekierowania" + where_pages: "SEO › Strony" + where_rules: "SEO › Reguły adresów" kind_orphan_page: "Strona osierocona (brak linków wewnętrznych)" sum_orphan_page: "Propozycje: :count" kind_h1_in_content: "H1 w treści" diff --git a/mcp.go b/mcp.go index 35e5532..aae416c 100644 --- a/mcp.go +++ b/mcp.go @@ -382,16 +382,97 @@ func (p *Plugin) MCP() pact.MCP { Permissions: []string{mcpPermission}, }, - // Audit (D-13, D-14 CMS part). + // Site audit and crawls (D-13, D-14, SC-3). { Name: "list_issues", Title: "List SEO issues", - Description: "Lists the CMS-data SEO issues: missing and duplicate titles and descriptions, URL rules overriding a record, redirect chains, canonicals pointing to noindex or redirected addresses, and noindex addresses robots.txt blocks; each names the tool that fixes it.", + Description: "Lists the site audit's current issues, errors first: what the newest crawl found per URL (statuses, sitemaps, canonicals, titles and descriptions, headings, structured data, hreflang, links, images, content that needs JavaScript, orphan pages) and the CMS-data checks (missing and duplicate titles, URL rules over records, redirect chains, canonicals to noindex or redirected addresses, robots.txt blocks, images without alt text, H1 in content). Each issue names the MCP tool that fixes it (fix tool), or fix code with details.what naming the template element the site's developer changes. Ignored issues are left out unless status is ignored or all.", Guard: mcpGuard, Method: http.MethodGet, Path: "/_seo/api/v1/issues", Input: apicontrollers.ListIssuesInput{}, Permissions: []string{mcpPermission}, }, + { + Name: "recheck_url", + Title: "Check an issue's address again", + Description: "Fetches the address of one issue again through the crawler and replaces that address's issues; answers the issues left for it. Use it after fixing an issue to confirm the fix.", + Guard: mcpGuard, + Method: http.MethodPost, + Path: "/_seo/api/v1/issues/{id}/recheck", + Input: apicontrollers.IssueIDInput{}, + Permissions: []string{mcpPermission}, + }, + { + Name: "ignore_issues", + Title: "Ignore SEO issues", + Description: "Ignores issues by id: they stay hidden in later crawls until seo_unignore_issues shows them again.", + Guard: mcpGuard, + Method: http.MethodPost, + Path: "/_seo/api/v1/issues/ignore", + Input: apicontrollers.IssueIDsInput{}, + Permissions: []string{mcpPermission}, + }, + { + Name: "unignore_issues", + Title: "Stop ignoring SEO issues", + Description: "Shows ignored issues in the audit again, by id.", + Guard: mcpGuard, + Method: http.MethodPost, + Path: "/_seo/api/v1/issues/unignore", + Input: apicontrollers.IssueIDsInput{}, + Permissions: []string{mcpPermission}, + }, + { + Name: "start_crawl", + Title: "Start a site crawl", + Description: "Starts a crawl of the site: the crawler fetches the site's own pages like a search engine (two requests at a time, within the Site audit limits) and stores what it finds as issues. One crawl runs at a time; a second start answers crawls.already_running. Follow it with seo_get_crawl; when it finishes, read the issues with seo_list_issues.", + Guard: mcpGuard, + Method: http.MethodPost, + Path: "/_seo/api/v1/crawls", + Input: apicontrollers.StartCrawlInput{}, + Permissions: []string{mcpPermission}, + }, + { + Name: "cancel_crawl", + Title: "Cancel a crawl", + Description: "Cancels a queued or running crawl by id; the pages fetched so far and their issues are kept.", + Guard: mcpGuard, + Method: http.MethodPost, + Path: "/_seo/api/v1/crawls/{id}/cancel", + Input: apicontrollers.CrawlIDInput{}, + Permissions: []string{mcpPermission}, + }, + { + Name: "list_crawls", + Title: "List crawls", + Description: "Lists the crawls, newest first, with their status, pages, issue count, who started them and the comparison with the JavaScript-rendered pages (render: skipped, done or failed).", + Guard: mcpGuard, + Method: http.MethodGet, + Path: "/_seo/api/v1/crawls", + Input: apicontrollers.ListCrawlsInput{}, + Permissions: []string{mcpPermission}, + }, + { + Name: "get_crawl", + Title: "Get a crawl", + Description: "Shows one crawl: status, pages fetched and whether the page limit stopped it, the error of a failed crawl, the issues per kind and severity, whether the staging lock's noindex was left out, and render {status, pages, note}: whether the pages were compared with their JavaScript-rendered versions or that comparison was skipped (no Chrome path configured) or failed.", + Guard: mcpGuard, + Method: http.MethodGet, + Path: "/_seo/api/v1/crawls/{id}", + Input: apicontrollers.CrawlIDInput{}, + Permissions: []string{mcpPermission}, + }, + { + Name: "delete_crawls", + Title: "Delete crawls", + Description: "Deletes crawls by id with their pages and issues; a queued or running crawl refuses the call (crawls.delete_running).", + Guard: mcpGuard, + Method: http.MethodDelete, + Path: "/_seo/api/v1/crawls", + Input: apicontrollers.DeleteCrawlsInput{}, + Permissions: []string{mcpPermission}, + Destructive: &destructive, + }, }} } diff --git a/models/crawl/columns.yaml b/models/crawl/columns.yaml new file mode 100644 index 0000000..954a61a --- /dev/null +++ b/models/crawl/columns.yaml @@ -0,0 +1,27 @@ +# Crawls columns (UI-SPEC C24). status shows its label; pages_cell, +# issues, trigger_cell and duration are filled by ListExtendRecords. +columns: + started_at: + label: golem15.seo::lang.crawls.started_at + type: datetime + sortable: true + status: + label: golem15.seo::lang.crawls.status + type: text + sortable: true + pages_cell: + label: golem15.seo::lang.crawls.pages + type: text + sortable: false + issues: + label: golem15.seo::lang.crawls.issues + type: text + sortable: false + trigger_cell: + label: golem15.seo::lang.crawls.trigger + type: text + sortable: false + duration: + label: golem15.seo::lang.crawls.duration + type: text + sortable: false diff --git a/models/crawl/fields.yaml b/models/crawl/fields.yaml new file mode 100644 index 0000000..a49df6d --- /dev/null +++ b/models/crawl/fields.yaml @@ -0,0 +1,29 @@ +# Crawl preview (UI-SPEC C24): every field is read-only. +fields: + started_at: + label: golem15.seo::lang.crawls.started_at + type: datepicker + mode: datetime + span: left + context: preview + finished_at: + label: golem15.seo::lang.crawls.finished_at + type: datepicker + mode: datetime + span: right + context: preview + status: + label: golem15.seo::lang.crawls.status + type: dropdown + span: left + context: preview + trigger: + label: golem15.seo::lang.crawls.trigger + type: dropdown + span: right + context: preview + site: + label: golem15.seo::lang.crawls.site + type: text + span: full + context: preview diff --git a/models/issue/columns.yaml b/models/issue/columns.yaml new file mode 100644 index 0000000..8223957 --- /dev/null +++ b/models/issue/columns.yaml @@ -0,0 +1,43 @@ +# Site audit columns (UI-SPEC C23). severity, kind_cell, summary, source +# and fix_cell are filled by the controller's ListExtendRecords. +columns: + severity: + label: golem15.seo::lang.issues.severity + type: text + sortable: false + kind_cell: + label: golem15.seo::lang.issues.kind + type: text + sortable: false + kind: + label: golem15.seo::lang.issues.kind_code + type: text + sortable: true + invisible: true + url: + label: golem15.seo::lang.issues.url + type: text + searchable: true + sortable: true + summary: + label: golem15.seo::lang.issues.summary + type: text + sortable: false + source: + label: golem15.seo::lang.issues.source + type: text + sortable: true + fix_cell: + label: golem15.seo::lang.issues.fix + type: text + sortable: false + found_at: + label: golem15.seo::lang.issues.found + type: datetime + sortable: true + invisible: true + locale: + label: golem15.seo::lang.issues.locale + type: text + sortable: true + invisible: true diff --git a/models/issue/fields.yaml b/models/issue/fields.yaml new file mode 100644 index 0000000..0d6b8d2 --- /dev/null +++ b/models/issue/fields.yaml @@ -0,0 +1,33 @@ +# Issue preview (UI-SPEC C23): every field is read-only. +fields: + url: + label: golem15.seo::lang.issues.url + type: text + span: full + context: preview + kind: + label: golem15.seo::lang.issues.kind + type: dropdown + span: left + context: preview + severity: + label: golem15.seo::lang.issues.severity + type: dropdown + span: right + context: preview + source: + label: golem15.seo::lang.issues.source + type: dropdown + span: left + context: preview + found_at: + label: golem15.seo::lang.issues.found + type: datepicker + mode: datetime + span: right + context: preview + locale: + label: golem15.seo::lang.issues.locale + type: text + span: left + context: preview diff --git a/operations.go b/operations.go index cc9e629..ad43b2f 100644 --- a/operations.go +++ b/operations.go @@ -26,6 +26,8 @@ const ( OpBulkAction = "bulk_action" // OpWidgetAction is a widget action ("widget_action:"). OpWidgetAction = "widget_action" + // OpToolbarAction is a list toolbar action ("toolbar_action:"). + OpToolbarAction = "toolbar_action" ) // ContentController stands for the form of every content type the SEO tab @@ -94,6 +96,23 @@ var AdminOperations = []AdminOperation{ {controllers.PagesControllerID, OpDelete, "delete_page"}, {controllers.PagesControllerID, OpExtensionSave, "update_page"}, + // Site audit: the current issues, their record and bulk actions, and + // the toolbar's Start crawl. + {controllers.IssuesControllerID, OpList, "list_issues"}, + {controllers.IssuesControllerID, OpShow, "list_issues"}, + {controllers.IssuesControllerID, OpToolbarAction + ":" + controllers.ActionStartCrawl, "start_crawl"}, + {controllers.IssuesControllerID, OpRecordAction + ":" + controllers.IssueActionCheck, "recheck_url"}, + {controllers.IssuesControllerID, OpRecordAction + ":" + controllers.IssueActionIgnore, "ignore_issues"}, + {controllers.IssuesControllerID, OpRecordAction + ":" + controllers.IssueActionShow, "unignore_issues"}, + {controllers.IssuesControllerID, OpBulkAction + ":" + controllers.IssueActionIgnore, "ignore_issues"}, + + // Crawls. + {controllers.CrawlsControllerID, OpList, "list_crawls"}, + {controllers.CrawlsControllerID, OpShow, "get_crawl"}, + {controllers.CrawlsControllerID, OpDelete, "delete_crawls"}, + {controllers.CrawlsControllerID, OpToolbarAction + ":" + controllers.ActionStartCrawl, "start_crawl"}, + {controllers.CrawlsControllerID, OpRecordAction + ":" + controllers.CrawlActionCancel, "cancel_crawl"}, + // robots.txt and AI bots. {controllers.RobotsControllerID, OpShow, "show_robots"}, {controllers.RobotsControllerID, OpUpdate, "update_robots"}, diff --git a/operations_test.go b/operations_test.go index d620eb8..daac6a4 100644 --- a/operations_test.go +++ b/operations_test.go @@ -128,7 +128,13 @@ func requiredOperations(t *testing.T, p *Plugin) []AdminOperation { } if h, ok := c.(pact.HasAdminActions); ok { for _, a := range h.AdminActions() { - need(id, OpWidgetAction+":"+a.Name) + // An action named in the list toolbar is a toolbar action, + // any other one a form widget's. + if slices.Contains(buttons, a.Name) { + need(id, OpToolbarAction+":"+a.Name) + } else { + need(id, OpWidgetAction+":"+a.Name) + } } } } diff --git a/routes.go b/routes.go index ef7bfc0..b4e2dda 100644 --- a/routes.go +++ b/routes.go @@ -81,7 +81,14 @@ func (p *Plugin) Routes(r pact.Router) error { g.Post("/import", apicontrollers.ImportCSV(p.Service), api) g.Post("/batch", apicontrollers.ApplyBatch(p.Service), api) g.Get("/issues", apicontrollers.ListIssues(p.Service), api) + g.Post("/issues/ignore", apicontrollers.IgnoreIssues(p.Service), api) + g.Post("/issues/unignore", apicontrollers.UnignoreIssues(p.Service), api) + g.Post("/issues/{id}/recheck", apicontrollers.RecheckIssue(p.Service), api) + g.Get("/crawls", apicontrollers.ListCrawls(p.Service), api) g.Post("/crawls", apicontrollers.StartCrawl(p.Service), api) + g.Delete("/crawls", apicontrollers.DeleteCrawls(p.Service), api) + g.Get("/crawls/{id}", apicontrollers.ShowCrawl(p.Service), api) + g.Post("/crawls/{id}/cancel", apicontrollers.CancelCrawl(p.Service), api) }) return nil } diff --git a/seo_tab_test.go b/seo_tab_test.go index b177637..72d4ce8 100644 --- a/seo_tab_test.go +++ b/seo_tab_test.go @@ -594,7 +594,10 @@ func TestLangKeysComplete(t *testing.T) { } for _, m := range langKeyPattern.FindAllStringSubmatch(string(raw), -1) { key := m[1] - if strings.HasSuffix(key, "via_") || strings.HasSuffix(key, "source_") { + // Keys built from a value: via_, source_, and the + // issue kinds' kind_ and sum_ (TestIssueKindLabels + // checks every kind). + if strings.HasSuffix(key, "via_") || strings.HasSuffix(key, "source_") || strings.HasSuffix(key, "kind_") || strings.HasSuffix(key, "sum_") { continue } seen[key] = true