golem15/sm-seo-plugin
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Site audit and Crawls screens with their MCP tools

Browse Source 
- golem15.seo.issues (Site audit): current issues errors first, labels,
  summary and fix columns, filters, P7 issue_details, Check this address
  again, Ignore, Stop ignoring, bulk Ignore, ignored rows badged
- golem15.seo.crawls (Crawls): newest first, P6 crawl_summary with one
  status callout, staging and rendered-comparison lines, stats strip and
  per-kind table; Cancel; a running crawl refuses delete
- Start crawl toolbar action on both, refusing already_running and
  no_site_url; navigation Site audit (gauge) and Crawls (globe)
- routes and MCP tools start/cancel/list/get/delete crawls, list_issues
  with filters (open by default), recheck_url, ignore and unignore
- AdminOperations maps both controllers; toolbar actions get their own
  operation kind
- TestCrawlAdmin, TestCrawlTools, TestIssueKindLabels
This commit is contained in:
Jakub Zych
2026-11-04 15:34:22 +01:00
parent edc03cae4b
commit c4f0b4f94c
32 changed files with 2778 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

223
controllers/api/crawls.go
View File

@@ -4,6 +4,8 @@ import (
"context"
"errors"
"net/http"
"strconv"
"strings"
"time"
"git.golem15.com/golem15/sm-seo-plugin/classes"
@@ -61,7 +63,7 @@ func StartCrawl(service func() *classes.Service) http.HandlerFunc {
if err != nil {
return 0, nil, StartRefusal(ctx, svc, err)
}
return http.StatusAccepted, map[string]any{"data": crawlView(row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil
return http.StatusAccepted, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil
})
}
@@ -88,3 +90,222 @@ func StartRefusal(ctx context.Context, svc *classes.Service, err error) error {
func RunningMessage(ctx context.Context, svc *classes.Service, running *crawl.AlreadyRunningError) string {
return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.already_running", map[string]string{"time": running.Since().UTC().Format("2006-01-02 15:04 UTC")})
}
// RenderView is a crawl's rendered comparison in an answer (P33-55):
// skipped (no golem15.seo.crawl_chrome_path), done with the pages
// compared, or failed with the note; message says it in the request
// locale.
type RenderView struct {
Status string `json:"status"`
Pages int `json:"pages"`
Note string `json:"note,omitempty"`
Message string `json:"message"`
}
// renderView is the answer form of a crawl's comparison.
func renderView(ctx context.Context, svc *classes.Service, c models.Crawl) RenderView {
out := RenderView{Status: c.RenderStatus, Pages: c.RenderPages, Note: c.RenderNote}
switch c.RenderStatus {
case models.RenderDone:
out.Message = svc.ChoiceCtx(ctx, "golem15.seo::lang.crawls.render_done", c.RenderPages, nil)
case models.RenderFailed:
out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_failed", map[string]string{"error": c.RenderNote})
default:
out.Status = models.RenderSkipped
out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_skipped", nil)
}
return out
}
// ListCrawlsInput is the query of GET /_seo/api/v1/crawls.
type ListCrawlsInput struct {
Page int `json:"page,omitempty" query:"page" jsonschema:"the page number, from 1"`
PerPage int `json:"per_page,omitempty" query:"per_page" jsonschema:"rows per page, at most 200 (50 by default)"`
Status string `json:"status,omitempty" query:"status" jsonschema:"queued, running, finished, failed or cancelled"`
}
// CrawlIDInput names one crawl.
type CrawlIDInput struct {
ID uint `json:"id" path:"id" jsonschema:"the crawl id"`
}
// DeleteCrawlsInput is the query of DELETE /_seo/api/v1/crawls.
type DeleteCrawlsInput struct {
IDs string `json:"ids" query:"ids" jsonschema:"the crawl ids to delete, comma-separated, e.g. 3,4; a queued or running crawl refuses the whole call"`
}
// ListCrawls serves GET /_seo/api/v1/crawls: the crawls, newest first,
// with their issue counts and rendered comparison.
func ListCrawls(service func() *classes.Service) http.HandlerFunc {
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
db, err := read(ctx, svc)
if err != nil {
return 0, nil, err
}
in := listParams(r)
q := db.Model(&models.Crawl{}).Order("id DESC")
if s := strings.TrimSpace(r.URL.Query().Get("status")); s != "" {
q = q.Where("status = ?", s)
}
var rows []models.Crawl
total, err := paginate(q, in, &rows)
if err != nil {
return 0, nil, err
}
ids := make([]uint, 0, len(rows))
for _, c := range rows {
ids = append(ids, c.ID)
}
counts := map[uint]int{}
if len(ids) > 0 {
var cs []struct {
CrawlID uint
N int
}
if err := db.Raw(`SELECT crawl_id, COUNT(*) AS n FROM golem15_seo_issues WHERE crawl_id IN ? GROUP BY crawl_id`, ids).Scan(&cs).Error; err != nil {
return 0, nil, err
}
for _, c := range cs {
counts[c.CrawlID] = c.N
}
}
out := make([]map[string]any, 0, len(rows))
for _, c := range rows {
v := crawlMap(ctx, svc, c)
v["issues"] = counts[c.ID]
out = append(out, v)
}
return http.StatusOK, listBody(out, in, total), nil
})
}
// crawlMap is a crawl with its rendered comparison.
func crawlMap(ctx context.Context, svc *classes.Service, c models.Crawl) map[string]any {
v := crawlView(c)
return map[string]any{
"id": v.ID, "status": v.Status, "trigger": v.Trigger, "trigger_name": v.TriggerName, "site": v.Site,
"max_pages": v.MaxPages, "pages": v.Pages, "limit_hit": v.LimitHit, "error": v.Error,
"started_at": v.StartedAt, "finished_at": v.FinishedAt, "created_at": v.CreatedAt,
"cancelled_by": c.CancelledBy, "render": renderView(ctx, svc, c),
}
}
// ShowCrawl serves GET /_seo/api/v1/crawls/{id}: one crawl with its
// rendered comparison, whether the staging lock's noindex was left out,
// and its issues per kind and severity.
func ShowCrawl(service func() *classes.Service) http.HandlerFunc {
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
id, err := pathID(r, "id")
if err != nil {
return 0, nil, err
}
db, err := read(ctx, svc)
if err != nil {
return 0, nil, err
}
c, err := crawl.LoadCrawl(db, id)
if err != nil {
return 0, nil, err
}
var kinds []struct {
Kind string `json:"kind"`
Severity string `json:"severity"`
Count int `json:"count"`
}
if err := db.Raw(`SELECT kind, severity, COUNT(*) AS count FROM golem15_seo_issues WHERE crawl_id = ? GROUP BY kind, severity ORDER BY MIN(rank), COUNT(*) DESC, kind`, id).Scan(&kinds).Error; err != nil {
return 0, nil, err
}
out := crawlMap(ctx, svc, c)
out["staging"] = svc.Staging()
bySeverity := map[string]int{}
total := 0
for _, k := range kinds {
bySeverity[k.Severity] += k.Count
total += k.Count
}
out["issues"] = total
out["severities"] = bySeverity
out["kinds"] = kinds
return http.StatusOK, map[string]any{"data": out}, nil
})
}
// CancelCrawl serves POST /_seo/api/v1/crawls/{id}/cancel: a queued or
// running crawl stops and keeps what it fetched; another answers 409
// crawls.not_running.
func CancelCrawl(service func() *classes.Service) http.HandlerFunc {
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
id, err := pathID(r, "id")
if err != nil {
return 0, nil, err
}
var row models.Crawl
err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
c, err := crawl.LoadCrawl(tx, id)
if err != nil {
return err
}
_, name, _ := svc.Actor(ctx, tx)
if err := crawl.Cancel(ctx, svc, tx, c, name); err != nil {
return err
}
row, err = crawl.LoadCrawl(tx, id)
return err
})
if errors.Is(err, crawl.ErrNotRunning) {
return 0, nil, Fail(http.StatusConflict, "crawls.not_running")
}
if err != nil {
return 0, nil, err
}
return http.StatusOK, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.cancelled", nil)}, nil
})
}
// DeleteCrawls serves DELETE /_seo/api/v1/crawls?ids=…: the crawls go with
// their pages and issues; a queued or running one refuses the whole call
// with 403 crawls.delete_running.
func DeleteCrawls(service func() *classes.Service) http.HandlerFunc {
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
ids, err := idList(r.URL.Query().Get("ids"))
if err != nil {
return 0, nil, err
}
n := 0
err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
var err error
n, err = crawl.DeleteCrawls(tx, ids)
return err
})
if errors.Is(err, crawl.ErrDeleteRunning) {
return 0, nil, &apiError{Status: http.StatusForbidden, Body: map[string]any{
"error": "crawls.delete_running",
"message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.delete_running", nil),
}}
}
if err != nil {
return 0, nil, err
}
return http.StatusOK, map[string]any{"deleted": n}, nil
})
}
// idList reads a comma-separated id list of at most 500 ids.
func idList(raw string) ([]uint, error) {
var out []uint
for _, part := range strings.Split(raw, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
n, err := strconv.ParseUint(part, 10, 64)
if err != nil || n == 0 {
return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid")
}
out = append(out, uint(n))
}
if len(out) == 0 || len(out) > 500 {
return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid")
}
return out, nil
}

142
controllers/api/issues.go
View File

@@ -2,6 +2,7 @@ package api
import (
"context"
"errors"
"net/http"
"strconv"
"strings"
@@ -15,7 +16,23 @@ import (
// ListIssuesInput is the query of GET /_seo/api/v1/issues.
type ListIssuesInput struct {
Kind string `json:"kind,omitempty" query:"kind" jsonschema:"one kind only, e.g. h1_multiple, broken_link, missing_title, redirect_chain or sitemap_blocked"`
Kind string `json:"kind,omitempty" query:"kind" jsonschema:"one kind only, e.g. h1_multiple, broken_link, missing_title, content_js, orphan_page, redirect_chain or sitemap_blocked"`
Severity string `json:"severity,omitempty" query:"severity" jsonschema:"error, warning or notice"`
Source string `json:"source,omitempty" query:"source" jsonschema:"crawl (found by the crawler) or cms (CMS data checks)"`
Fix string `json:"fix,omitempty" query:"fix" jsonschema:"tool (fixable through the MCP tool the issue names) or code (needs a change in the site's templates)"`
Status string `json:"status,omitempty" query:"status" jsonschema:"open (the default), ignored or all"`
URL string `json:"url,omitempty" query:"url" jsonschema:"only issues of this address (a path such as /blog/post)"`
Crawl uint `json:"crawl,omitempty" query:"crawl" jsonschema:"the issues of this crawl id instead of the current ones"`
}
// IssueIDInput names one issue.
type IssueIDInput struct {
ID uint `json:"id" path:"id" jsonschema:"the issue id"`
}
// IssueIDsInput is the body of the ignore and unignore calls.
type IssueIDsInput struct {
IDs []uint `json:"ids" jsonschema:"the issue ids"`
}
// IssueView is a stored issue in an answer: the Phase 32 keys (kind, url,
@@ -61,8 +78,11 @@ func issueViews(db *gorm.DB, rows []models.Issue) ([]IssueView, error) {
// ListIssues serves GET /_seo/api/v1/issues (D-13, D-14): the audit's
// current issues, errors first: the newest crawl's findings with the
// CMS-data issues stored with it, or before any crawl the CMS-data issues
// as they are now. Each issue names the tool that fixes it, or fix code.
// counts holds the number per kind.
// as they are now (crawl selects another crawl's). Each issue names the
// tool that fixes it, or fix code. The filters are severity, kind,
// source, fix, url and status (open by default: ignored issues are left
// out; ignored or all). counts holds the number per kind before the kind
// filter.
func ListIssues(service func() *classes.Service) http.HandlerFunc {
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
db, err := read(ctx, svc)
@@ -86,15 +106,127 @@ func ListIssues(service func() *classes.Service) http.HandlerFunc {
if err != nil {
return 0, nil, err
}
kind := strings.TrimSpace(q.Get("kind"))
status := strings.TrimSpace(q.Get("status"))
if status == "" {
status = models.IssueOpen
}
match := map[string]string{
"kind": strings.TrimSpace(q.Get("kind")), "severity": strings.TrimSpace(q.Get("severity")),
"source": strings.TrimSpace(q.Get("source")), "fix": strings.TrimSpace(q.Get("fix")),
"url": strings.TrimSpace(q.Get("url")),
}
counts := map[string]int{}
out := make([]IssueView, 0, len(views))
for _, v := range views {
if status == models.IssueOpen && v.Ignored || status == models.IssueIgnored && !v.Ignored {
continue
}
if !matches(match["severity"], v.Severity) || !matches(match["source"], v.Source) || !matches(match["fix"], v.Fix) || !matches(match["url"], v.URL) {
continue
}
counts[v.Kind]++
if kind == "" || v.Kind == kind {
if matches(match["kind"], v.Kind) {
out = append(out, v)
}
}
return http.StatusOK, map[string]any{"data": out, "counts": counts, "total": len(out)}, nil
})
}
// matches reports whether a filter value (empty: any) matches.
func matches(filter, value string) bool { return filter == "" || filter == value }
// RecheckIssue serves POST /_seo/api/v1/issues/{id}/recheck: the issue's
// address is fetched again through the crawler client and its issues are
// replaced; the answer has the address, the issues left and the message.
func RecheckIssue(service func() *classes.Service) http.HandlerFunc {
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
id, err := pathID(r, "id")
if err != nil {
return 0, nil, err
}
var res crawl.RecheckResult
err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
issue, err := crawl.LoadIssue(tx, id)
if err != nil {
return err
}
res, err = crawl.Recheck(ctx, svc, tx, issue)
return err
})
if errors.Is(err, crawl.ErrNoSiteURL) {
return 0, nil, StartRefusal(ctx, svc, err)
}
if err != nil {
return 0, nil, err
}
msg := svc.PhraseCtx(ctx, "golem15.seo::lang.issues.recheck_fixed", nil)
if res.Remaining > 0 {
msg = svc.ChoiceCtx(ctx, "golem15.seo::lang.issues.rechecked", res.Remaining, nil)
}
return http.StatusOK, map[string]any{"url": res.URL, "remaining": res.Remaining, "message": msg}, nil
})
}
// IgnoreIssues serves POST /_seo/api/v1/issues/ignore: the issues are
// hidden in this and later crawls until they are no longer ignored.
func IgnoreIssues(service func() *classes.Service) http.HandlerFunc {
return ignoreHandler(service, true)
}
// UnignoreIssues serves POST /_seo/api/v1/issues/unignore: the issues are
// shown in the audit again.
func UnignoreIssues(service func() *classes.Service) http.HandlerFunc {
return ignoreHandler(service, false)
}
func ignoreHandler(service func() *classes.Service, ignore bool) http.HandlerFunc {
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
var in IssueIDsInput
if err := decode(r, maxBody, &in); err != nil {
return 0, nil, err
}
if len(in.IDs) == 0 || len(in.IDs) > 1000 {
return 0, nil, Fail(http.StatusUnprocessableEntity, "ids_invalid")
}
n := 0
err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
var rows []models.Issue
if err := tx.Where("id IN ?", in.IDs).Find(&rows).Error; err != nil {
return err
}
if len(rows) != len(uniqueIDs(in.IDs)) {
return gorm.ErrRecordNotFound
}
var err error
if ignore {
adminID, _, _ := svc.Actor(ctx, tx)
n, err = crawl.Ignore(tx, rows, adminID)
} else {
n, err = crawl.Unignore(tx, rows)
}
return err
})
if err != nil {
return 0, nil, err
}
key, field := "golem15.seo::lang.issues.unignored", "unignored"
if ignore {
key, field = "golem15.seo::lang.issues.ignored", "ignored"
}
return http.StatusOK, map[string]any{field: n, "message": svc.ChoiceCtx(ctx, key, n, nil)}, nil
})
}
// uniqueIDs drops repeated ids.
func uniqueIDs(ids []uint) []uint {
seen := map[uint]bool{}
var out []uint
for _, id := range ids {
if !seen[id] {
seen[id] = true
out = append(out, id)
}
}
return out
}