feat: Site audit and Crawls screens with their MCP tools
- golem15.seo.issues (Site audit): current issues errors first, labels, summary and fix columns, filters, P7 issue_details, Check this address again, Ignore, Stop ignoring, bulk Ignore, ignored rows badged - golem15.seo.crawls (Crawls): newest first, P6 crawl_summary with one status callout, staging and rendered-comparison lines, stats strip and per-kind table; Cancel; a running crawl refuses delete - Start crawl toolbar action on both, refusing already_running and no_site_url; navigation Site audit (gauge) and Crawls (globe) - routes and MCP tools start/cancel/list/get/delete crawls, list_issues with filters (open by default), recheck_url, ignore and unignore - AdminOperations maps both controllers; toolbar actions get their own operation kind - TestCrawlAdmin, TestCrawlTools, TestIssueKindLabels
This commit is contained in:
Jakub Zych
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
||||
@@ -61,7 +63,7 @@ func StartCrawl(service func() *classes.Service) http.HandlerFunc {
|
||||
if err != nil {
|
||||
return 0, nil, StartRefusal(ctx, svc, err)
|
||||
}
|
||||
return http.StatusAccepted, map[string]any{"data": crawlView(row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil
|
||||
return http.StatusAccepted, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil
|
||||
})
|
||||
}
|
||||
|
||||
@@ -88,3 +90,222 @@ func StartRefusal(ctx context.Context, svc *classes.Service, err error) error {
|
||||
func RunningMessage(ctx context.Context, svc *classes.Service, running *crawl.AlreadyRunningError) string {
|
||||
return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.already_running", map[string]string{"time": running.Since().UTC().Format("2006-01-02 15:04 UTC")})
|
||||
}
|
||||
|
||||
// RenderView is a crawl's rendered comparison in an answer (P33-55):
|
||||
// skipped (no golem15.seo.crawl_chrome_path), done with the pages
|
||||
// compared, or failed with the note; message says it in the request
|
||||
// locale.
|
||||
type RenderView struct {
|
||||
Status string `json:"status"`
|
||||
Pages int `json:"pages"`
|
||||
Note string `json:"note,omitempty"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// renderView is the answer form of a crawl's comparison.
|
||||
func renderView(ctx context.Context, svc *classes.Service, c models.Crawl) RenderView {
|
||||
out := RenderView{Status: c.RenderStatus, Pages: c.RenderPages, Note: c.RenderNote}
|
||||
switch c.RenderStatus {
|
||||
case models.RenderDone:
|
||||
out.Message = svc.ChoiceCtx(ctx, "golem15.seo::lang.crawls.render_done", c.RenderPages, nil)
|
||||
case models.RenderFailed:
|
||||
out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_failed", map[string]string{"error": c.RenderNote})
|
||||
default:
|
||||
out.Status = models.RenderSkipped
|
||||
out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_skipped", nil)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ListCrawlsInput is the query of GET /_seo/api/v1/crawls.
|
||||
type ListCrawlsInput struct {
|
||||
Page int `json:"page,omitempty" query:"page" jsonschema:"the page number, from 1"`
|
||||
PerPage int `json:"per_page,omitempty" query:"per_page" jsonschema:"rows per page, at most 200 (50 by default)"`
|
||||
Status string `json:"status,omitempty" query:"status" jsonschema:"queued, running, finished, failed or cancelled"`
|
||||
}
|
||||
|
||||
// CrawlIDInput names one crawl.
|
||||
type CrawlIDInput struct {
|
||||
ID uint `json:"id" path:"id" jsonschema:"the crawl id"`
|
||||
}
|
||||
|
||||
// DeleteCrawlsInput is the query of DELETE /_seo/api/v1/crawls.
|
||||
type DeleteCrawlsInput struct {
|
||||
IDs string `json:"ids" query:"ids" jsonschema:"the crawl ids to delete, comma-separated, e.g. 3,4; a queued or running crawl refuses the whole call"`
|
||||
}
|
||||
|
||||
// ListCrawls serves GET /_seo/api/v1/crawls: the crawls, newest first,
|
||||
// with their issue counts and rendered comparison.
|
||||
func ListCrawls(service func() *classes.Service) http.HandlerFunc {
|
||||
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
||||
db, err := read(ctx, svc)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
in := listParams(r)
|
||||
q := db.Model(&models.Crawl{}).Order("id DESC")
|
||||
if s := strings.TrimSpace(r.URL.Query().Get("status")); s != "" {
|
||||
q = q.Where("status = ?", s)
|
||||
}
|
||||
var rows []models.Crawl
|
||||
total, err := paginate(q, in, &rows)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
ids := make([]uint, 0, len(rows))
|
||||
for _, c := range rows {
|
||||
ids = append(ids, c.ID)
|
||||
}
|
||||
counts := map[uint]int{}
|
||||
if len(ids) > 0 {
|
||||
var cs []struct {
|
||||
CrawlID uint
|
||||
N int
|
||||
}
|
||||
if err := db.Raw(`SELECT crawl_id, COUNT(*) AS n FROM golem15_seo_issues WHERE crawl_id IN ? GROUP BY crawl_id`, ids).Scan(&cs).Error; err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
for _, c := range cs {
|
||||
counts[c.CrawlID] = c.N
|
||||
}
|
||||
}
|
||||
out := make([]map[string]any, 0, len(rows))
|
||||
for _, c := range rows {
|
||||
v := crawlMap(ctx, svc, c)
|
||||
v["issues"] = counts[c.ID]
|
||||
out = append(out, v)
|
||||
}
|
||||
return http.StatusOK, listBody(out, in, total), nil
|
||||
})
|
||||
}
|
||||
|
||||
// crawlMap is a crawl with its rendered comparison.
|
||||
func crawlMap(ctx context.Context, svc *classes.Service, c models.Crawl) map[string]any {
|
||||
v := crawlView(c)
|
||||
return map[string]any{
|
||||
"id": v.ID, "status": v.Status, "trigger": v.Trigger, "trigger_name": v.TriggerName, "site": v.Site,
|
||||
"max_pages": v.MaxPages, "pages": v.Pages, "limit_hit": v.LimitHit, "error": v.Error,
|
||||
"started_at": v.StartedAt, "finished_at": v.FinishedAt, "created_at": v.CreatedAt,
|
||||
"cancelled_by": c.CancelledBy, "render": renderView(ctx, svc, c),
|
||||
}
|
||||
}
|
||||
|
||||
// ShowCrawl serves GET /_seo/api/v1/crawls/{id}: one crawl with its
|
||||
// rendered comparison, whether the staging lock's noindex was left out,
|
||||
// and its issues per kind and severity.
|
||||
func ShowCrawl(service func() *classes.Service) http.HandlerFunc {
|
||||
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
||||
id, err := pathID(r, "id")
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
db, err := read(ctx, svc)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
c, err := crawl.LoadCrawl(db, id)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
var kinds []struct {
|
||||
Kind string `json:"kind"`
|
||||
Severity string `json:"severity"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
if err := db.Raw(`SELECT kind, severity, COUNT(*) AS count FROM golem15_seo_issues WHERE crawl_id = ? GROUP BY kind, severity ORDER BY MIN(rank), COUNT(*) DESC, kind`, id).Scan(&kinds).Error; err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
out := crawlMap(ctx, svc, c)
|
||||
out["staging"] = svc.Staging()
|
||||
bySeverity := map[string]int{}
|
||||
total := 0
|
||||
for _, k := range kinds {
|
||||
bySeverity[k.Severity] += k.Count
|
||||
total += k.Count
|
||||
}
|
||||
out["issues"] = total
|
||||
out["severities"] = bySeverity
|
||||
out["kinds"] = kinds
|
||||
return http.StatusOK, map[string]any{"data": out}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// CancelCrawl serves POST /_seo/api/v1/crawls/{id}/cancel: a queued or
|
||||
// running crawl stops and keeps what it fetched; another answers 409
|
||||
// crawls.not_running.
|
||||
func CancelCrawl(service func() *classes.Service) http.HandlerFunc {
|
||||
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
||||
id, err := pathID(r, "id")
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
var row models.Crawl
|
||||
err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
|
||||
c, err := crawl.LoadCrawl(tx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, name, _ := svc.Actor(ctx, tx)
|
||||
if err := crawl.Cancel(ctx, svc, tx, c, name); err != nil {
|
||||
return err
|
||||
}
|
||||
row, err = crawl.LoadCrawl(tx, id)
|
||||
return err
|
||||
})
|
||||
if errors.Is(err, crawl.ErrNotRunning) {
|
||||
return 0, nil, Fail(http.StatusConflict, "crawls.not_running")
|
||||
}
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
return http.StatusOK, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.cancelled", nil)}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteCrawls serves DELETE /_seo/api/v1/crawls?ids=…: the crawls go with
|
||||
// their pages and issues; a queued or running one refuses the whole call
|
||||
// with 403 crawls.delete_running.
|
||||
func DeleteCrawls(service func() *classes.Service) http.HandlerFunc {
|
||||
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
||||
ids, err := idList(r.URL.Query().Get("ids"))
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
n := 0
|
||||
err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
|
||||
var err error
|
||||
n, err = crawl.DeleteCrawls(tx, ids)
|
||||
return err
|
||||
})
|
||||
if errors.Is(err, crawl.ErrDeleteRunning) {
|
||||
return 0, nil, &apiError{Status: http.StatusForbidden, Body: map[string]any{
|
||||
"error": "crawls.delete_running",
|
||||
"message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.delete_running", nil),
|
||||
}}
|
||||
}
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
return http.StatusOK, map[string]any{"deleted": n}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// idList reads a comma-separated id list of at most 500 ids.
|
||||
func idList(raw string) ([]uint, error) {
|
||||
var out []uint
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.ParseUint(part, 10, 64)
|
||||
if err != nil || n == 0 {
|
||||
return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid")
|
||||
}
|
||||
out = append(out, uint(n))
|
||||
}
|
||||
if len(out) == 0 || len(out) > 500 {
|
||||
return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user