feat: Site audit and Crawls screens with their MCP tools
- golem15.seo.issues (Site audit): current issues errors first, labels, summary and fix columns, filters, P7 issue_details, Check this address again, Ignore, Stop ignoring, bulk Ignore, ignored rows badged - golem15.seo.crawls (Crawls): newest first, P6 crawl_summary with one status callout, staging and rendered-comparison lines, stats strip and per-kind table; Cancel; a running crawl refuses delete - Start crawl toolbar action on both, refusing already_running and no_site_url; navigation Site audit (gauge) and Crawls (globe) - routes and MCP tools start/cancel/list/get/delete crawls, list_issues with filters (open by default), recheck_url, ignore and unignore - AdminOperations maps both controllers; toolbar actions get their own operation kind - TestCrawlAdmin, TestCrawlTools, TestIssueKindLabels
This commit is contained in:
Jakub Zych
350
controllers/crawls.go
Normal file
350
controllers/crawls.go
Normal file
@@ -0,0 +1,350 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/cabana"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// CrawlsControllerID is the Crawls screen (UI-SPEC C24).
|
||||
const CrawlsControllerID = "golem15.seo.crawls"
|
||||
|
||||
// CrawlActionCancel is the preview's Cancel crawl.
|
||||
const CrawlActionCancel = "cancel"
|
||||
|
||||
// errCrawlReadOnly refuses creating or editing crawls.
|
||||
var errCrawlReadOnly = errors.New("golem15.seo: crawls are read-only")
|
||||
|
||||
// crawlsAdminController lists the crawls, newest first, and previews one
|
||||
// with P6 and Cancel while it is queued or running; the toolbar starts a
|
||||
// crawl and deletes the selected ones (a running crawl refuses).
|
||||
type crawlsAdminController struct {
|
||||
app func() *backpack.App
|
||||
}
|
||||
|
||||
func (crawlsAdminController) ID() string { return CrawlsControllerID }
|
||||
func (crawlsAdminController) ModelName() string { return `Golem15\SEO\Models\Crawl` }
|
||||
func (crawlsAdminController) ConfigDir() string { return "controllers/crawls" }
|
||||
func (crawlsAdminController) NewRecord() any { return &models.Crawl{} }
|
||||
func (crawlsAdminController) RequiredPermissions() []string {
|
||||
return []string{PermissionManage}
|
||||
}
|
||||
|
||||
// FormBeforeCreate refuses: crawls are started with Start crawl.
|
||||
func (crawlsAdminController) FormBeforeCreate(context.Context, any) error { return errCrawlReadOnly }
|
||||
|
||||
// FormBeforeUpdate refuses: crawls are never edited.
|
||||
func (crawlsAdminController) FormBeforeUpdate(context.Context, any) error { return errCrawlReadOnly }
|
||||
|
||||
// FormBeforeDelete refuses a queued or running crawl with
|
||||
// crawls.delete_running (403).
|
||||
func (c crawlsAdminController) FormBeforeDelete(ctx context.Context, model any) error {
|
||||
row, ok := model.(*models.Crawl)
|
||||
if ok && row != nil && row.Active() {
|
||||
return &cabana.ForbiddenError{Message: Phrase(ctx, translator(c.app), "golem15.seo::lang.crawls.delete_running", nil)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c crawlsAdminController) service() *classes.Service {
|
||||
svc, err := classes.NewService(appOf(c.app))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return svc
|
||||
}
|
||||
|
||||
// AdminActions is the toolbar's Start crawl.
|
||||
func (c crawlsAdminController) AdminActions() []pact.AdminAction {
|
||||
return []pact.AdminAction{startCrawl(c.app)}
|
||||
}
|
||||
|
||||
// AdminRecordActions is Cancel crawl while queued or running; the pages
|
||||
// and issues fetched so far are kept.
|
||||
func (c crawlsAdminController) AdminRecordActions() []pact.AdminRecordAction {
|
||||
return []pact.AdminRecordAction{{
|
||||
Name: CrawlActionCancel,
|
||||
Label: "golem15.seo::lang.crawls.cancel",
|
||||
Confirm: "golem15.seo::lang.crawls.cancel_confirm",
|
||||
Permissions: []string{PermissionManage},
|
||||
Applies: func(_ context.Context, record any) (bool, error) {
|
||||
row, ok := record.(*models.Crawl)
|
||||
return ok && row != nil && row.Active(), nil
|
||||
},
|
||||
Run: func(ctx context.Context, in pact.AdminRecordActionInput) (pact.AdminRecordActionResult, error) {
|
||||
row, ok := in.Record.(*models.Crawl)
|
||||
svc := c.service()
|
||||
if !ok || row == nil || svc == nil {
|
||||
return pact.AdminRecordActionResult{}, classes.ErrNotBooted
|
||||
}
|
||||
db, err := database(ctx, c.app)
|
||||
if err != nil {
|
||||
return pact.AdminRecordActionResult{}, err
|
||||
}
|
||||
_, name, _ := svc.Actor(ctx, db)
|
||||
if err := crawl.Cancel(ctx, svc, db, *row, name); err != nil {
|
||||
return pact.AdminRecordActionResult{}, err
|
||||
}
|
||||
return pact.AdminRecordActionResult{Message: "golem15.seo::lang.crawls.cancelled"}, nil
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
// statusKeys are the phrase keys of the crawl states.
|
||||
var statusKeys = map[string]string{
|
||||
models.CrawlQueued: "golem15.seo::lang.crawls.st_queued",
|
||||
models.CrawlRunning: "golem15.seo::lang.crawls.st_running",
|
||||
models.CrawlFinished: "golem15.seo::lang.crawls.st_finished",
|
||||
models.CrawlFailed: "golem15.seo::lang.crawls.st_failed",
|
||||
models.CrawlCancelled: "golem15.seo::lang.crawls.st_cancelled",
|
||||
}
|
||||
|
||||
// TriggerLabel is the Started by cell: Manual: name, Schedule or MCP:
|
||||
// client.
|
||||
func TriggerLabel(ctx context.Context, svc *classes.Service, row models.Crawl) string {
|
||||
switch row.Trigger {
|
||||
case models.TriggerSchedule:
|
||||
return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.by_schedule", nil)
|
||||
case models.TriggerMCP:
|
||||
return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.by_mcp", map[string]string{"client": row.TriggerName})
|
||||
}
|
||||
return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.by_admin", map[string]string{"name": row.TriggerName})
|
||||
}
|
||||
|
||||
// Duration is a crawl's duration, e.g. "4 min 12 s"; "—" before it ends.
|
||||
func Duration(row models.Crawl) string {
|
||||
if row.StartedAt == nil {
|
||||
return "—"
|
||||
}
|
||||
end := time.Now().UTC()
|
||||
if row.FinishedAt != nil {
|
||||
end = *row.FinishedAt
|
||||
}
|
||||
d := end.Sub(*row.StartedAt).Round(time.Second)
|
||||
if d < 0 {
|
||||
d = 0
|
||||
}
|
||||
h, m, s := int(d.Hours()), int(d.Minutes())%60, int(d.Seconds())%60
|
||||
switch {
|
||||
case h > 0:
|
||||
return fmt.Sprintf("%d h %d min", h, m)
|
||||
case m > 0:
|
||||
return fmt.Sprintf("%d min %d s", m, s)
|
||||
}
|
||||
return fmt.Sprintf("%d s", s)
|
||||
}
|
||||
|
||||
// issueCounts counts the issues of crawls.
|
||||
func issueCounts(db *gorm.DB, ids []uint) (map[uint]int, error) {
|
||||
out := map[uint]int{}
|
||||
if len(ids) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
var rows []struct {
|
||||
CrawlID uint
|
||||
N int
|
||||
}
|
||||
if err := db.Raw(`SELECT crawl_id, COUNT(*) AS n FROM golem15_seo_issues WHERE crawl_id IN ? GROUP BY crawl_id`, ids).Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, r := range rows {
|
||||
out[r.CrawlID] = r.N
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListExtendRecords shows the state, pages (with the limit note), issue
|
||||
// count, trigger and duration of each crawl in the request locale.
|
||||
func (c crawlsAdminController) ListExtendRecords(ctx context.Context, db *gorm.DB, records []any) error {
|
||||
svc := c.service()
|
||||
if svc == nil {
|
||||
return nil
|
||||
}
|
||||
var ids []uint
|
||||
for _, r := range records {
|
||||
if row, ok := r.(*models.Crawl); ok && row != nil {
|
||||
ids = append(ids, row.ID)
|
||||
}
|
||||
}
|
||||
counts, err := issueCounts(db.WithContext(ctx), ids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, r := range records {
|
||||
row, ok := r.(*models.Crawl)
|
||||
if !ok || row == nil {
|
||||
continue
|
||||
}
|
||||
row.PagesCell = strconv.Itoa(row.Pages)
|
||||
if row.LimitHit {
|
||||
row.PagesCell = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.pages_limit", map[string]string{"count": strconv.Itoa(row.Pages)})
|
||||
}
|
||||
row.IssuesCell = strconv.Itoa(counts[row.ID])
|
||||
row.DurationCell = Duration(*row)
|
||||
row.TriggerCell = TriggerLabel(ctx, svc, *row)
|
||||
if key, ok := statusKeys[row.Status]; ok {
|
||||
row.Status = svc.PhraseCtx(ctx, key, nil)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DropdownOptions labels the preview's read-only status and trigger.
|
||||
func (crawlsAdminController) DropdownOptions(field string) []pact.Option {
|
||||
switch field {
|
||||
case "status":
|
||||
out := make([]pact.Option, 0, len(models.CrawlStatuses))
|
||||
for _, s := range models.CrawlStatuses {
|
||||
out = append(out, pact.Option{Value: s, Label: statusKeys[s]})
|
||||
}
|
||||
return out
|
||||
case "trigger":
|
||||
return []pact.Option{
|
||||
{Value: models.TriggerAdmin, Label: "golem15.seo::lang.crawls.trigger_admin"},
|
||||
{Value: models.TriggerSchedule, Label: "golem15.seo::lang.crawls.trigger_schedule"},
|
||||
{Value: models.TriggerMCP, Label: "golem15.seo::lang.crawls.trigger_mcp"},
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FilterOptions serves the status filter.
|
||||
func (c crawlsAdminController) FilterOptions(scope string) []pact.Option {
|
||||
if scope != models.FilterCrawlStatus {
|
||||
return nil
|
||||
}
|
||||
return c.DropdownOptions("status")
|
||||
}
|
||||
|
||||
// PartialData serves P6 crawl_summary (UI-SPEC C24).
|
||||
func (c crawlsAdminController) PartialData(ctx context.Context, name string, record any) (any, error) {
|
||||
row, ok := record.(*models.Crawl)
|
||||
svc := c.service()
|
||||
if name != "crawl_summary" || !ok || row == nil || svc == nil {
|
||||
return classes.CrawlSummaryView{}, nil
|
||||
}
|
||||
db, err := database(ctx, c.app)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return CrawlSummary(ctx, svc, db, c.app, *row)
|
||||
}
|
||||
|
||||
// CrawlSummary builds P6: one status callout, the staging and rendered
|
||||
// comparison lines, the stats strip and the per-kind table whose counts
|
||||
// link to the Site audit list filtered by the kind.
|
||||
func CrawlSummary(ctx context.Context, svc *classes.Service, db *gorm.DB, app func() *backpack.App, row models.Crawl) (classes.CrawlSummaryView, error) {
|
||||
view := classes.CrawlSummaryView{}
|
||||
p := func(key string, params map[string]string) string {
|
||||
return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls."+key, params)
|
||||
}
|
||||
stamp := func(t *time.Time) string {
|
||||
if t == nil {
|
||||
return "—"
|
||||
}
|
||||
return t.UTC().Format("2006-01-02 15:04 UTC")
|
||||
}
|
||||
switch row.Status {
|
||||
case models.CrawlRunning:
|
||||
view.Title = p("running_title", nil)
|
||||
view.Text = p("running_text", map[string]string{"done": strconv.Itoa(row.Pages), "limit": strconv.Itoa(row.MaxPages)})
|
||||
case models.CrawlQueued:
|
||||
view.Text = p("queued_text", nil)
|
||||
case models.CrawlFailed:
|
||||
view.Tone = "danger"
|
||||
view.Title = p("failed_title", nil)
|
||||
view.Text = p("failed_text", map[string]string{"error": row.Error})
|
||||
case models.CrawlCancelled:
|
||||
view.Text = p("cancelled_text", map[string]string{"name": row.CancelledBy, "time": stamp(row.FinishedAt)})
|
||||
case models.CrawlFinished:
|
||||
if row.LimitHit {
|
||||
view.Tone = "warning"
|
||||
view.Text = p("limit_text", map[string]string{"limit": strconv.Itoa(row.MaxPages)})
|
||||
}
|
||||
}
|
||||
view.HasCallout = view.Text != ""
|
||||
if svc.Staging() {
|
||||
view.Staging = p("staging_text", nil)
|
||||
}
|
||||
switch row.RenderStatus {
|
||||
case models.RenderDone:
|
||||
view.Render = svc.ChoiceCtx(ctx, "golem15.seo::lang.crawls.render_done", row.RenderPages, nil)
|
||||
case models.RenderFailed:
|
||||
view.Render = p("render_failed", map[string]string{"error": row.RenderNote})
|
||||
default:
|
||||
view.Render = p("render_skipped", nil)
|
||||
}
|
||||
var rows []struct {
|
||||
Kind string
|
||||
Severity string
|
||||
N int
|
||||
}
|
||||
if err := db.Raw(`SELECT kind, severity, COUNT(*) AS n FROM golem15_seo_issues WHERE crawl_id = ? GROUP BY kind, severity`, row.ID).Scan(&rows).Error; err != nil {
|
||||
return view, err
|
||||
}
|
||||
sevCount := map[string]int{}
|
||||
prefix, err := cabana.AdminPrefix(appOf(app))
|
||||
if err != nil {
|
||||
prefix = cabana.DefaultAdminPrefix
|
||||
}
|
||||
for _, r := range rows {
|
||||
sevCount[r.Severity] += r.N
|
||||
view.Kinds = append(view.Kinds, classes.CrawlKindRow{
|
||||
Kind: r.Kind,
|
||||
Label: svc.PhraseCtx(ctx, "golem15.seo::lang.issues.kind_"+r.Kind, nil),
|
||||
Severity: svc.PhraseCtx(ctx, labelKeys[r.Severity], nil),
|
||||
Count: r.N,
|
||||
Link: prefix + "/golem15/seo/issues?" + url.Values{"filter[kind]": {r.Kind}}.Encode(),
|
||||
})
|
||||
}
|
||||
rank := map[string]int{}
|
||||
for _, r := range rows {
|
||||
rank[r.Kind] = models.SeverityRank(r.Severity)
|
||||
}
|
||||
sort.SliceStable(view.Kinds, func(i, j int) bool {
|
||||
a, b := view.Kinds[i], view.Kinds[j]
|
||||
if rank[a.Kind] != rank[b.Kind] {
|
||||
return rank[a.Kind] < rank[b.Kind]
|
||||
}
|
||||
if a.Count != b.Count {
|
||||
return a.Count > b.Count
|
||||
}
|
||||
return a.Kind < b.Kind
|
||||
})
|
||||
view.NoIssues = len(view.Kinds) == 0 && !row.Active()
|
||||
view.Stats = []classes.CrawlStat{
|
||||
{Label: p("stat_pages", nil), Value: strconv.Itoa(row.Pages)},
|
||||
{Label: p("stat_errors", nil), Value: strconv.Itoa(sevCount[models.SeverityError])},
|
||||
{Label: p("stat_warnings", nil), Value: strconv.Itoa(sevCount[models.SeverityWarning])},
|
||||
{Label: p("stat_notices", nil), Value: strconv.Itoa(sevCount[models.SeverityNotice])},
|
||||
{Label: p("stat_duration", nil), Value: Duration(row)},
|
||||
}
|
||||
return view, nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ pact.AdminController = crawlsAdminController{}
|
||||
_ pact.AdminPermissioned = crawlsAdminController{}
|
||||
_ pact.AdminRecordSource = crawlsAdminController{}
|
||||
_ pact.FormBeforeCreate = crawlsAdminController{}
|
||||
_ pact.FormBeforeUpdate = crawlsAdminController{}
|
||||
_ pact.FormBeforeDelete = crawlsAdminController{}
|
||||
_ pact.HasAdminActions = crawlsAdminController{}
|
||||
_ pact.HasAdminRecordActions = crawlsAdminController{}
|
||||
_ pact.ListExtendRecords = crawlsAdminController{}
|
||||
_ pact.DropdownOptionsProvider = crawlsAdminController{}
|
||||
_ pact.FilterOptions = crawlsAdminController{}
|
||||
_ pact.AdminPartialData = crawlsAdminController{}
|
||||
)
|
||||
Reference in New Issue
Block a user