golem15/sm-seo-plugin
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: crawler finds a page with two H1 headings and stores the issue (tracer)

Browse Source 
- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37)
- classes/crawl: fetchguard client on the site host and its www twin,
  sitemap index and robots.txt seeds, two workers with a delay, limits
- h1_missing and h1_multiple with the P33-38 fix route
- POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues
  answers the stored issues with id, severity, source, fix, found_at,
  ignored and crawl_id
This commit is contained in:
Jakub Zych
2026-11-04 15:05:45 +01:00
parent d38049a956
commit c2b61bf8e9
23 changed files with 2967 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
controllers/api/crawls.go Normal file
View File

@@ -0,0 +1,90 @@
package api
import (
"context"
"errors"
"net/http"
"time"
"git.golem15.com/golem15/sm-seo-plugin/classes"
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
"git.golem15.com/golem15/sm-seo-plugin/models"
"gorm.io/gorm"
)
// StartCrawlInput is the body of POST /_seo/api/v1/crawls: nothing.
type StartCrawlInput struct{}
// CrawlView is a crawl in an answer.
type CrawlView struct {
ID uint `json:"id"`
Status string `json:"status"`
Trigger string `json:"trigger"`
TriggerName string `json:"trigger_name,omitempty"`
Site string `json:"site"`
MaxPages int `json:"max_pages"`
Pages int `json:"pages"`
LimitHit bool `json:"limit_hit"`
Error string `json:"error,omitempty"`
StartedAt *time.Time `json:"started_at"`
FinishedAt *time.Time `json:"finished_at"`
CreatedAt *time.Time `json:"created_at"`
}
// crawlView is the answer form of a crawl.
func crawlView(c models.Crawl) CrawlView {
return CrawlView{
ID: c.ID, Status: c.Status, Trigger: c.Trigger, TriggerName: c.TriggerName, Site: c.Site,
MaxPages: c.MaxPages, Pages: c.Pages, LimitHit: c.LimitHit, Error: c.Error,
StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, CreatedAt: c.CreatedAt,
}
}
// StartCrawl serves POST /_seo/api/v1/crawls (D-14, P33-39): it queues a
// crawl started by the administrator (trigger admin, or mcp through MCP)
// and answers 202 with it. A crawl already queued or running refuses with
// 409 crawls.already_running, a missing site address with 422
// crawls.no_site_url.
func StartCrawl(service func() *classes.Service) http.HandlerFunc {
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
var row models.Crawl
err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
_, name, viaMCP := svc.Actor(ctx, tx)
trigger := models.TriggerAdmin
if viaMCP {
trigger = models.TriggerMCP
}
var err error
row, err = crawl.Start(ctx, svc, tx, trigger, name)
return err
})
if err != nil {
return 0, nil, StartRefusal(ctx, svc, err)
}
return http.StatusAccepted, map[string]any{"data": crawlView(row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil
})
}
// StartRefusal maps a refused start to its answer.
func StartRefusal(ctx context.Context, svc *classes.Service, err error) error {
var running *crawl.AlreadyRunningError
switch {
case errors.As(err, &running):
return &apiError{Status: http.StatusConflict, Body: map[string]any{
"error": "crawls.already_running",
"crawl_id": running.Crawl.ID,
"message": RunningMessage(ctx, svc, running),
}}
case errors.Is(err, crawl.ErrNoSiteURL):
return &apiError{Status: http.StatusUnprocessableEntity, Body: map[string]any{
"error": "crawls.no_site_url",
"message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.no_site_url", nil),
}}
}
return err
}
// RunningMessage is crawls.already_running with the running crawl's start.
func RunningMessage(ctx context.Context, svc *classes.Service, running *crawl.AlreadyRunningError) string {
return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.already_running", map[string]string{"time": running.Since().UTC().Format("2006-01-02 15:04 UTC")})
}

87
controllers/api/issues.go
View File

@@ -3,31 +3,96 @@ package api
import (
"context"
"net/http"
"strconv"
"strings"
"time"
"git.golem15.com/golem15/sm-seo-plugin/classes"
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
"git.golem15.com/golem15/sm-seo-plugin/models"
"gorm.io/gorm"
)
// ListIssuesInput is the query of GET /_seo/api/v1/issues.
type ListIssuesInput struct {
Kind string `json:"kind,omitempty" query:"kind" jsonschema:"one kind only: missing_title, missing_description, duplicate_title, duplicate_description, rule_overrides_record, redirect_chain, canonical_noindex, canonical_redirected, noindex_blocked or sitemap_blocked"`
Kind string `json:"kind,omitempty" query:"kind" jsonschema:"one kind only, e.g. h1_multiple, broken_link, missing_title, redirect_chain or sitemap_blocked"`
}
// ListIssues serves GET /_seo/api/v1/issues (D-13, D-14 CMS part): the
// CMS-data issue report, each issue with its URL, kind, details and the
// tool that fixes it, and a count per kind.
// IssueView is a stored issue in an answer: the Phase 32 keys (kind, url,
// locale, details, tool) and the audit's id, severity, source, fix,
// found_at, ignored and crawl_id. tool is the fixing MCP tool when fix is
// tool; for fix code, details.what names the template element.
type IssueView struct {
ID uint `json:"id"`
Kind string `json:"kind"`
URL string `json:"url"`
Locale string `json:"locale,omitempty"`
Details map[string]any `json:"details,omitempty"`
Tool string `json:"tool"`
Severity string `json:"severity"`
Source string `json:"source"`
Fix string `json:"fix"`
FoundAt *time.Time `json:"found_at"`
Ignored bool `json:"ignored"`
CrawlID *uint `json:"crawl_id"`
}
// issueViews turns stored issues into their answer form.
func issueViews(db *gorm.DB, rows []models.Issue) ([]IssueView, error) {
fps := make([]string, 0, len(rows))
for _, r := range rows {
fps = append(fps, r.Fingerprint)
}
ignored, err := crawl.IgnoredSet(db, fps)
if err != nil {
return nil, err
}
out := make([]IssueView, 0, len(rows))
for _, r := range rows {
out = append(out, IssueView{
ID: r.ID, Kind: r.Kind, URL: r.URL, Locale: r.Locale, Details: r.DetailsMap(), Tool: r.Tool,
Severity: r.Severity, Source: r.Source, Fix: r.Fix, FoundAt: r.FoundAt,
Ignored: ignored[r.Fingerprint], CrawlID: r.CrawlID,
})
}
return out, nil
}
// ListIssues serves GET /_seo/api/v1/issues (D-13, D-14): the audit's
// current issues, errors first: the newest crawl's findings with the
// CMS-data issues stored with it, or before any crawl the CMS-data issues
// as they are now. Each issue names the tool that fixes it, or fix code.
// counts holds the number per kind.
func ListIssues(service func() *classes.Service) http.HandlerFunc {
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
issues, err := svc.Issues(ctx)
db, err := read(ctx, svc)
if err != nil {
return 0, nil, err
}
kind := r.URL.Query().Get("kind")
latest, err := crawl.Current(ctx, svc, db)
if err != nil {
return 0, nil, err
}
q := r.URL.Query()
base := crawl.CurrentScope(db.Model(&models.Issue{}), latest)
if id, err := strconv.ParseUint(strings.TrimSpace(q.Get("crawl")), 10, 64); err == nil && id > 0 {
base = db.Model(&models.Issue{}).Where("golem15_seo_issues.crawl_id = ?", id)
}
var all []models.Issue
if err := base.Order("rank ASC, kind ASC, url ASC, id ASC").Find(&all).Error; err != nil {
return 0, nil, err
}
views, err := issueViews(db, all)
if err != nil {
return 0, nil, err
}
kind := strings.TrimSpace(q.Get("kind"))
counts := map[string]int{}
out := make([]classes.Issue, 0, len(issues))
for _, issue := range issues {
counts[issue.Kind]++
if kind == "" || issue.Kind == kind {
out = append(out, issue)
out := make([]IssueView, 0, len(views))
for _, v := range views {
counts[v.Kind]++
if kind == "" || v.Kind == kind {
out = append(out, v)
}
}
return http.StatusOK, map[string]any{"data": out, "counts": counts, "total": len(out)}, nil