package api import ( "context" "errors" "net/http" "strconv" "strings" "time" "git.golem15.com/golem15/sm-seo-plugin/classes" "git.golem15.com/golem15/sm-seo-plugin/classes/crawl" "git.golem15.com/golem15/sm-seo-plugin/models" "gorm.io/gorm" ) // StartCrawlInput is the body of POST /_seo/api/v1/crawls: nothing. type StartCrawlInput struct{} // CrawlView is a crawl in an answer. type CrawlView struct { ID uint `json:"id"` Status string `json:"status"` Trigger string `json:"trigger"` TriggerName string `json:"trigger_name,omitempty"` Site string `json:"site"` MaxPages int `json:"max_pages"` Pages int `json:"pages"` LimitHit bool `json:"limit_hit"` Error string `json:"error,omitempty"` StartedAt *time.Time `json:"started_at"` FinishedAt *time.Time `json:"finished_at"` CreatedAt *time.Time `json:"created_at"` } // crawlView is the answer form of a crawl. func crawlView(c models.Crawl) CrawlView { return CrawlView{ ID: c.ID, Status: c.Status, Trigger: c.Trigger, TriggerName: c.TriggerName, Site: c.Site, MaxPages: c.MaxPages, Pages: c.Pages, LimitHit: c.LimitHit, Error: c.Error, StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, CreatedAt: c.CreatedAt, } } // StartCrawl serves POST /_seo/api/v1/crawls (D-14, P33-39): it queues a // crawl started by the administrator (trigger admin, or mcp through MCP) // and answers 202 with it. A crawl already queued or running refuses with // 409 crawls.already_running, a missing site address with 422 // crawls.no_site_url. func StartCrawl(service func() *classes.Service) http.HandlerFunc { return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { var row models.Crawl err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error { _, name, viaMCP := svc.Actor(ctx, tx) trigger := models.TriggerAdmin if viaMCP { trigger = models.TriggerMCP } var err error row, err = crawl.Start(ctx, svc, tx, trigger, name) return err }) if err != nil { return 0, nil, StartRefusal(ctx, svc, err) } return http.StatusAccepted, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil }) } // StartRefusal maps a refused start to its answer. func StartRefusal(ctx context.Context, svc *classes.Service, err error) error { var running *crawl.AlreadyRunningError switch { case errors.As(err, &running): return &apiError{Status: http.StatusConflict, Body: map[string]any{ "error": "crawls.already_running", "crawl_id": running.Crawl.ID, "message": RunningMessage(ctx, svc, running), }} case errors.Is(err, crawl.ErrNoSiteURL): return &apiError{Status: http.StatusUnprocessableEntity, Body: map[string]any{ "error": "crawls.no_site_url", "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.no_site_url", nil), }} } return err } // RunningMessage is crawls.already_running with the running crawl's start. func RunningMessage(ctx context.Context, svc *classes.Service, running *crawl.AlreadyRunningError) string { return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.already_running", map[string]string{"time": running.Since().UTC().Format("2006-01-02 15:04 UTC")}) } // RenderView is a crawl's rendered comparison in an answer (P33-55): // skipped (no golem15.seo.crawl_chrome_path), done with the pages // compared, or failed with the note; message says it in the request // locale. type RenderView struct { Status string `json:"status"` Pages int `json:"pages"` Note string `json:"note,omitempty"` Message string `json:"message"` } // renderView is the answer form of a crawl's comparison. func renderView(ctx context.Context, svc *classes.Service, c models.Crawl) RenderView { out := RenderView{Status: c.RenderStatus, Pages: c.RenderPages, Note: c.RenderNote} switch c.RenderStatus { case models.RenderDone: out.Message = svc.ChoiceCtx(ctx, "golem15.seo::lang.crawls.render_done", c.RenderPages, nil) case models.RenderFailed: out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_failed", map[string]string{"error": c.RenderNote}) default: out.Status = models.RenderSkipped out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_skipped", nil) } return out } // ListCrawlsInput is the query of GET /_seo/api/v1/crawls. type ListCrawlsInput struct { Page int `json:"page,omitempty" query:"page" jsonschema:"the page number, from 1"` PerPage int `json:"per_page,omitempty" query:"per_page" jsonschema:"rows per page, at most 200 (50 by default)"` Status string `json:"status,omitempty" query:"status" jsonschema:"queued, running, finished, failed or cancelled"` } // CrawlIDInput names one crawl. type CrawlIDInput struct { ID uint `json:"id" path:"id" jsonschema:"the crawl id"` } // DeleteCrawlsInput is the query of DELETE /_seo/api/v1/crawls. type DeleteCrawlsInput struct { IDs string `json:"ids" query:"ids" jsonschema:"the crawl ids to delete, comma-separated, e.g. 3,4; a queued or running crawl refuses the whole call"` } // ListCrawls serves GET /_seo/api/v1/crawls: the crawls, newest first, // with their issue counts and rendered comparison. func ListCrawls(service func() *classes.Service) http.HandlerFunc { return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { db, err := read(ctx, svc) if err != nil { return 0, nil, err } in := listParams(r) q := db.Model(&models.Crawl{}).Order("id DESC") if s := strings.TrimSpace(r.URL.Query().Get("status")); s != "" { q = q.Where("status = ?", s) } var rows []models.Crawl total, err := paginate(q, in, &rows) if err != nil { return 0, nil, err } ids := make([]uint, 0, len(rows)) for _, c := range rows { ids = append(ids, c.ID) } counts := map[uint]int{} if len(ids) > 0 { var cs []struct { CrawlID uint N int } if err := db.Raw(`SELECT crawl_id, COUNT(*) AS n FROM golem15_seo_issues WHERE crawl_id IN ? GROUP BY crawl_id`, ids).Scan(&cs).Error; err != nil { return 0, nil, err } for _, c := range cs { counts[c.CrawlID] = c.N } } out := make([]map[string]any, 0, len(rows)) for _, c := range rows { v := crawlMap(ctx, svc, c) v["issues"] = counts[c.ID] out = append(out, v) } return http.StatusOK, listBody(out, in, total), nil }) } // crawlMap is a crawl with its rendered comparison. func crawlMap(ctx context.Context, svc *classes.Service, c models.Crawl) map[string]any { v := crawlView(c) return map[string]any{ "id": v.ID, "status": v.Status, "trigger": v.Trigger, "trigger_name": v.TriggerName, "site": v.Site, "max_pages": v.MaxPages, "pages": v.Pages, "limit_hit": v.LimitHit, "error": v.Error, "started_at": v.StartedAt, "finished_at": v.FinishedAt, "created_at": v.CreatedAt, "cancelled_by": c.CancelledBy, "render": renderView(ctx, svc, c), } } // ShowCrawl serves GET /_seo/api/v1/crawls/{id}: one crawl with its // rendered comparison, whether the staging lock's noindex was left out, // and its issues per kind and severity. func ShowCrawl(service func() *classes.Service) http.HandlerFunc { return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { id, err := pathID(r, "id") if err != nil { return 0, nil, err } db, err := read(ctx, svc) if err != nil { return 0, nil, err } c, err := crawl.LoadCrawl(db, id) if err != nil { return 0, nil, err } var kinds []struct { Kind string `json:"kind"` Severity string `json:"severity"` Count int `json:"count"` } if err := db.Raw(`SELECT kind, severity, COUNT(*) AS count FROM golem15_seo_issues WHERE crawl_id = ? GROUP BY kind, severity ORDER BY MIN(rank), COUNT(*) DESC, kind`, id).Scan(&kinds).Error; err != nil { return 0, nil, err } out := crawlMap(ctx, svc, c) out["staging"] = svc.Staging() bySeverity := map[string]int{} total := 0 for _, k := range kinds { bySeverity[k.Severity] += k.Count total += k.Count } out["issues"] = total out["severities"] = bySeverity out["kinds"] = kinds return http.StatusOK, map[string]any{"data": out}, nil }) } // CancelCrawl serves POST /_seo/api/v1/crawls/{id}/cancel: a queued or // running crawl stops and keeps what it fetched; another answers 409 // crawls.not_running. func CancelCrawl(service func() *classes.Service) http.HandlerFunc { return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { id, err := pathID(r, "id") if err != nil { return 0, nil, err } var row models.Crawl err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error { c, err := crawl.LoadCrawl(tx, id) if err != nil { return err } _, name, _ := svc.Actor(ctx, tx) if err := crawl.Cancel(ctx, svc, tx, c, name); err != nil { return err } row, err = crawl.LoadCrawl(tx, id) return err }) if errors.Is(err, crawl.ErrNotRunning) { return 0, nil, Fail(http.StatusConflict, "crawls.not_running") } if err != nil { return 0, nil, err } return http.StatusOK, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.cancelled", nil)}, nil }) } // DeleteCrawls serves DELETE /_seo/api/v1/crawls?ids=…: the crawls go with // their pages and issues; a queued or running one refuses the whole call // with 403 crawls.delete_running. func DeleteCrawls(service func() *classes.Service) http.HandlerFunc { return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { ids, err := idList(r.URL.Query().Get("ids")) if err != nil { return 0, nil, err } n := 0 err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error { var err error n, err = crawl.DeleteCrawls(tx, ids) return err }) if errors.Is(err, crawl.ErrDeleteRunning) { return 0, nil, &apiError{Status: http.StatusForbidden, Body: map[string]any{ "error": "crawls.delete_running", "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.delete_running", nil), }} } if err != nil { return 0, nil, err } return http.StatusOK, map[string]any{"deleted": n}, nil }) } // idList reads a comma-separated id list of at most 500 ids. func idList(raw string) ([]uint, error) { var out []uint for _, part := range strings.Split(raw, ",") { part = strings.TrimSpace(part) if part == "" { continue } n, err := strconv.ParseUint(part, 10, 64) if err != nil || n == 0 { return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid") } out = append(out, uint(n)) } if len(out) == 0 || len(out) > 500 { return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid") } return out, nil }