cd7182395f
Cancelling a running crawl records cancel_requested_at and leaves the row running until its job stops and writes cancelled, so no second crawl starts and the row cannot be deleted or pruned meanwhile. The job writes heartbeat_at every minute; a running crawl without a heartbeat for 10 minutes is ended by the next start or delete.
313 lines
11 KiB
Go
313 lines
11 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
|
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
|
|
"git.golem15.com/golem15/sm-seo-plugin/models"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// StartCrawlInput is the body of POST /_seo/api/v1/crawls: nothing.
|
|
type StartCrawlInput struct{}
|
|
|
|
// CrawlView is a crawl in an answer.
|
|
type CrawlView struct {
|
|
ID uint `json:"id"`
|
|
Status string `json:"status"`
|
|
Trigger string `json:"trigger"`
|
|
TriggerName string `json:"trigger_name,omitempty"`
|
|
Site string `json:"site"`
|
|
MaxPages int `json:"max_pages"`
|
|
Pages int `json:"pages"`
|
|
LimitHit bool `json:"limit_hit"`
|
|
Error string `json:"error,omitempty"`
|
|
StartedAt *time.Time `json:"started_at"`
|
|
FinishedAt *time.Time `json:"finished_at"`
|
|
CreatedAt *time.Time `json:"created_at"`
|
|
}
|
|
|
|
// crawlView is the answer form of a crawl.
|
|
func crawlView(c models.Crawl) CrawlView {
|
|
return CrawlView{
|
|
ID: c.ID, Status: c.Status, Trigger: c.Trigger, TriggerName: c.TriggerName, Site: c.Site,
|
|
MaxPages: c.MaxPages, Pages: c.Pages, LimitHit: c.LimitHit, Error: c.Error,
|
|
StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, CreatedAt: c.CreatedAt,
|
|
}
|
|
}
|
|
|
|
// StartCrawl serves POST /_seo/api/v1/crawls (D-14, P33-39): it queues a
|
|
// crawl started by the administrator (trigger admin, or mcp through MCP)
|
|
// and answers 202 with it. A crawl already queued or running refuses with
|
|
// 409 crawls.already_running, a missing site address with 422
|
|
// crawls.no_site_url.
|
|
func StartCrawl(service func() *classes.Service) http.HandlerFunc {
|
|
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
|
var row models.Crawl
|
|
err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
|
|
_, name, viaMCP := svc.Actor(ctx, tx)
|
|
trigger := models.TriggerAdmin
|
|
if viaMCP {
|
|
trigger = models.TriggerMCP
|
|
}
|
|
var err error
|
|
row, err = crawl.Start(ctx, svc, tx, trigger, name)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return 0, nil, StartRefusal(ctx, svc, err)
|
|
}
|
|
return http.StatusAccepted, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil
|
|
})
|
|
}
|
|
|
|
// StartRefusal maps a refused start to its answer.
|
|
func StartRefusal(ctx context.Context, svc *classes.Service, err error) error {
|
|
var running *crawl.AlreadyRunningError
|
|
switch {
|
|
case errors.As(err, &running):
|
|
return &apiError{Status: http.StatusConflict, Body: map[string]any{
|
|
"error": "crawls.already_running",
|
|
"crawl_id": running.Crawl.ID,
|
|
"message": RunningMessage(ctx, svc, running),
|
|
}}
|
|
case errors.Is(err, crawl.ErrNoSiteURL):
|
|
return &apiError{Status: http.StatusUnprocessableEntity, Body: map[string]any{
|
|
"error": "crawls.no_site_url",
|
|
"message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.no_site_url", nil),
|
|
}}
|
|
}
|
|
return err
|
|
}
|
|
|
|
// RunningMessage is crawls.already_running with the running crawl's start.
|
|
func RunningMessage(ctx context.Context, svc *classes.Service, running *crawl.AlreadyRunningError) string {
|
|
return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.already_running", map[string]string{"time": running.Since().UTC().Format("2006-01-02 15:04 UTC")})
|
|
}
|
|
|
|
// RenderView is a crawl's rendered comparison in an answer (P33-55):
|
|
// skipped (no golem15.seo.crawl_chrome_path), done with the pages
|
|
// compared, or failed with the note; message says it in the request
|
|
// locale.
|
|
type RenderView struct {
|
|
Status string `json:"status"`
|
|
Pages int `json:"pages"`
|
|
Note string `json:"note,omitempty"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
// renderView is the answer form of a crawl's comparison.
|
|
func renderView(ctx context.Context, svc *classes.Service, c models.Crawl) RenderView {
|
|
out := RenderView{Status: c.RenderStatus, Pages: c.RenderPages, Note: c.RenderNote}
|
|
switch c.RenderStatus {
|
|
case models.RenderDone:
|
|
out.Message = svc.ChoiceCtx(ctx, "golem15.seo::lang.crawls.render_done", c.RenderPages, nil)
|
|
case models.RenderFailed:
|
|
out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_failed", map[string]string{"error": c.RenderNote})
|
|
default:
|
|
out.Status = models.RenderSkipped
|
|
out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_skipped", nil)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ListCrawlsInput is the query of GET /_seo/api/v1/crawls.
|
|
type ListCrawlsInput struct {
|
|
Page int `json:"page,omitempty" query:"page" jsonschema:"the page number, from 1"`
|
|
PerPage int `json:"per_page,omitempty" query:"per_page" jsonschema:"rows per page, at most 200 (50 by default)"`
|
|
Status string `json:"status,omitempty" query:"status" jsonschema:"queued, running, finished, failed or cancelled"`
|
|
}
|
|
|
|
// CrawlIDInput names one crawl.
|
|
type CrawlIDInput struct {
|
|
ID uint `json:"id" path:"id" jsonschema:"the crawl id"`
|
|
}
|
|
|
|
// DeleteCrawlsInput is the query of DELETE /_seo/api/v1/crawls.
|
|
type DeleteCrawlsInput struct {
|
|
IDs string `json:"ids" query:"ids" jsonschema:"the crawl ids to delete, comma-separated, e.g. 3,4; a queued or running crawl refuses the whole call"`
|
|
}
|
|
|
|
// ListCrawls serves GET /_seo/api/v1/crawls: the crawls, newest first,
|
|
// with their issue counts and rendered comparison.
|
|
func ListCrawls(service func() *classes.Service) http.HandlerFunc {
|
|
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
|
db, err := read(ctx, svc)
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
in := listParams(r)
|
|
q := db.Model(&models.Crawl{}).Order("id DESC")
|
|
if s := strings.TrimSpace(r.URL.Query().Get("status")); s != "" {
|
|
q = q.Where("status = ?", s)
|
|
}
|
|
var rows []models.Crawl
|
|
total, err := paginate(q, in, &rows)
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
ids := make([]uint, 0, len(rows))
|
|
for _, c := range rows {
|
|
ids = append(ids, c.ID)
|
|
}
|
|
counts := map[uint]int{}
|
|
if len(ids) > 0 {
|
|
var cs []struct {
|
|
CrawlID uint
|
|
N int
|
|
}
|
|
if err := db.Raw(`SELECT crawl_id, COUNT(*) AS n FROM golem15_seo_issues WHERE crawl_id IN ? GROUP BY crawl_id`, ids).Scan(&cs).Error; err != nil {
|
|
return 0, nil, err
|
|
}
|
|
for _, c := range cs {
|
|
counts[c.CrawlID] = c.N
|
|
}
|
|
}
|
|
out := make([]map[string]any, 0, len(rows))
|
|
for _, c := range rows {
|
|
v := crawlMap(ctx, svc, c)
|
|
v["issues"] = counts[c.ID]
|
|
out = append(out, v)
|
|
}
|
|
return http.StatusOK, listBody(out, in, total), nil
|
|
})
|
|
}
|
|
|
|
// crawlMap is a crawl with its rendered comparison.
|
|
func crawlMap(ctx context.Context, svc *classes.Service, c models.Crawl) map[string]any {
|
|
v := crawlView(c)
|
|
return map[string]any{
|
|
"id": v.ID, "status": v.Status, "trigger": v.Trigger, "trigger_name": v.TriggerName, "site": v.Site,
|
|
"max_pages": v.MaxPages, "pages": v.Pages, "limit_hit": v.LimitHit, "error": v.Error,
|
|
"started_at": v.StartedAt, "finished_at": v.FinishedAt, "created_at": v.CreatedAt,
|
|
"cancelled_by": c.CancelledBy, "cancel_requested": c.CancelRequestedAt != nil, "render": renderView(ctx, svc, c),
|
|
}
|
|
}
|
|
|
|
// ShowCrawl serves GET /_seo/api/v1/crawls/{id}: one crawl with its
|
|
// rendered comparison, whether the staging lock's noindex was left out,
|
|
// and its issues per kind and severity.
|
|
func ShowCrawl(service func() *classes.Service) http.HandlerFunc {
|
|
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
|
id, err := pathID(r, "id")
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
db, err := read(ctx, svc)
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
c, err := crawl.LoadCrawl(db, id)
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
var kinds []struct {
|
|
Kind string `json:"kind"`
|
|
Severity string `json:"severity"`
|
|
Count int `json:"count"`
|
|
}
|
|
if err := db.Raw(`SELECT kind, severity, COUNT(*) AS count FROM golem15_seo_issues WHERE crawl_id = ? GROUP BY kind, severity ORDER BY MIN(rank), COUNT(*) DESC, kind`, id).Scan(&kinds).Error; err != nil {
|
|
return 0, nil, err
|
|
}
|
|
out := crawlMap(ctx, svc, c)
|
|
out["staging"] = svc.Staging()
|
|
bySeverity := map[string]int{}
|
|
total := 0
|
|
for _, k := range kinds {
|
|
bySeverity[k.Severity] += k.Count
|
|
total += k.Count
|
|
}
|
|
out["issues"] = total
|
|
out["severities"] = bySeverity
|
|
out["kinds"] = kinds
|
|
return http.StatusOK, map[string]any{"data": out}, nil
|
|
})
|
|
}
|
|
|
|
// CancelCrawl serves POST /_seo/api/v1/crawls/{id}/cancel: a queued crawl
|
|
// ends cancelled at once; a running one answers with cancel_requested true
|
|
// and stays running until its job stops, then ends cancelled and keeps
|
|
// what it fetched; another answers 409 crawls.not_running.
|
|
func CancelCrawl(service func() *classes.Service) http.HandlerFunc {
|
|
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
|
id, err := pathID(r, "id")
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
var row models.Crawl
|
|
err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
|
|
c, err := crawl.LoadCrawl(tx, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, name, _ := svc.Actor(ctx, tx)
|
|
if err := crawl.Cancel(ctx, svc, tx, c, name); err != nil {
|
|
return err
|
|
}
|
|
row, err = crawl.LoadCrawl(tx, id)
|
|
return err
|
|
})
|
|
if errors.Is(err, crawl.ErrNotRunning) {
|
|
return 0, nil, Fail(http.StatusConflict, "crawls.not_running")
|
|
}
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
return http.StatusOK, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.cancelled", nil)}, nil
|
|
})
|
|
}
|
|
|
|
// DeleteCrawls serves DELETE /_seo/api/v1/crawls?ids=…: the crawls go with
|
|
// their pages and issues; a queued or running one refuses the whole call
|
|
// with 403 crawls.delete_running.
|
|
func DeleteCrawls(service func() *classes.Service) http.HandlerFunc {
|
|
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
|
ids, err := idList(r.URL.Query().Get("ids"))
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
n := 0
|
|
err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
|
|
var err error
|
|
n, err = crawl.DeleteCrawls(tx, ids)
|
|
return err
|
|
})
|
|
if errors.Is(err, crawl.ErrDeleteRunning) {
|
|
return 0, nil, &apiError{Status: http.StatusForbidden, Body: map[string]any{
|
|
"error": "crawls.delete_running",
|
|
"message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.delete_running", nil),
|
|
}}
|
|
}
|
|
if err != nil {
|
|
return 0, nil, err
|
|
}
|
|
return http.StatusOK, map[string]any{"deleted": n}, nil
|
|
})
|
|
}
|
|
|
|
// idList reads a comma-separated id list of at most 500 ids.
|
|
func idList(raw string) ([]uint, error) {
|
|
var out []uint
|
|
for _, part := range strings.Split(raw, ",") {
|
|
part = strings.TrimSpace(part)
|
|
if part == "" {
|
|
continue
|
|
}
|
|
n, err := strconv.ParseUint(part, 10, 64)
|
|
if err != nil || n == 0 {
|
|
return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid")
|
|
}
|
|
out = append(out, uint(n))
|
|
}
|
|
if len(out) == 0 || len(out) > 500 {
|
|
return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid")
|
|
}
|
|
return out, nil
|
|
}
|