c2b61bf8e9
- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37) - classes/crawl: fetchguard client on the site host and its www twin, sitemap index and robots.txt seeds, two workers with a delay, limits - h1_missing and h1_multiple with the P33-38 fix route - POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues answers the stored issues with id, severity, source, fix, found_at, ignored and crawl_id
91 lines
3.2 KiB
Go
91 lines
3.2 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
|
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
|
|
"git.golem15.com/golem15/sm-seo-plugin/models"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// StartCrawlInput is the body of POST /_seo/api/v1/crawls: nothing.
|
|
type StartCrawlInput struct{}
|
|
|
|
// CrawlView is a crawl in an answer.
|
|
type CrawlView struct {
|
|
ID uint `json:"id"`
|
|
Status string `json:"status"`
|
|
Trigger string `json:"trigger"`
|
|
TriggerName string `json:"trigger_name,omitempty"`
|
|
Site string `json:"site"`
|
|
MaxPages int `json:"max_pages"`
|
|
Pages int `json:"pages"`
|
|
LimitHit bool `json:"limit_hit"`
|
|
Error string `json:"error,omitempty"`
|
|
StartedAt *time.Time `json:"started_at"`
|
|
FinishedAt *time.Time `json:"finished_at"`
|
|
CreatedAt *time.Time `json:"created_at"`
|
|
}
|
|
|
|
// crawlView is the answer form of a crawl.
|
|
func crawlView(c models.Crawl) CrawlView {
|
|
return CrawlView{
|
|
ID: c.ID, Status: c.Status, Trigger: c.Trigger, TriggerName: c.TriggerName, Site: c.Site,
|
|
MaxPages: c.MaxPages, Pages: c.Pages, LimitHit: c.LimitHit, Error: c.Error,
|
|
StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, CreatedAt: c.CreatedAt,
|
|
}
|
|
}
|
|
|
|
// StartCrawl serves POST /_seo/api/v1/crawls (D-14, P33-39): it queues a
|
|
// crawl started by the administrator (trigger admin, or mcp through MCP)
|
|
// and answers 202 with it. A crawl already queued or running refuses with
|
|
// 409 crawls.already_running, a missing site address with 422
|
|
// crawls.no_site_url.
|
|
func StartCrawl(service func() *classes.Service) http.HandlerFunc {
|
|
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
|
var row models.Crawl
|
|
err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
|
|
_, name, viaMCP := svc.Actor(ctx, tx)
|
|
trigger := models.TriggerAdmin
|
|
if viaMCP {
|
|
trigger = models.TriggerMCP
|
|
}
|
|
var err error
|
|
row, err = crawl.Start(ctx, svc, tx, trigger, name)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return 0, nil, StartRefusal(ctx, svc, err)
|
|
}
|
|
return http.StatusAccepted, map[string]any{"data": crawlView(row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil
|
|
})
|
|
}
|
|
|
|
// StartRefusal maps a refused start to its answer.
|
|
func StartRefusal(ctx context.Context, svc *classes.Service, err error) error {
|
|
var running *crawl.AlreadyRunningError
|
|
switch {
|
|
case errors.As(err, &running):
|
|
return &apiError{Status: http.StatusConflict, Body: map[string]any{
|
|
"error": "crawls.already_running",
|
|
"crawl_id": running.Crawl.ID,
|
|
"message": RunningMessage(ctx, svc, running),
|
|
}}
|
|
case errors.Is(err, crawl.ErrNoSiteURL):
|
|
return &apiError{Status: http.StatusUnprocessableEntity, Body: map[string]any{
|
|
"error": "crawls.no_site_url",
|
|
"message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.no_site_url", nil),
|
|
}}
|
|
}
|
|
return err
|
|
}
|
|
|
|
// RunningMessage is crawls.already_running with the running crawl's start.
|
|
func RunningMessage(ctx context.Context, svc *classes.Service, running *crawl.AlreadyRunningError) string {
|
|
return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.already_running", map[string]string{"time": running.Since().UTC().Format("2006-01-02 15:04 UTC")})
|
|
}
|