sm-seo-plugin/controllers/api/crawls.go

package api

import (
	"context"
	"errors"
	"net/http"
	"strconv"
	"strings"
	"time"

	"git.golem15.com/golem15/sm-seo-plugin/classes"
	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
	"git.golem15.com/golem15/sm-seo-plugin/models"
	"gorm.io/gorm"
)

// StartCrawlInput is the body of POST /_seo/api/v1/crawls: nothing.
type StartCrawlInput struct{}

// CrawlView is a crawl in an answer.
type CrawlView struct {
	ID          uint       `json:"id"`
	Status      string     `json:"status"`
	Trigger     string     `json:"trigger"`
	TriggerName string     `json:"trigger_name,omitempty"`
	Site        string     `json:"site"`
	MaxPages    int        `json:"max_pages"`
	Pages       int        `json:"pages"`
	LimitHit    bool       `json:"limit_hit"`
	Error       string     `json:"error,omitempty"`
	StartedAt   *time.Time `json:"started_at"`
	FinishedAt  *time.Time `json:"finished_at"`
	CreatedAt   *time.Time `json:"created_at"`
}

// crawlView is the answer form of a crawl.
func crawlView(c models.Crawl) CrawlView {
	return CrawlView{
		ID: c.ID, Status: c.Status, Trigger: c.Trigger, TriggerName: c.TriggerName, Site: c.Site,
		MaxPages: c.MaxPages, Pages: c.Pages, LimitHit: c.LimitHit, Error: c.Error,
		StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, CreatedAt: c.CreatedAt,
	}
}

// StartCrawl serves POST /_seo/api/v1/crawls (D-14, P33-39): it queues a
// crawl started by the administrator (trigger admin, or mcp through MCP)
// and answers 202 with it. A crawl already queued or running refuses with
// 409 crawls.already_running, a missing site address with 422
// crawls.no_site_url.
func StartCrawl(service func() *classes.Service) http.HandlerFunc {
	return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
		var row models.Crawl
		err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
			_, name, viaMCP := svc.Actor(ctx, tx)
			trigger := models.TriggerAdmin
			if viaMCP {
				trigger = models.TriggerMCP
			}
			var err error
			row, err = crawl.Start(ctx, svc, tx, trigger, name)
			return err
		})
		if err != nil {
			return 0, nil, StartRefusal(ctx, svc, err)
		}
		return http.StatusAccepted, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil
	})
}

// StartRefusal maps a refused start to its answer.
func StartRefusal(ctx context.Context, svc *classes.Service, err error) error {
	var running *crawl.AlreadyRunningError
	switch {
	case errors.As(err, &running):
		return &apiError{Status: http.StatusConflict, Body: map[string]any{
			"error":    "crawls.already_running",
			"crawl_id": running.Crawl.ID,
			"message":  RunningMessage(ctx, svc, running),
		}}
	case errors.Is(err, crawl.ErrNoSiteURL):
		return &apiError{Status: http.StatusUnprocessableEntity, Body: map[string]any{
			"error":   "crawls.no_site_url",
			"message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.no_site_url", nil),
		}}
	}
	return err
}

// RunningMessage is crawls.already_running with the running crawl's start.
func RunningMessage(ctx context.Context, svc *classes.Service, running *crawl.AlreadyRunningError) string {
	return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.already_running", map[string]string{"time": running.Since().UTC().Format("2006-01-02 15:04 UTC")})
}

// RenderView is a crawl's rendered comparison in an answer (P33-55):
// skipped (no golem15.seo.crawl_chrome_path), done with the pages
// compared, or failed with the note; message says it in the request
// locale.
type RenderView struct {
	Status  string `json:"status"`
	Pages   int    `json:"pages"`
	Note    string `json:"note,omitempty"`
	Message string `json:"message"`
}

// renderView is the answer form of a crawl's comparison.
func renderView(ctx context.Context, svc *classes.Service, c models.Crawl) RenderView {
	out := RenderView{Status: c.RenderStatus, Pages: c.RenderPages, Note: c.RenderNote}
	switch c.RenderStatus {
	case models.RenderDone:
		out.Message = svc.ChoiceCtx(ctx, "golem15.seo::lang.crawls.render_done", c.RenderPages, nil)
	case models.RenderFailed:
		out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_failed", map[string]string{"error": c.RenderNote})
	default:
		out.Status = models.RenderSkipped
		out.Message = svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.render_skipped", nil)
	}
	return out
}

// ListCrawlsInput is the query of GET /_seo/api/v1/crawls.
type ListCrawlsInput struct {
	Page    int    `json:"page,omitempty" query:"page" jsonschema:"the page number, from 1"`
	PerPage int    `json:"per_page,omitempty" query:"per_page" jsonschema:"rows per page, at most 200 (50 by default)"`
	Status  string `json:"status,omitempty" query:"status" jsonschema:"queued, running, finished, failed or cancelled"`
}

// CrawlIDInput names one crawl.
type CrawlIDInput struct {
	ID uint `json:"id" path:"id" jsonschema:"the crawl id"`
}

// DeleteCrawlsInput is the query of DELETE /_seo/api/v1/crawls.
type DeleteCrawlsInput struct {
	IDs string `json:"ids" query:"ids" jsonschema:"the crawl ids to delete, comma-separated, e.g. 3,4; a queued or running crawl refuses the whole call"`
}

// ListCrawls serves GET /_seo/api/v1/crawls: the crawls, newest first,
// with their issue counts and rendered comparison.
func ListCrawls(service func() *classes.Service) http.HandlerFunc {
	return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
		db, err := read(ctx, svc)
		if err != nil {
			return 0, nil, err
		}
		in := listParams(r)
		q := db.Model(&models.Crawl{}).Order("id DESC")
		if s := strings.TrimSpace(r.URL.Query().Get("status")); s != "" {
			q = q.Where("status = ?", s)
		}
		var rows []models.Crawl
		total, err := paginate(q, in, &rows)
		if err != nil {
			return 0, nil, err
		}
		ids := make([]uint, 0, len(rows))
		for _, c := range rows {
			ids = append(ids, c.ID)
		}
		counts := map[uint]int{}
		if len(ids) > 0 {
			var cs []struct {
				CrawlID uint
				N       int
			}
			if err := db.Raw(`SELECT crawl_id, COUNT(*) AS n FROM golem15_seo_issues WHERE crawl_id IN ? GROUP BY crawl_id`, ids).Scan(&cs).Error; err != nil {
				return 0, nil, err
			}
			for _, c := range cs {
				counts[c.CrawlID] = c.N
			}
		}
		out := make([]map[string]any, 0, len(rows))
		for _, c := range rows {
			v := crawlMap(ctx, svc, c)
			v["issues"] = counts[c.ID]
			out = append(out, v)
		}
		return http.StatusOK, listBody(out, in, total), nil
	})
}

// crawlMap is a crawl with its rendered comparison.
func crawlMap(ctx context.Context, svc *classes.Service, c models.Crawl) map[string]any {
	v := crawlView(c)
	return map[string]any{
		"id": v.ID, "status": v.Status, "trigger": v.Trigger, "trigger_name": v.TriggerName, "site": v.Site,
		"max_pages": v.MaxPages, "pages": v.Pages, "limit_hit": v.LimitHit, "error": v.Error,
		"started_at": v.StartedAt, "finished_at": v.FinishedAt, "created_at": v.CreatedAt,
		"cancelled_by": c.CancelledBy, "render": renderView(ctx, svc, c),
	}
}

// ShowCrawl serves GET /_seo/api/v1/crawls/{id}: one crawl with its
// rendered comparison, whether the staging lock's noindex was left out,
// and its issues per kind and severity.
func ShowCrawl(service func() *classes.Service) http.HandlerFunc {
	return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
		id, err := pathID(r, "id")
		if err != nil {
			return 0, nil, err
		}
		db, err := read(ctx, svc)
		if err != nil {
			return 0, nil, err
		}
		c, err := crawl.LoadCrawl(db, id)
		if err != nil {
			return 0, nil, err
		}
		var kinds []struct {
			Kind     string `json:"kind"`
			Severity string `json:"severity"`
			Count    int    `json:"count"`
		}
		if err := db.Raw(`SELECT kind, severity, COUNT(*) AS count FROM golem15_seo_issues WHERE crawl_id = ? GROUP BY kind, severity ORDER BY MIN(rank), COUNT(*) DESC, kind`, id).Scan(&kinds).Error; err != nil {
			return 0, nil, err
		}
		out := crawlMap(ctx, svc, c)
		out["staging"] = svc.Staging()
		bySeverity := map[string]int{}
		total := 0
		for _, k := range kinds {
			bySeverity[k.Severity] += k.Count
			total += k.Count
		}
		out["issues"] = total
		out["severities"] = bySeverity
		out["kinds"] = kinds
		return http.StatusOK, map[string]any{"data": out}, nil
	})
}

// CancelCrawl serves POST /_seo/api/v1/crawls/{id}/cancel: a queued or
// running crawl stops and keeps what it fetched; another answers 409
// crawls.not_running.
func CancelCrawl(service func() *classes.Service) http.HandlerFunc {
	return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
		id, err := pathID(r, "id")
		if err != nil {
			return 0, nil, err
		}
		var row models.Crawl
		err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
			c, err := crawl.LoadCrawl(tx, id)
			if err != nil {
				return err
			}
			_, name, _ := svc.Actor(ctx, tx)
			if err := crawl.Cancel(ctx, svc, tx, c, name); err != nil {
				return err
			}
			row, err = crawl.LoadCrawl(tx, id)
			return err
		})
		if errors.Is(err, crawl.ErrNotRunning) {
			return 0, nil, Fail(http.StatusConflict, "crawls.not_running")
		}
		if err != nil {
			return 0, nil, err
		}
		return http.StatusOK, map[string]any{"data": crawlMap(ctx, svc, row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.cancelled", nil)}, nil
	})
}

// DeleteCrawls serves DELETE /_seo/api/v1/crawls?ids=…: the crawls go with
// their pages and issues; a queued or running one refuses the whole call
// with 403 crawls.delete_running.
func DeleteCrawls(service func() *classes.Service) http.HandlerFunc {
	return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
		ids, err := idList(r.URL.Query().Get("ids"))
		if err != nil {
			return 0, nil, err
		}
		n := 0
		err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
			var err error
			n, err = crawl.DeleteCrawls(tx, ids)
			return err
		})
		if errors.Is(err, crawl.ErrDeleteRunning) {
			return 0, nil, &apiError{Status: http.StatusForbidden, Body: map[string]any{
				"error":   "crawls.delete_running",
				"message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.delete_running", nil),
			}}
		}
		if err != nil {
			return 0, nil, err
		}
		return http.StatusOK, map[string]any{"deleted": n}, nil
	})
}

// idList reads a comma-separated id list of at most 500 ids.
func idList(raw string) ([]uint, error) {
	var out []uint
	for _, part := range strings.Split(raw, ",") {
		part = strings.TrimSpace(part)
		if part == "" {
			continue
		}
		n, err := strconv.ParseUint(part, 10, 64)
		if err != nil || n == 0 {
			return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid")
		}
		out = append(out, uint(n))
	}
	if len(out) == 0 || len(out) > 500 {
		return nil, Fail(http.StatusUnprocessableEntity, "ids_invalid")
	}
	return out, nil
}