fix: a cancelled or timed-out crawl stores what it fetched and ends

The crawl job's context ends on a cancel or at the job timeout. Storing
the pages, links and issues now runs on a context detached from it with
its own 10-minute bound, a timeout ends the crawl failed with a
time-limit error, a LoadLimits failure falls back to the defaults, and a
fetch cut short by the cancellation is recorded as canceled.

classes/crawl/crawler.go

@@ -247,6 +247,10 @@ func Crawl(ctx context.Context, f *Fetcher, opt Options) (*Result, error) {
 					return
 				}
 				p.Resp = f.Get(ctx, it.key)
+				if p.Resp.Err != "" && ctx.Err() != nil {
+					// The fetch was cut short by the cancellation.
+					p.Resp = &Response{Err: "canceled"}
+				}
 				p.FetchedAt = time.Now().UTC()
 				if p.Resp.Err == "" && p.Resp.Status == 200 && p.Resp.HTML() {
 					base, _ := url.Parse(f.Absolute(it.key))

classes/crawl/job.go

@@ -3,6 +3,7 @@ package crawl
 import (
 	"context"
 	"errors"
+	"strconv"
 	"strings"
 	"time"
 
@@ -42,6 +43,14 @@ type CrawlArgs struct {
 // Kind is CrawlKind.
 func (CrawlArgs) Kind() string { return CrawlKind }
 
+// The crawl job's time bounds (P33-60). JobTimeout is the job's deadline;
+// FinishTimeout bounds storing what a crawl found, on a context the job's
+// cancellation cannot reach.
+const (
+	JobTimeout    = 6 * time.Hour
+	FinishTimeout = 10 * time.Minute
+)
+
 // The crawl limits used until the Site audit settings say otherwise.
 const (
 	DefaultMaxPages = 5000
@@ -168,6 +177,11 @@ func LoadCrawl(db *gorm.DB, id uint) (models.Crawl, error) {
 // the crawl finished, cancelled (pages and issues fetched so far are kept)
 // or failed (with the error; what was fetched before the failure is kept).
 // A crawl that is no longer queued is left alone.
+//
+// The job's context ends on a cancel (River cancels a running job's
+// context) or at the job timeout; what was fetched is then stored on a
+// context detached from it with its own FinishTimeout bound, so a
+// cancelled or timed-out crawl keeps its results and never stays running.
 func Run(ctx context.Context, svc *classes.Service, id uint) error {
 	gdb, err := svc.GormDB()
 	if err != nil {
@@ -191,7 +205,9 @@ func Run(ctx context.Context, svc *classes.Service, id uint) error {
 		return nil
 	}
 	res, runErr := run(ctx, svc, db, row)
-	return finish(ctx, svc, db, row, res, runErr)
+	fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), FinishTimeout)
+	defer cancel()
+	return finish(fctx, svc, gdb.WithContext(fctx), row, res, runErr, ctx.Err())
 }
 
 // run fetches and checks the site of one crawl.
@@ -228,8 +244,7 @@ func run(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.Craw
 		RenderErr: renderErr,
 		RenderMax: RenderMax(svc.Config()),
 		Canceled: func(ctx context.Context) bool {
-			var status string
-			if err := db.Raw(`SELECT status FROM golem15_seo_crawls WHERE id = ?`, row.ID).Scan(&status).Error; err == nil && status == models.CrawlCancelled {
+			if cancelRequested(db, row.ID) {
 				return true
 			}
 			if m != nil && row.JobID != nil {
@@ -252,21 +267,55 @@ func run(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.Craw
 	return Crawl(ctx, f, opt)
 }
 
-// finish stores what a crawl found and writes its final state.
-func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.Crawl, res *Result, runErr error) error {
-	status := models.CrawlFinished
-	errText := ""
-	if runErr != nil {
-		status, errText = models.CrawlFailed, runErr.Error()
+// cancelRequested reports whether the crawl was cancelled: its status is
+// cancelled.
+func cancelRequested(db *gorm.DB, id uint) bool {
+	var status string
+	if err := db.Raw(`SELECT status FROM golem15_seo_crawls WHERE id = ?`, id).Scan(&status).Error; err != nil {
+		return false
 	}
-	var current string
-	_ = db.Raw(`SELECT status FROM golem15_seo_crawls WHERE id = ?`, row.ID).Scan(&current).Error
-	if current == models.CrawlCancelled || (res != nil && res.Canceled) {
-		status = models.CrawlCancelled
+	return status == models.CrawlCancelled
+}
+
+// timeLimitError is the error of a crawl that reached JobTimeout.
+func timeLimitError() string {
+	limit := JobTimeout.String()
+	if JobTimeout%time.Hour == 0 {
+		limit = strconv.Itoa(int(JobTimeout/time.Hour)) + "h"
 	}
+	return "the crawl reached its time limit of " + limit
+}
+
+// finalStatus decides a crawl's final status and error: a requested
+// cancel, or a cancelled job context, ends cancelled; the job's deadline
+// fails it with the time-limit error; another run error fails it with that
+// error; else it finished (cancelled when the crawl stopped on a cancel).
+func finalStatus(requested bool, res *Result, runErr, ctxErr error) (string, string) {
+	switch {
+	case requested:
+		return models.CrawlCancelled, ""
+	case errors.Is(ctxErr, context.DeadlineExceeded) || errors.Is(runErr, context.DeadlineExceeded):
+		return models.CrawlFailed, timeLimitError()
+	case errors.Is(ctxErr, context.Canceled) || errors.Is(runErr, context.Canceled):
+		return models.CrawlCancelled, ""
+	case runErr != nil:
+		return models.CrawlFailed, runErr.Error()
+	case res != nil && res.Canceled:
+		return models.CrawlCancelled, ""
+	}
+	return models.CrawlFinished, ""
+}
+
+// finish stores what a crawl found and writes its final state. ctx is the
+// detached, bounded context of Run, ctxErr the job context's error (nil
+// while it was alive). A LoadLimits failure falls back to the default
+// limits.
+func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.Crawl, res *Result, runErr, ctxErr error) error {
+	status, errText := finalStatus(cancelRequested(db, row.ID), res, runErr, ctxErr)
 	limits, err := LoadLimits(db)
 	if err != nil {
-		return err
+		pluginLog.Warn("golem15.seo: crawl limits, using the defaults", "crawl", row.ID, "error", err)
+		limits = Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay, Keep: DefaultKeep}
 	}
 	store := func(ctx context.Context, tx *gorm.DB) error {
 		if res == nil {