fix: a cancelled or timed-out crawl stores what it fetched and ends

The crawl job's context ends on a cancel or at the job timeout. Storing
the pages, links and issues now runs on a context detached from it with
its own 10-minute bound, a timeout ends the crawl failed with a
time-limit error, a LoadLimits failure falls back to the defaults, and a
fetch cut short by the cancellation is recorded as canceled.

crawl_lifecycle_test.go

@@ -0,0 +1,165 @@
+package seo_test
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/fetchguard"
+)
+
+// blockingSite is a fakeSite whose answer for one path waits until it is
+// released or the request's context ends (then the request fails with the
+// context's error, as a real transport does). reached is closed when that
+// request arrives.
+type blockingSite struct {
+	*fakeSite
+	path     string
+	reached  chan struct{}
+	release  chan struct{}
+	once     sync.Once
+	released sync.Once
+}
+
+func newBlockingSite(path string) *blockingSite {
+	return &blockingSite{fakeSite: newFakeSite(), path: path, reached: make(chan struct{}), release: make(chan struct{})}
+}
+
+func (s *blockingSite) RoundTrip(r *http.Request) (*http.Response, error) {
+	if r.URL.EscapedPath() == s.path {
+		s.once.Do(func() { close(s.reached) })
+		select {
+		case <-s.release:
+		case <-r.Context().Done():
+			return nil, r.Context().Err()
+		}
+	}
+	return s.fakeSite.RoundTrip(r)
+}
+
+func (s *blockingSite) unblock() { s.released.Do(func() { close(s.release) }) }
+
+// lifecycleSite plants /, /a (no title: missing_title) and /b, listed in
+// the sitemap, so the crawl fetches / and /a in its first batch and /b
+// alone in the second.
+func lifecycleSite(site *blockingSite) {
+	site.sitemap("/a", "/b")
+	site.plant(
+		doc{path: "/", title: "Home"},
+		doc{path: "/a", noTitle: true},
+		doc{path: "/b", title: "Page b"},
+	)
+}
+
+// crawlRow reads a crawl row.
+func (e *testEnv) crawlRow(t *testing.T, id uint) models.Crawl {
+	t.Helper()
+	row, err := crawl.LoadCrawl(e.db, id)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return row
+}
+
+// runCrawlAsync runs the crawl job of id on ctx in a goroutine; the
+// returned channel answers the job's error.
+func (e *testEnv) runCrawlAsync(t *testing.T, ctx context.Context, id uint) <-chan error {
+	t.Helper()
+	job := e.crawlJob(t)
+	done := make(chan error, 1)
+	go func() { done <- job.Work(ctx, crawl.CrawlArgs{CrawlID: id}) }()
+	return done
+}
+
+// waitReached waits for the blocking request.
+func waitReached(t *testing.T, site *blockingSite) {
+	t.Helper()
+	select {
+	case <-site.reached:
+	case <-time.After(20 * time.Second):
+		t.Fatal("the crawl never requested the blocking page")
+	}
+}
+
+// waitJob waits for the crawl job to return.
+func waitJob(t *testing.T, done <-chan error) error {
+	t.Helper()
+	select {
+	case err := <-done:
+		return err
+	case <-time.After(30 * time.Second):
+		t.Fatal("the crawl job did not return")
+		return nil
+	}
+}
+
+// storedPages lists the stored pages of a crawl with their status.
+func (e *testEnv) storedPages(t *testing.T, id uint) map[string]int {
+	t.Helper()
+	var rows []struct {
+		URL    string
+		Status int
+	}
+	if err := e.db.Raw(`SELECT url, status FROM golem15_seo_crawl_pages WHERE crawl_id = ?`, id).Scan(&rows).Error; err != nil {
+		t.Fatal(err)
+	}
+	out := map[string]int{}
+	for _, r := range rows {
+		out[r.URL] = r.Status
+	}
+	return out
+}
+
+// crawlIssueKinds lists the issue kinds per address of a crawl.
+func (e *testEnv) crawlIssueKinds(t *testing.T, id uint) map[string][]string {
+	t.Helper()
+	var rows []struct {
+		URL  string
+		Kind string
+	}
+	if err := e.db.Raw(`SELECT url, kind FROM golem15_seo_issues WHERE crawl_id = ?`, id).Scan(&rows).Error; err != nil {
+		t.Fatal(err)
+	}
+	out := map[string][]string{}
+	for _, r := range rows {
+		out[r.URL] = append(out[r.URL], r.Kind)
+	}
+	return out
+}
+
+// TestCrawlTimeout pins L3 CR-01: a crawl whose job context reaches its
+// deadline stores what it fetched and ends failed with the time-limit
+// error and finished_at; a later Start succeeds.
+func TestCrawlTimeout(t *testing.T) {
+	env := bootCrawlEnv(t)
+	token := env.admin(t, true)
+	if err := env.db.Exec(`UPDATE golem15_seo_settings SET crawl_delay = 100 WHERE id = 1`).Error; err != nil {
+		t.Fatal(err)
+	}
+	site := newBlockingSite("/b")
+	defer site.unblock()
+	lifecycleSite(site)
+	id := env.startCrawl(t, token)
+	ctx, cancel := context.WithTimeout(fetchguard.WithTransport(t.Context(), site), 3*time.Second)
+	defer cancel()
+	if err := waitJob(t, env.runCrawlAsync(t, ctx, id)); err != nil {
+		t.Fatalf("the timed-out job answered %v", err)
+	}
+	if !errors.Is(ctx.Err(), context.DeadlineExceeded) {
+		t.Fatal("the job returned before its deadline")
+	}
+	row := env.crawlRow(t, id)
+	if row.Status != models.CrawlFailed || !strings.Contains(row.Error, "time limit of 6h") || row.FinishedAt == nil {
+		t.Fatalf("row after the timeout = %+v", row)
+	}
+	if pages := env.storedPages(t, id); pages["/"] != 200 || pages["/a"] != 200 {
+		t.Fatalf("stored pages = %v, want / and /a", pages)
+	}
+	env.startCrawl(t, token)
+}