fix: a cancelled or timed-out crawl stores what it fetched and ends

The crawl job's context ends on a cancel or at the job timeout. Storing
the pages, links and issues now runs on a context detached from it with
its own 10-minute bound, a timeout ends the crawl failed with a
time-limit error, a LoadLimits failure falls back to the defaults, and a
fetch cut short by the cancellation is recorded as canceled.
This commit is contained in:
Jakub Zych
2026-11-04 22:50:22 +01:00
parent 9de6b2ec83
commit d7597bb1a5
5 changed files with 234 additions and 17 deletions

View File

@@ -247,6 +247,10 @@ func Crawl(ctx context.Context, f *Fetcher, opt Options) (*Result, error) {
return
}
p.Resp = f.Get(ctx, it.key)
if p.Resp.Err != "" && ctx.Err() != nil {
// The fetch was cut short by the cancellation.
p.Resp = &Response{Err: "canceled"}
}
p.FetchedAt = time.Now().UTC()
if p.Resp.Err == "" && p.Resp.Status == 200 && p.Resp.HTML() {
base, _ := url.Parse(f.Absolute(it.key))