package seo_test import ( "context" "errors" "net/http" "strings" "sync" "testing" "time" "git.golem15.com/golem15/sm-seo-plugin/classes/crawl" "git.golem15.com/golem15/sm-seo-plugin/models" "git.golem15.com/golem15/summercms/modules/fetchguard" ) // blockingSite is a fakeSite whose answer for one path waits until it is // released or the request's context ends (then the request fails with the // context's error, as a real transport does). reached is closed when that // request arrives. type blockingSite struct { *fakeSite path string reached chan struct{} release chan struct{} once sync.Once released sync.Once } func newBlockingSite(path string) *blockingSite { return &blockingSite{fakeSite: newFakeSite(), path: path, reached: make(chan struct{}), release: make(chan struct{})} } func (s *blockingSite) RoundTrip(r *http.Request) (*http.Response, error) { if r.URL.EscapedPath() == s.path { s.once.Do(func() { close(s.reached) }) select { case <-s.release: case <-r.Context().Done(): return nil, r.Context().Err() } } return s.fakeSite.RoundTrip(r) } func (s *blockingSite) unblock() { s.released.Do(func() { close(s.release) }) } // lifecycleSite plants /, /a (no title: missing_title) and /b, listed in // the sitemap, so the crawl fetches / and /a in its first batch and /b // alone in the second. func lifecycleSite(site *blockingSite) { site.sitemap("/a", "/b") site.plant( doc{path: "/", title: "Home"}, doc{path: "/a", noTitle: true}, doc{path: "/b", title: "Page b"}, ) } // crawlRow reads a crawl row. func (e *testEnv) crawlRow(t *testing.T, id uint) models.Crawl { t.Helper() row, err := crawl.LoadCrawl(e.db, id) if err != nil { t.Fatal(err) } return row } // runCrawlAsync runs the crawl job of id on ctx in a goroutine; the // returned channel answers the job's error. func (e *testEnv) runCrawlAsync(t *testing.T, ctx context.Context, id uint) <-chan error { t.Helper() job := e.crawlJob(t) done := make(chan error, 1) go func() { done <- job.Work(ctx, crawl.CrawlArgs{CrawlID: id}) }() return done } // waitReached waits for the blocking request. func waitReached(t *testing.T, site *blockingSite) { t.Helper() select { case <-site.reached: case <-time.After(20 * time.Second): t.Fatal("the crawl never requested the blocking page") } } // waitJob waits for the crawl job to return. func waitJob(t *testing.T, done <-chan error) error { t.Helper() select { case err := <-done: return err case <-time.After(30 * time.Second): t.Fatal("the crawl job did not return") return nil } } // storedPages lists the stored pages of a crawl with their status. func (e *testEnv) storedPages(t *testing.T, id uint) map[string]int { t.Helper() var rows []struct { URL string Status int } if err := e.db.Raw(`SELECT url, status FROM golem15_seo_crawl_pages WHERE crawl_id = ?`, id).Scan(&rows).Error; err != nil { t.Fatal(err) } out := map[string]int{} for _, r := range rows { out[r.URL] = r.Status } return out } // crawlIssueKinds lists the issue kinds per address of a crawl. func (e *testEnv) crawlIssueKinds(t *testing.T, id uint) map[string][]string { t.Helper() var rows []struct { URL string Kind string } if err := e.db.Raw(`SELECT url, kind FROM golem15_seo_issues WHERE crawl_id = ?`, id).Scan(&rows).Error; err != nil { t.Fatal(err) } out := map[string][]string{} for _, r := range rows { out[r.URL] = append(out[r.URL], r.Kind) } return out } // TestCrawlTimeout pins L3 CR-01: a crawl whose job context reaches its // deadline stores what it fetched and ends failed with the time-limit // error and finished_at; a later Start succeeds. func TestCrawlTimeout(t *testing.T) { env := bootCrawlEnv(t) token := env.admin(t, true) if err := env.db.Exec(`UPDATE golem15_seo_settings SET crawl_delay = 100 WHERE id = 1`).Error; err != nil { t.Fatal(err) } site := newBlockingSite("/b") defer site.unblock() lifecycleSite(site) id := env.startCrawl(t, token) ctx, cancel := context.WithTimeout(fetchguard.WithTransport(t.Context(), site), 3*time.Second) defer cancel() if err := waitJob(t, env.runCrawlAsync(t, ctx, id)); err != nil { t.Fatalf("the timed-out job answered %v", err) } if !errors.Is(ctx.Err(), context.DeadlineExceeded) { t.Fatal("the job returned before its deadline") } row := env.crawlRow(t, id) if row.Status != models.CrawlFailed || !strings.Contains(row.Error, "time limit of 6h") || row.FinishedAt == nil { t.Fatalf("row after the timeout = %+v", row) } if pages := env.storedPages(t, id); pages["/"] != 200 || pages["/a"] != 200 { t.Fatalf("stored pages = %v, want / and /a", pages) } env.startCrawl(t, token) }