sm-seo-plugin/crawl_lifecycle_test.go

package seo_test

import (
	"context"
	"errors"
	"net/http"
	"strings"
	"sync"
	"testing"
	"time"

	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
	"git.golem15.com/golem15/sm-seo-plugin/models"
	"git.golem15.com/golem15/summercms/modules/fetchguard"
)

// blockingSite is a fakeSite whose answer for one path waits until it is
// released or the request's context ends (then the request fails with the
// context's error, as a real transport does). reached is closed when that
// request arrives.
type blockingSite struct {
	*fakeSite
	path     string
	reached  chan struct{}
	release  chan struct{}
	once     sync.Once
	released sync.Once
}

func newBlockingSite(path string) *blockingSite {
	return &blockingSite{fakeSite: newFakeSite(), path: path, reached: make(chan struct{}), release: make(chan struct{})}
}

func (s *blockingSite) RoundTrip(r *http.Request) (*http.Response, error) {
	if r.URL.EscapedPath() == s.path {
		s.once.Do(func() { close(s.reached) })
		select {
		case <-s.release:
		case <-r.Context().Done():
			return nil, r.Context().Err()
		}
	}
	return s.fakeSite.RoundTrip(r)
}

func (s *blockingSite) unblock() { s.released.Do(func() { close(s.release) }) }

// lifecycleSite plants /, /a (no title: missing_title) and /b, listed in
// the sitemap, so the crawl fetches / and /a in its first batch and /b
// alone in the second.
func lifecycleSite(site *blockingSite) {
	site.sitemap("/a", "/b")
	site.plant(
		doc{path: "/", title: "Home"},
		doc{path: "/a", noTitle: true},
		doc{path: "/b", title: "Page b"},
	)
}

// crawlRow reads a crawl row.
func (e *testEnv) crawlRow(t *testing.T, id uint) models.Crawl {
	t.Helper()
	row, err := crawl.LoadCrawl(e.db, id)
	if err != nil {
		t.Fatal(err)
	}
	return row
}

// runCrawlAsync runs the crawl job of id on ctx in a goroutine; the
// returned channel answers the job's error.
func (e *testEnv) runCrawlAsync(t *testing.T, ctx context.Context, id uint) <-chan error {
	t.Helper()
	job := e.crawlJob(t)
	done := make(chan error, 1)
	go func() { done <- job.Work(ctx, crawl.CrawlArgs{CrawlID: id}) }()
	return done
}

// waitReached waits for the blocking request.
func waitReached(t *testing.T, site *blockingSite) {
	t.Helper()
	select {
	case <-site.reached:
	case <-time.After(20 * time.Second):
		t.Fatal("the crawl never requested the blocking page")
	}
}

// waitJob waits for the crawl job to return.
func waitJob(t *testing.T, done <-chan error) error {
	t.Helper()
	select {
	case err := <-done:
		return err
	case <-time.After(30 * time.Second):
		t.Fatal("the crawl job did not return")
		return nil
	}
}

// storedPages lists the stored pages of a crawl with their status.
func (e *testEnv) storedPages(t *testing.T, id uint) map[string]int {
	t.Helper()
	var rows []struct {
		URL    string
		Status int
	}
	if err := e.db.Raw(`SELECT url, status FROM golem15_seo_crawl_pages WHERE crawl_id = ?`, id).Scan(&rows).Error; err != nil {
		t.Fatal(err)
	}
	out := map[string]int{}
	for _, r := range rows {
		out[r.URL] = r.Status
	}
	return out
}

// crawlIssueKinds lists the issue kinds per address of a crawl.
func (e *testEnv) crawlIssueKinds(t *testing.T, id uint) map[string][]string {
	t.Helper()
	var rows []struct {
		URL  string
		Kind string
	}
	if err := e.db.Raw(`SELECT url, kind FROM golem15_seo_issues WHERE crawl_id = ?`, id).Scan(&rows).Error; err != nil {
		t.Fatal(err)
	}
	out := map[string][]string{}
	for _, r := range rows {
		out[r.URL] = append(out[r.URL], r.Kind)
	}
	return out
}

// TestCrawlTimeout pins L3 CR-01: a crawl whose job context reaches its
// deadline stores what it fetched and ends failed with the time-limit
// error and finished_at; a later Start succeeds.
func TestCrawlTimeout(t *testing.T) {
	env := bootCrawlEnv(t)
	token := env.admin(t, true)
	if err := env.db.Exec(`UPDATE golem15_seo_settings SET crawl_delay = 100 WHERE id = 1`).Error; err != nil {
		t.Fatal(err)
	}
	site := newBlockingSite("/b")
	defer site.unblock()
	lifecycleSite(site)
	id := env.startCrawl(t, token)
	ctx, cancel := context.WithTimeout(fetchguard.WithTransport(t.Context(), site), 3*time.Second)
	defer cancel()
	if err := waitJob(t, env.runCrawlAsync(t, ctx, id)); err != nil {
		t.Fatalf("the timed-out job answered %v", err)
	}
	if !errors.Is(ctx.Err(), context.DeadlineExceeded) {
		t.Fatal("the job returned before its deadline")
	}
	row := env.crawlRow(t, id)
	if row.Status != models.CrawlFailed || !strings.Contains(row.Error, "time limit of 6h") || row.FinishedAt == nil {
		t.Fatalf("row after the timeout = %+v", row)
	}
	if pages := env.storedPages(t, id); pages["/"] != 200 || pages["/a"] != 200 {
		t.Fatalf("stored pages = %v, want / and /a", pages)
	}
	env.startCrawl(t, token)
}