sm-seo-plugin/crawl_site_test.go

package seo_test

import (
	"context"
	"encoding/json"
	"html"
	"io"
	"net/http"
	"net/url"
	"slices"
	"sort"
	"strings"
	"sync"
	"testing"

	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
	"git.golem15.com/golem15/summercms/modules/backpack"
	"git.golem15.com/golem15/summercms/modules/conga"
	"git.golem15.com/golem15/summercms/modules/fetchguard"
	"git.golem15.com/golem15/summercms/modules/pact"
	"git.golem15.com/golem15/summercms/modules/party"
	"git.golem15.com/golem15/summercms/modules/signpost"
)

// The test site the crawler fetches: a fake http.RoundTripper handed to
// the crawler through fetchguard.WithTransport, so the crawler's real
// requests (URL checks, User-Agent, no redirects followed) are asserted
// offline. Paths it does not know answer 404.

// sitePage is one answer of the test site.
type sitePage struct {
	status int
	body   string
	header map[string]string
}

// fakeSite serves the test site and records every request URL.
type fakeSite struct {
	mu       sync.Mutex
	pages    map[string]sitePage
	requests []string
	agents   []string
}

func newFakeSite() *fakeSite { return &fakeSite{pages: map[string]sitePage{}} }

// set serves body with status at path (path with query).
func (s *fakeSite) set(path string, status int, body string, header ...string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	h := map[string]string{"Content-Type": "text/html; charset=utf-8"}
	for i := 0; i+1 < len(header); i += 2 {
		h[header[i]] = header[i+1]
	}
	s.pages[path] = sitePage{status: status, body: body, header: h}
}

// html serves a 200 HTML page.
func (s *fakeSite) html(path, body string) { s.set(path, http.StatusOK, body) }

func (s *fakeSite) RoundTrip(r *http.Request) (*http.Response, error) {
	s.mu.Lock()
	s.requests = append(s.requests, r.URL.String())
	s.agents = append(s.agents, r.Header.Get("User-Agent"))
	key := r.URL.EscapedPath()
	if r.URL.RawQuery != "" {
		key += "?" + r.URL.RawQuery
	}
	p, ok := s.pages[key]
	s.mu.Unlock()
	if !ok || !strings.EqualFold(r.URL.Hostname(), "www.example.test") {
		p = sitePage{status: http.StatusNotFound, body: "<html><body>Not found</body></html>", header: map[string]string{"Content-Type": "text/html"}}
	}
	header := http.Header{}
	for k, v := range p.header {
		header.Set(k, v)
	}
	return &http.Response{StatusCode: p.status, Header: header, Body: io.NopCloser(strings.NewReader(p.body)), Request: r}, nil
}

// requested reports whether any request went to an URL containing part.
func (s *fakeSite) requested(part string) bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	for _, u := range s.requests {
		if strings.Contains(u, part) {
			return true
		}
	}
	return false
}

// sitemap serves /sitemap.xml as an index of one file listing paths.
func (s *fakeSite) sitemap(paths ...string) {
	s.set("/sitemap.xml", http.StatusOK, `<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>`+siteURL+`/sitemap-pages-1.xml</loc></sitemap></sitemapindex>`, "Content-Type", "application/xml")
	var b strings.Builder
	b.WriteString(`<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`)
	for _, p := range paths {
		b.WriteString("<url><loc>" + siteURL + html.EscapeString(p) + "</loc></url>")
	}
	b.WriteString("</urlset>")
	s.set("/sitemap-pages-1.xml", http.StatusOK, b.String(), "Content-Type", "application/xml")
}

// doc is one planted page: a complete, clean document (unique title and
// description, a self canonical, one H1 matching the title) that a test
// mutates for the kind it plants.
type doc struct {
	path        string
	lang        string
	title       string
	noTitle     bool
	description string
	noDesc      bool
	canonical   string
	noCanonical bool
	robots      string
	head        string
	h1          []string
	body        string
	links       []string
}

// render writes the document's HTML.
func (d doc) render() string {
	var b strings.Builder
	lang := d.lang
	if lang == "" {
		lang = "en"
	}
	b.WriteString(`<!doctype html><html lang="` + lang + `"><head><meta charset="utf-8">`)
	title := d.title
	if title == "" {
		title = "Page " + d.path
	}
	if !d.noTitle {
		b.WriteString("<title>" + html.EscapeString(title) + "</title>")
	}
	if !d.noDesc {
		desc := d.description
		if desc == "" {
			desc = "About " + d.path
		}
		b.WriteString(`<meta name="description" content="` + html.EscapeString(desc) + `">`)
	}
	if !d.noCanonical {
		c := d.canonical
		if c == "" {
			c = siteURL + d.path
		}
		b.WriteString(`<link rel="canonical" href="` + html.EscapeString(c) + `">`)
	}
	if d.robots != "" {
		b.WriteString(`<meta name="robots" content="` + d.robots + `">`)
	}
	b.WriteString(d.head + "</head><body><main>")
	h1 := d.h1
	if h1 == nil {
		h1 = []string{title}
	}
	for _, h := range h1 {
		b.WriteString("<h1>" + html.EscapeString(h) + "</h1>")
	}
	b.WriteString(d.body)
	for _, l := range d.links {
		b.WriteString(`<p><a href="` + l + `">Read about ` + html.EscapeString(strings.Trim(l, "/")) + `</a></p>`)
	}
	b.WriteString("</main></body></html>")
	return b.String()
}

// plant serves the documents on the site.
func (s *fakeSite) plant(docs ...doc) {
	for _, d := range docs {
		s.html(d.path, d.render())
	}
}

// The fake content plugin acme.crawl: in-memory records behind two types
// with descriptor facts and fixing tools, so the audit's content routes
// (P33-38) are exercised. acme-crawl-post (/c/:slug) names edit, headings
// and image_alt tools; acme-crawl-category (/cat/:slug) names only a
// headings tool.
const (
	crawlPostType     = "acme-crawl-post"
	crawlCategoryType = "acme-crawl-category"
)

// crawlRecord is one in-memory record.
type crawlRecord struct {
	id         string
	path       string
	title      string
	words      int
	headings   []pact.ContentHeading
	images     []pact.ContentImage
	topHeading int
	parents    []pact.ContentRef
}

var (
	crawlRecordsMu sync.Mutex
	crawlRecords   = map[string][]crawlRecord{}
)

// setCrawlRecords replaces the records of the fake types for one test.
func setCrawlRecords(t *testing.T, byType map[string][]crawlRecord) {
	t.Helper()
	crawlRecordsMu.Lock()
	crawlRecords = byType
	crawlRecordsMu.Unlock()
	t.Cleanup(func() {
		crawlRecordsMu.Lock()
		crawlRecords = map[string][]crawlRecord{}
		crawlRecordsMu.Unlock()
	})
}

func crawlRecordsOf(typeKey string) []crawlRecord {
	crawlRecordsMu.Lock()
	defer crawlRecordsMu.Unlock()
	return slices.Clone(crawlRecords[typeKey])
}

func (r crawlRecord) content() pact.ContentRecord {
	words := r.words
	var wp *int
	if words > 0 {
		wp = &words
	}
	return pact.ContentRecord{
		ID: r.id, Published: true, TopHeading: r.topHeading, Parents: r.parents,
		Locales: []pact.ContentLocale{{Locale: "en", URL: r.path, Title: r.title, Words: wp, Headings: r.headings, Images: r.images}},
	}
}

type crawlContentPlugin struct{}

func (crawlContentPlugin) ID() string                   { return "acme.crawl" }
func (crawlContentPlugin) Requires() []string           { return nil }
func (crawlContentPlugin) Register(*backpack.App) error { return nil }
func (crawlContentPlugin) Boot(*backpack.App) error     { return nil }
func (crawlContentPlugin) MenuItemTypes() []pact.MenuItemType {
	return []pact.MenuItemType{
		crawlType(crawlPostType, "/c/:slug", map[string]string{
			pact.ContentToolEdit: "acme_edit_post", pact.ContentToolHeadings: "acme_fix_headings", pact.ContentToolImageAlt: "acme_set_alt",
		}),
		crawlType(crawlCategoryType, "/cat/:slug", map[string]string{pact.ContentToolHeadings: "acme_category_headings"}),
	}
}

// crawlType is one fake type over the in-memory records.
func crawlType(key, pattern string, tools map[string]string) pact.MenuItemType {
	find := func(match func(crawlRecord) bool) *crawlRecord {
		for _, r := range crawlRecordsOf(key) {
			if match(r) {
				return &r
			}
		}
		return nil
	}
	return pact.MenuItemType{
		Key: key, Label: key, DefaultPattern: pattern, Placeholders: []string{"slug"},
		Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { return nil, nil },
		Content: &pact.ContentDescriptor{
			Model: "acme_crawl_" + strings.ReplaceAll(key, "-", "_"),
			Tools: tools,
			Describe: func(_ context.Context, in pact.ContentDescribe) (*pact.ContentRecord, error) {
				r := find(func(r crawlRecord) bool { return r.id == in.ID })
				if r == nil {
					return nil, nil
				}
				rec := r.content()
				return &rec, nil
			},
			Records: func(_ context.Context, in pact.ContentList) (pact.ContentPage, error) {
				var out pact.ContentPage
				if in.After != "" {
					return out, nil
				}
				for _, r := range crawlRecordsOf(key) {
					out.Records = append(out.Records, r.content())
				}
				return out, nil
			},
			Match: func(_ context.Context, in pact.ContentMatch) (*pact.ContentMatchResult, error) {
				if _, ok := signpost.MatchPattern(in.Pattern, in.Path); !ok {
					return nil, nil
				}
				r := find(func(r crawlRecord) bool { return r.path == in.Path })
				if r == nil {
					return nil, nil
				}
				return &pact.ContentMatchResult{ID: r.id, Locale: "en", Exact: true}, nil
			},
		},
	}
}

var _ pact.HasMenuItemTypes = crawlContentPlugin{}

func init() {
	party.Register(crawlContentPlugin{})
}

// bootCrawlEnv boots acme.pages, acme.crawl and golem15.seo.
func bootCrawlEnv(t *testing.T, extraEnv ...string) *testEnv {
	t.Helper()
	return bootPlugins(t, []string{"acme.pages", "acme.crawl", "golem15.seo"}, extraEnv...)
}

// crawlJob returns the SEO plugin's crawl job.
func (e *testEnv) crawlJob(t *testing.T) pact.Job {
	t.Helper()
	for _, j := range e.seoPlugin(t).Jobs() {
		if info, ok := conga.Describe(j); ok && info.Kind == crawl.CrawlKind {
			return j
		}
	}
	t.Fatal("the SEO plugin registers no crawl job")
	return nil
}

// siteCtx is a context whose crawler requests go to site.
func siteCtx(t *testing.T, site *fakeSite) context.Context {
	return fetchguard.WithTransport(t.Context(), site)
}

// startCrawl starts a crawl through the API and returns its id.
func (e *testEnv) startCrawl(t *testing.T, token string) uint {
	t.Helper()
	rec := e.expect(t, http.StatusAccepted, http.MethodPost, "/_seo/api/v1/crawls", token, map[string]any{})
	var out struct {
		Data struct {
			ID uint `json:"id"`
		} `json:"data"`
	}
	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil || out.Data.ID == 0 {
		t.Fatalf("start crawl: %s (%v)", rec.Body.String(), err)
	}
	return out.Data.ID
}

// runCrawl runs the crawl job of id against the site.
func (e *testEnv) runCrawl(t *testing.T, ctx context.Context, id uint) {
	t.Helper()
	if err := e.crawlJob(t).Work(ctx, crawl.CrawlArgs{CrawlID: id}); err != nil {
		t.Fatalf("crawl job: %v", err)
	}
}

// crawlOnce starts a crawl and runs it against the site.
func (e *testEnv) crawlOnce(t *testing.T, token string, ctx context.Context) uint {
	t.Helper()
	id := e.startCrawl(t, token)
	e.runCrawl(t, ctx, id)
	return id
}

// apiIssue is one issue of GET /_seo/api/v1/issues.
type apiIssue struct {
	ID       uint           `json:"id"`
	Kind     string         `json:"kind"`
	URL      string         `json:"url"`
	Locale   string         `json:"locale"`
	Details  map[string]any `json:"details"`
	Tool     string         `json:"tool"`
	Severity string         `json:"severity"`
	Source   string         `json:"source"`
	Fix      string         `json:"fix"`
	Ignored  bool           `json:"ignored"`
	CrawlID  *uint          `json:"crawl_id"`
}

// issues lists the audit's issues through the API (query appended).
func (e *testEnv) issues(t *testing.T, token, query string) []apiIssue {
	t.Helper()
	path := "/_seo/api/v1/issues"
	if query != "" {
		path += "?" + query
	}
	rec := e.expect(t, http.StatusOK, http.MethodGet, path, token, nil)
	var out struct {
		Data []apiIssue `json:"data"`
	}
	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
		t.Fatal(err)
	}
	return out.Data
}

// issueSet is "kind url" for every issue of source, sorted.
func issueSet(list []apiIssue, source string) []string {
	var out []string
	for _, i := range list {
		if source == "" || i.Source == source {
			out = append(out, i.Kind+" "+i.URL)
		}
	}
	sort.Strings(out)
	return out
}

// findIssue returns the one issue of kind at url, failing when there is
// not exactly one.
func findIssue(t *testing.T, list []apiIssue, kind, url string) apiIssue {
	t.Helper()
	var found []apiIssue
	for _, i := range list {
		if i.Kind == kind && i.URL == url {
			found = append(found, i)
		}
	}
	if len(found) != 1 {
		t.Fatalf("%d %s issues for %s, want 1; all: %v", len(found), kind, url, issueSet(list, ""))
	}
	return found[0]
}

// plantedPage is one page of the audit site: the document (or a raw
// answer), whether a sitemap lists it, whether the home page links to it
// and the issues it plants ("kind" each, at its own address).
type plantedPage struct {
	doc    doc
	status int
	header []string
	raw    string
	listed bool
	linked bool
	kinds  []string
}

// auditPages is the planted audit site: one page per kind of UI-SPEC C23
// and the clean control page /clean.
func auditPages() []plantedPage {
	jsonld := func(v string) string { return `<script type="application/ld+json">` + v + `</script>` }
	alt := func(lang, href string) string {
		return `<link rel="alternate" hreflang="` + lang + `" href="` + siteURL + href + `">`
	}
	page := func(d doc, listed, linked bool, kinds ...string) plantedPage {
		return plantedPage{doc: d, listed: listed, linked: linked, kinds: kinds}
	}
	return []plantedPage{
		page(doc{path: "/clean", head: jsonld(`{"@context":"https://schema.org","@graph":[{"@type":"FAQPage","@id":"#faq"}]}`),
			body: `<p>Plain text of the control page.</p><img src="/deco.png" alt="" width="10" height="10"><img src="/photo.jpg" alt="A photo" width="10" height="10">`}, true, true),
		// Statuses and sitemaps.
		{doc: doc{path: "/p/status-500"}, status: 500, linked: true, kinds: []string{"page_status"}},
		{doc: doc{path: "/p/sitemap-404"}, status: 404, listed: true, kinds: []string{"sitemap_error"}},
		page(doc{path: "/p/noindex", robots: "noindex, follow"}, true, true, "sitemap_noindex"),
		page(doc{path: "/p/blocked"}, true, true, "sitemap_blocked"),
		page(doc{path: "/p/unlisted"}, false, true, "sitemap_missing"),
		// Head.
		page(doc{path: "/p/no-canonical", noCanonical: true}, true, true, "canonical_missing"),
		page(doc{path: "/p/bad-canonical", canonical: siteURL + "/p/canonical-404"}, true, true, "canonical_error"),
		page(doc{path: "/p/no-title", noTitle: true, h1: []string{"No title page"}}, true, true, "missing_title"),
		page(doc{path: "/p/dup-title-a", title: "Same title"}, true, true, "duplicate_title"),
		page(doc{path: "/p/dup-title-b", title: "Same title"}, true, true, "duplicate_title"),
		page(doc{path: "/p/no-desc", noDesc: true}, true, true, "missing_description"),
		page(doc{path: "/p/dup-desc-a", description: "Same description"}, true, true, "duplicate_description"),
		page(doc{path: "/p/dup-desc-b", description: "Same description"}, true, true, "duplicate_description"),
		// Headings.
		page(doc{path: "/p/no-h1", h1: []string{}, body: "<h2>Only a subheading</h2>"}, true, true, "h1_missing"),
		page(doc{path: "/p/two-h1", h1: []string{"Page /p/two-h1", "Second"}}, true, true, "h1_multiple"),
		page(doc{path: "/p/skip", body: "<h3>Deep</h3>"}, true, true, "heading_skipped"),
		page(doc{path: "/p/empty-heading", body: "<h2></h2><p>Text</p>"}, true, true, "heading_empty"),
		page(doc{path: "/p/mismatch", title: "Alpha page", h1: []string{"Completely different"}}, true, true, "h1_title_mismatch"),
		// JSON-LD.
		page(doc{path: "/p/jsonld-bad", head: jsonld(`{"@context": "https://schema.org", bad`)}, true, true, "jsonld_invalid"),
		page(doc{path: "/p/jsonld-two", head: jsonld(`{"@context":"https://schema.org","@type":"WebPage","name":"A"}`) + jsonld(`{"@context":"https://schema.org","@type":"WebPage","name":"B"}`)}, true, true, "jsonld_multiple"),
		page(doc{path: "/p/jsonld-req", head: jsonld(`{"@context":"https://schema.org","@graph":[{"@type":"BreadcrumbList","@id":"#crumbs"}]}`)}, true, true, "jsonld_required"),
		page(doc{path: "/p/jsonld-rec", head: jsonld(`{"@context":"https://schema.org","@type":"Organization","name":"Acme","url":"https://www.example.test/"}`)}, true, true, "jsonld_recommended"),
		// hreflang.
		page(doc{path: "/en/x", head: alt("en", "/en/x") + alt("pl", "/pl/x")}, true, true, "hreflang_reciprocal"),
		page(doc{path: "/pl/x", lang: "pl", head: alt("pl", "/pl/x")}, true, true),
		page(doc{path: "/p/hreflang-err", head: alt("en", "/p/hreflang-err") + alt("de", "/de/missing")}, true, true, "hreflang_error"),
		// Links.
		page(doc{path: "/p/generic", body: `<p><a href="/clean">kliknij tutaj</a> and <a href="/clean">Click here</a></p>`}, true, true, "generic_anchor"),
		{doc: doc{path: "/p/gone"}, status: 404, linked: true, kinds: []string{"broken_link"}},
		{doc: doc{path: "/p/moved"}, status: 301, header: []string{"Location", "/clean"}, linked: true, kinds: []string{"redirected_link"}},
		// Images.
		page(doc{path: "/p/img-noalt", body: `<img src="/a.jpg" width="10" height="10">`}, true, true, "image_alt"),
		page(doc{path: "/p/img-nosize", body: `<img src="/b.jpg" alt="B">`}, true, true, "image_size"),
		page(doc{path: "/p/lazy", head: `<meta property="og:image" content="` + siteURL + `/hero.jpg">`, body: `<img src="/hero.jpg" alt="Hero" width="10" height="10" loading="lazy">`}, true, true, "featured_lazy"),
		// Raw HTML against the CMS records (content_js, check cms).
		page(doc{path: "/c/js-h1", title: "Template heading"}, true, true, "content_js"),
		page(doc{path: "/c/js-words", title: "Words page", body: "<p>Just a few words here.</p>"}, true, true, "content_js"),
		page(doc{path: "/c/js-links", title: "Linked post", links: []string{"/clean"}}, true, true, "content_js"),
		page(doc{path: "/c/parent", title: "Parent post"}, true, true),
		// Raw against rendered (content_js, check rendered).
		page(doc{path: "/p/render", body: "<p>Short raw text.</p>"}, true, true, "content_js"),
		page(doc{path: "/p/render-h1", h1: []string{}}, true, true, "h1_missing", "content_js"),
		// Orphans.
		page(doc{path: "/p/orphan", title: "Orphan topic"}, true, false, "orphan_page"),
		page(doc{path: "/p/mentions", body: "<p>We wrote about the orphan topic before.</p>"}, true, true),
	}
}

// auditRecords are the CMS records behind the /c/ pages.
func auditRecords() map[string][]crawlRecord {
	return map[string][]crawlRecord{crawlPostType: {
		{id: "1", path: "/c/js-h1", title: "Record heading"},
		{id: "2", path: "/c/js-words", title: "Words page", words: 400},
		{id: "3", path: "/c/js-links", title: "Linked post", parents: []pact.ContentRef{{Type: crawlPostType, ID: "4"}}},
		{id: "4", path: "/c/parent", title: "Parent post"},
	}}
}

// plantAudit serves the audit site and returns the expected crawl issues
// ("kind url", sorted).
func plantAudit(site *fakeSite) []string {
	var listed, linked, want []string
	listed = append(listed, "/")
	for _, p := range auditPages() {
		switch {
		case p.raw != "":
			site.set(p.doc.path, p.status, p.raw, p.header...)
		case p.status != 0:
			site.set(p.doc.path, p.status, p.doc.render(), p.header...)
		default:
			site.plant(p.doc)
		}
		if p.listed {
			listed = append(listed, p.doc.path)
		}
		if p.linked {
			linked = append(linked, p.doc.path)
		}
		for _, k := range p.kinds {
			want = append(want, k+" "+p.doc.path)
		}
	}
	home := doc{path: "/", title: "Home", links: linked, body: `<p><a href="https://evil.example.com/x">Partner</a> <a href="http://169.254.169.254/latest">Metadata</a> <a href="http://www.example.test/plain">Plain http</a></p>`}
	site.plant(home)
	site.sitemap(listed...)
	site.set("/robots.txt", http.StatusOK, "User-agent: *\nDisallow: /p/blocked\n", "Content-Type", "text/plain")
	sort.Strings(want)
	return want
}

// fakeRenderer is the test renderer of the comparison: the DOM planted
// for a path, else the raw page (an identical rendering).
type fakeRenderer struct {
	site   *fakeSite
	dom    map[string]string
	fail   error
	mu     sync.Mutex
	calls  int
	closed bool
}

func (r *fakeRenderer) Render(_ context.Context, pageURL string) (string, error) {
	r.mu.Lock()
	r.calls++
	r.mu.Unlock()
	if r.fail != nil {
		return "", r.fail
	}
	u, err := url.Parse(pageURL)
	if err != nil {
		return "", err
	}
	if d, ok := r.dom[u.Path]; ok {
		return d, nil
	}
	r.site.mu.Lock()
	defer r.site.mu.Unlock()
	return r.site.pages[u.Path].body, nil
}

func (r *fakeRenderer) Close() error {
	r.closed = true
	return nil
}

// auditRenderer renders /p/render with a link and 80 more words, and
// /p/render-h1 with an H1.
func auditRenderer(site *fakeSite) *fakeRenderer {
	words := strings.Repeat("rendered word ", 40)
	return &fakeRenderer{site: site, dom: map[string]string{
		"/p/render":    doc{path: "/p/render", body: `<p>Short raw text.</p><p>` + words + `</p><a href="/p/js-only">Only after JavaScript</a>`}.render(),
		"/p/render-h1": doc{path: "/p/render-h1"}.render(),
	}}
}