sm-seo-plugin/crawl_render_test.go

package seo_test

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"slices"
	"strings"
	"testing"

	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
	"git.golem15.com/golem15/summercms/modules/compass"
)

// chromeBinary is the Chrome or Chromium of the test machine: SEO_TEST_CHROME,
// else chromium, chromium-browser or google-chrome on PATH.
func chromeBinary() string {
	if p := strings.TrimSpace(os.Getenv("SEO_TEST_CHROME")); p != "" {
		return p
	}
	for _, name := range []string{"chromium", "chromium-browser", "google-chrome"} {
		if p, err := exec.LookPath(name); err == nil {
			if abs, err := filepath.EvalSymlinks(p); err == nil {
				return abs
			}
			return p
		}
	}
	return ""
}

// renderConfig is a config with the rendered comparison's keys.
func renderConfig(t *testing.T, env ...string) *compass.Config {
	t.Helper()
	dir := t.TempDir()
	if err := os.WriteFile(filepath.Join(dir, "app.yaml"), []byte("name: seo-render-test\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	cfg, err := compass.Open(compass.Options{Dir: dir, Environ: env})
	if err != nil {
		t.Fatal(err)
	}
	return cfg
}

// jsSite is a site whose page /page adds a same-host link and two
// paragraphs after JavaScript, loads a third-party script and asks a
// link-local address; hits counts the requests that reached it.
func jsSite(t *testing.T) (*httptest.Server, *[]string) {
	t.Helper()
	var hits []string
	page := `<!doctype html><html lang="en"><head><title>JS page</title><link rel="canonical" href="/page">
<script src="https://cdn.example.test/x.js"></script></head>
<body><main><h1>JS page</h1><p>Raw text.</p></main>
<script>
try { var x = new XMLHttpRequest(); x.open("GET", "http://169.254.169.254/latest/meta-data", false); x.send(); } catch (e) {}
var main = document.querySelector("main");
var a = document.createElement("a"); a.href = "/js-only"; a.textContent = "Only after JavaScript"; main.appendChild(a);
for (var i = 0; i < 2; i++) { var p = document.createElement("p"); p.textContent = "Generated paragraph text ".repeat(20); main.appendChild(p); }
</script></body></html>`
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		hits = append(hits, r.URL.Path)
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		switch r.URL.Path {
		case "/":
			fmt.Fprint(w, `<!doctype html><html lang="en"><head><title>Home</title><link rel="canonical" href="/"></head><body><main><h1>Home</h1><a href="/page">The page</a></main></body></html>`)
		case "/page":
			fmt.Fprint(w, page)
		default:
			http.NotFound(w, r)
		}
	}))
	t.Cleanup(srv.Close)
	return srv, &hits
}

// TestChromeRenderer renders a page in headless Chrome (D-70, P33-55):
// the JavaScript-added link and text are in the DOM and the crawl reports
// them as content_js check rendered naming /js-only; the third-party script
// and the link-local request are blocked and never reach a server; the
// temporary profile is gone after Close.
func TestChromeRenderer(t *testing.T) {
	if testing.Short() {
		t.Skip("needs Chrome and Docker")
	}
	chrome := chromeBinary()
	if chrome == "" {
		t.Skip("no Chrome or Chromium on this machine (set SEO_TEST_CHROME)")
	}
	srv, hits := jsSite(t)
	u, _ := url.Parse(srv.URL)
	site := "http://localhost:" + u.Port()
	siteURL, _ := url.Parse(site)

	cfg := renderConfig(t, "SUMMER_GOLEM15__SEO__CRAWL_CHROME_PATH="+chrome, "SUMMER_GOLEM15__SEO__CRAWL_TRUSTED=true")
	r, err := crawl.NewRenderer(t.Context(), cfg, siteURL, true, crawl.UserAgent(site))
	if err != nil {
		t.Fatalf("NewRenderer: %v", err)
	}
	inspect, ok := r.(interface {
		Blocked() []string
		ProfileDir() string
	})
	if !ok {
		t.Fatalf("renderer %T has no Blocked and ProfileDir", r)
	}
	profile := inspect.ProfileDir()
	if _, err := os.Stat(profile); err != nil {
		t.Fatalf("profile %s: %v", profile, err)
	}
	dom, err := r.Render(t.Context(), site+"/page")
	if err != nil {
		t.Fatalf("Render: %v", err)
	}
	if !strings.Contains(dom, `href="/js-only"`) || !strings.Contains(dom, "Generated paragraph text") {
		t.Fatalf("rendered DOM lacks the JavaScript content:\n%s", dom)
	}
	blocked := strings.Join(inspect.Blocked(), "\n")
	for _, want := range []string{"https://cdn.example.test/x.js", "http://169.254.169.254/latest/meta-data"} {
		if !strings.Contains(blocked, want) {
			t.Fatalf("blocked = %q, want %s", blocked, want)
		}
	}
	if slices.ContainsFunc(*hits, func(p string) bool { return strings.Contains(p, "latest") || strings.Contains(p, "x.js") }) {
		t.Fatalf("a blocked request reached the site: %v", *hits)
	}
	if err := r.Close(); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(profile); !os.IsNotExist(err) {
		t.Fatalf("profile %s still exists after Close: %v", profile, err)
	}

	// The crawl with the Chrome path compares the page with its rendering.
	env := bootCrawlEnv(t,
		"SUMMER_GOLEM15__SEO__SITE_URL="+site,
		"SUMMER_GOLEM15__SEO__CRAWL_TRUSTED=true",
		"SUMMER_GOLEM15__SEO__CRAWL_CHROME_PATH="+chrome,
	)
	token := env.admin(t, true)
	id := env.crawlOnce(t, token, t.Context())
	got := env.expect(t, http.StatusOK, http.MethodGet, fmt.Sprintf("/_seo/api/v1/crawls/%d", id), token, nil).Body.String()
	if !jsonContains(got, `"render":{"status":"done","pages":2`) {
		t.Fatalf("crawl render = %s", got)
	}
	issue := findIssue(t, env.issues(t, token, ""), "content_js", "/page")
	if issue.Details["check"] != "rendered" || !strings.Contains(fmt.Sprint(issue.Details["what"]), "/js-only") {
		t.Fatalf("content_js = %+v", issue)
	}
}

// TestRendererGuards checks the renderer's refusals (T-33-66, T-33-69):
// an empty Chrome path skips the comparison, a relative or non-executable
// path fails it, and outside trusted mode a site host that resolves to a
// private address refuses before Chrome starts; a crawl with an invalid
// path records render_status failed with the note and still runs its raw
// checks.
func TestRendererGuards(t *testing.T) {
	site, _ := url.Parse("https://localhost")
	exe, err := exec.LookPath("sh")
	if err != nil {
		t.Skip("no sh")
	}
	exe, _ = filepath.EvalSymlinks(exe)
	notExec := filepath.Join(t.TempDir(), "chrome")
	if err := os.WriteFile(notExec, []byte("x"), 0o644); err != nil {
		t.Fatal(err)
	}
	for _, tc := range []struct {
		path, want string
	}{
		{"", ""},
		{"bin/chrome", "is not absolute"},
		{notExec, "is not an executable file"},
		{exe, "private address"},
	} {
		cfg := renderConfig(t, "SUMMER_GOLEM15__SEO__CRAWL_CHROME_PATH="+tc.path)
		r, err := crawl.NewRenderer(t.Context(), cfg, site, false, "test")
		if tc.want == "" {
			if r != nil || err != nil {
				t.Fatalf("empty path: %v %v", r, err)
			}
			continue
		}
		if r != nil || err == nil || !strings.Contains(err.Error(), tc.want) {
			t.Fatalf("path %q: renderer %v, error %v, want %q", tc.path, r, err, tc.want)
		}
	}
	if testing.Short() {
		return
	}
	env := bootCrawlEnv(t, "SUMMER_GOLEM15__SEO__CRAWL_CHROME_PATH=bin/chrome")
	token := env.admin(t, true)
	fake := newFakeSite()
	fake.sitemap("/")
	fake.plant(doc{path: "/", title: "Home", h1: []string{"Home", "Again"}})
	id := env.crawlOnce(t, token, siteCtx(t, fake))
	got := env.expect(t, http.StatusOK, http.MethodGet, fmt.Sprintf("/_seo/api/v1/crawls/%d", id), token, nil).Body.String()
	if !jsonContains(got, `"status":"finished"`) || !jsonContains(got, `"render":{"status":"failed"`) || !strings.Contains(got, "is not absolute") {
		t.Fatalf("crawl with a relative Chrome path = %s", got)
	}
	findIssue(t, env.issues(t, token, ""), "h1_multiple", "/")
}