- golem15.seo.issues (Site audit): current issues errors first, labels, summary and fix columns, filters, P7 issue_details, Check this address again, Ignore, Stop ignoring, bulk Ignore, ignored rows badged - golem15.seo.crawls (Crawls): newest first, P6 crawl_summary with one status callout, staging and rendered-comparison lines, stats strip and per-kind table; Cancel; a running crawl refuses delete - Start crawl toolbar action on both, refusing already_running and no_site_url; navigation Site audit (gauge) and Crawls (globe) - routes and MCP tools start/cancel/list/get/delete crawls, list_issues with filters (open by default), recheck_url, ignore and unignore - AdminOperations maps both controllers; toolbar actions get their own operation kind - TestCrawlAdmin, TestCrawlTools, TestIssueKindLabels
612 lines
21 KiB
Go
612 lines
21 KiB
Go
package seo_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"html"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
|
|
"git.golem15.com/golem15/summercms/modules/backpack"
|
|
"git.golem15.com/golem15/summercms/modules/conga"
|
|
"git.golem15.com/golem15/summercms/modules/fetchguard"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"git.golem15.com/golem15/summercms/modules/party"
|
|
"git.golem15.com/golem15/summercms/modules/signpost"
|
|
)
|
|
|
|
// The test site the crawler fetches: a fake http.RoundTripper handed to
|
|
// the crawler through fetchguard.WithTransport, so the crawler's real
|
|
// requests (URL checks, User-Agent, no redirects followed) are asserted
|
|
// offline. Paths it does not know answer 404.
|
|
|
|
// sitePage is one answer of the test site.
|
|
type sitePage struct {
|
|
status int
|
|
body string
|
|
header map[string]string
|
|
}
|
|
|
|
// fakeSite serves the test site and records every request URL.
|
|
type fakeSite struct {
|
|
mu sync.Mutex
|
|
pages map[string]sitePage
|
|
requests []string
|
|
agents []string
|
|
}
|
|
|
|
func newFakeSite() *fakeSite { return &fakeSite{pages: map[string]sitePage{}} }
|
|
|
|
// set serves body with status at path (path with query).
|
|
func (s *fakeSite) set(path string, status int, body string, header ...string) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
h := map[string]string{"Content-Type": "text/html; charset=utf-8"}
|
|
for i := 0; i+1 < len(header); i += 2 {
|
|
h[header[i]] = header[i+1]
|
|
}
|
|
s.pages[path] = sitePage{status: status, body: body, header: h}
|
|
}
|
|
|
|
// html serves a 200 HTML page.
|
|
func (s *fakeSite) html(path, body string) { s.set(path, http.StatusOK, body) }
|
|
|
|
func (s *fakeSite) RoundTrip(r *http.Request) (*http.Response, error) {
|
|
s.mu.Lock()
|
|
s.requests = append(s.requests, r.URL.String())
|
|
s.agents = append(s.agents, r.Header.Get("User-Agent"))
|
|
key := r.URL.EscapedPath()
|
|
if r.URL.RawQuery != "" {
|
|
key += "?" + r.URL.RawQuery
|
|
}
|
|
p, ok := s.pages[key]
|
|
s.mu.Unlock()
|
|
if !ok || !strings.EqualFold(r.URL.Hostname(), "www.example.test") {
|
|
p = sitePage{status: http.StatusNotFound, body: "<html><body>Not found</body></html>", header: map[string]string{"Content-Type": "text/html"}}
|
|
}
|
|
header := http.Header{}
|
|
for k, v := range p.header {
|
|
header.Set(k, v)
|
|
}
|
|
return &http.Response{StatusCode: p.status, Header: header, Body: io.NopCloser(strings.NewReader(p.body)), Request: r}, nil
|
|
}
|
|
|
|
// requested reports whether any request went to an URL containing part.
|
|
func (s *fakeSite) requested(part string) bool {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
for _, u := range s.requests {
|
|
if strings.Contains(u, part) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// sitemap serves /sitemap.xml as an index of one file listing paths.
|
|
func (s *fakeSite) sitemap(paths ...string) {
|
|
s.set("/sitemap.xml", http.StatusOK, `<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>`+siteURL+`/sitemap-pages-1.xml</loc></sitemap></sitemapindex>`, "Content-Type", "application/xml")
|
|
var b strings.Builder
|
|
b.WriteString(`<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`)
|
|
for _, p := range paths {
|
|
b.WriteString("<url><loc>" + siteURL + html.EscapeString(p) + "</loc></url>")
|
|
}
|
|
b.WriteString("</urlset>")
|
|
s.set("/sitemap-pages-1.xml", http.StatusOK, b.String(), "Content-Type", "application/xml")
|
|
}
|
|
|
|
// doc is one planted page: a complete, clean document (unique title and
|
|
// description, a self canonical, one H1 matching the title) that a test
|
|
// mutates for the kind it plants.
|
|
type doc struct {
|
|
path string
|
|
lang string
|
|
title string
|
|
noTitle bool
|
|
description string
|
|
noDesc bool
|
|
canonical string
|
|
noCanonical bool
|
|
robots string
|
|
head string
|
|
h1 []string
|
|
body string
|
|
links []string
|
|
}
|
|
|
|
// render writes the document's HTML.
|
|
func (d doc) render() string {
|
|
var b strings.Builder
|
|
lang := d.lang
|
|
if lang == "" {
|
|
lang = "en"
|
|
}
|
|
b.WriteString(`<!doctype html><html lang="` + lang + `"><head><meta charset="utf-8">`)
|
|
title := d.title
|
|
if title == "" {
|
|
title = "Page " + d.path
|
|
}
|
|
if !d.noTitle {
|
|
b.WriteString("<title>" + html.EscapeString(title) + "</title>")
|
|
}
|
|
if !d.noDesc {
|
|
desc := d.description
|
|
if desc == "" {
|
|
desc = "About " + d.path
|
|
}
|
|
b.WriteString(`<meta name="description" content="` + html.EscapeString(desc) + `">`)
|
|
}
|
|
if !d.noCanonical {
|
|
c := d.canonical
|
|
if c == "" {
|
|
c = siteURL + d.path
|
|
}
|
|
b.WriteString(`<link rel="canonical" href="` + html.EscapeString(c) + `">`)
|
|
}
|
|
if d.robots != "" {
|
|
b.WriteString(`<meta name="robots" content="` + d.robots + `">`)
|
|
}
|
|
b.WriteString(d.head + "</head><body><main>")
|
|
h1 := d.h1
|
|
if h1 == nil {
|
|
h1 = []string{title}
|
|
}
|
|
for _, h := range h1 {
|
|
b.WriteString("<h1>" + html.EscapeString(h) + "</h1>")
|
|
}
|
|
b.WriteString(d.body)
|
|
for _, l := range d.links {
|
|
b.WriteString(`<p><a href="` + l + `">Read about ` + html.EscapeString(strings.Trim(l, "/")) + `</a></p>`)
|
|
}
|
|
b.WriteString("</main></body></html>")
|
|
return b.String()
|
|
}
|
|
|
|
// plant serves the documents on the site.
|
|
func (s *fakeSite) plant(docs ...doc) {
|
|
for _, d := range docs {
|
|
s.html(d.path, d.render())
|
|
}
|
|
}
|
|
|
|
// The fake content plugin acme.crawl: in-memory records behind two types
|
|
// with descriptor facts and fixing tools, so the audit's content routes
|
|
// (P33-38) are exercised. acme-crawl-post (/c/:slug) names edit, headings
|
|
// and image_alt tools; acme-crawl-category (/cat/:slug) names only a
|
|
// headings tool.
|
|
const (
|
|
crawlPostType = "acme-crawl-post"
|
|
crawlCategoryType = "acme-crawl-category"
|
|
)
|
|
|
|
// crawlRecord is one in-memory record.
|
|
type crawlRecord struct {
|
|
id string
|
|
path string
|
|
title string
|
|
words int
|
|
headings []pact.ContentHeading
|
|
images []pact.ContentImage
|
|
topHeading int
|
|
parents []pact.ContentRef
|
|
}
|
|
|
|
var (
|
|
crawlRecordsMu sync.Mutex
|
|
crawlRecords = map[string][]crawlRecord{}
|
|
)
|
|
|
|
// setCrawlRecords replaces the records of the fake types for one test.
|
|
func setCrawlRecords(t *testing.T, byType map[string][]crawlRecord) {
|
|
t.Helper()
|
|
crawlRecordsMu.Lock()
|
|
crawlRecords = byType
|
|
crawlRecordsMu.Unlock()
|
|
t.Cleanup(func() {
|
|
crawlRecordsMu.Lock()
|
|
crawlRecords = map[string][]crawlRecord{}
|
|
crawlRecordsMu.Unlock()
|
|
})
|
|
}
|
|
|
|
func crawlRecordsOf(typeKey string) []crawlRecord {
|
|
crawlRecordsMu.Lock()
|
|
defer crawlRecordsMu.Unlock()
|
|
return slices.Clone(crawlRecords[typeKey])
|
|
}
|
|
|
|
func (r crawlRecord) content() pact.ContentRecord {
|
|
words := r.words
|
|
var wp *int
|
|
if words > 0 {
|
|
wp = &words
|
|
}
|
|
return pact.ContentRecord{
|
|
ID: r.id, Published: true, TopHeading: r.topHeading, Parents: r.parents,
|
|
Locales: []pact.ContentLocale{{Locale: "en", URL: r.path, Title: r.title, Words: wp, Headings: r.headings, Images: r.images}},
|
|
}
|
|
}
|
|
|
|
type crawlContentPlugin struct{}
|
|
|
|
func (crawlContentPlugin) ID() string { return "acme.crawl" }
|
|
func (crawlContentPlugin) Requires() []string { return nil }
|
|
func (crawlContentPlugin) Register(*backpack.App) error { return nil }
|
|
func (crawlContentPlugin) Boot(*backpack.App) error { return nil }
|
|
func (crawlContentPlugin) MenuItemTypes() []pact.MenuItemType {
|
|
return []pact.MenuItemType{
|
|
crawlType(crawlPostType, "/c/:slug", map[string]string{
|
|
pact.ContentToolEdit: "acme_edit_post", pact.ContentToolHeadings: "acme_fix_headings", pact.ContentToolImageAlt: "acme_set_alt",
|
|
}),
|
|
crawlType(crawlCategoryType, "/cat/:slug", map[string]string{pact.ContentToolHeadings: "acme_category_headings"}),
|
|
}
|
|
}
|
|
|
|
// crawlType is one fake type over the in-memory records.
|
|
func crawlType(key, pattern string, tools map[string]string) pact.MenuItemType {
|
|
find := func(match func(crawlRecord) bool) *crawlRecord {
|
|
for _, r := range crawlRecordsOf(key) {
|
|
if match(r) {
|
|
return &r
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
return pact.MenuItemType{
|
|
Key: key, Label: key, DefaultPattern: pattern, Placeholders: []string{"slug"},
|
|
Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { return nil, nil },
|
|
Content: &pact.ContentDescriptor{
|
|
Model: "acme_crawl_" + strings.ReplaceAll(key, "-", "_"),
|
|
Tools: tools,
|
|
Describe: func(_ context.Context, in pact.ContentDescribe) (*pact.ContentRecord, error) {
|
|
r := find(func(r crawlRecord) bool { return r.id == in.ID })
|
|
if r == nil {
|
|
return nil, nil
|
|
}
|
|
rec := r.content()
|
|
return &rec, nil
|
|
},
|
|
Records: func(_ context.Context, in pact.ContentList) (pact.ContentPage, error) {
|
|
var out pact.ContentPage
|
|
if in.After != "" {
|
|
return out, nil
|
|
}
|
|
for _, r := range crawlRecordsOf(key) {
|
|
out.Records = append(out.Records, r.content())
|
|
}
|
|
return out, nil
|
|
},
|
|
Match: func(_ context.Context, in pact.ContentMatch) (*pact.ContentMatchResult, error) {
|
|
if _, ok := signpost.MatchPattern(in.Pattern, in.Path); !ok {
|
|
return nil, nil
|
|
}
|
|
r := find(func(r crawlRecord) bool { return r.path == in.Path })
|
|
if r == nil {
|
|
return nil, nil
|
|
}
|
|
return &pact.ContentMatchResult{ID: r.id, Locale: "en", Exact: true}, nil
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
var _ pact.HasMenuItemTypes = crawlContentPlugin{}
|
|
|
|
func init() {
|
|
party.Register(crawlContentPlugin{})
|
|
}
|
|
|
|
// bootCrawlEnv boots acme.pages, acme.crawl and golem15.seo.
|
|
func bootCrawlEnv(t *testing.T, extraEnv ...string) *testEnv {
|
|
t.Helper()
|
|
return bootPlugins(t, []string{"acme.pages", "acme.crawl", "golem15.seo"}, extraEnv...)
|
|
}
|
|
|
|
// crawlJob returns the SEO plugin's crawl job.
|
|
func (e *testEnv) crawlJob(t *testing.T) pact.Job {
|
|
t.Helper()
|
|
for _, j := range e.seoPlugin(t).Jobs() {
|
|
if info, ok := conga.Describe(j); ok && info.Kind == crawl.CrawlKind {
|
|
return j
|
|
}
|
|
}
|
|
t.Fatal("the SEO plugin registers no crawl job")
|
|
return nil
|
|
}
|
|
|
|
// siteCtx is a context whose crawler requests go to site.
|
|
func siteCtx(t *testing.T, site *fakeSite) context.Context {
|
|
return fetchguard.WithTransport(t.Context(), site)
|
|
}
|
|
|
|
// startCrawl starts a crawl through the API and returns its id.
|
|
func (e *testEnv) startCrawl(t *testing.T, token string) uint {
|
|
t.Helper()
|
|
rec := e.expect(t, http.StatusAccepted, http.MethodPost, "/_seo/api/v1/crawls", token, map[string]any{})
|
|
var out struct {
|
|
Data struct {
|
|
ID uint `json:"id"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil || out.Data.ID == 0 {
|
|
t.Fatalf("start crawl: %s (%v)", rec.Body.String(), err)
|
|
}
|
|
return out.Data.ID
|
|
}
|
|
|
|
// runCrawl runs the crawl job of id against the site.
|
|
func (e *testEnv) runCrawl(t *testing.T, ctx context.Context, id uint) {
|
|
t.Helper()
|
|
if err := e.crawlJob(t).Work(ctx, crawl.CrawlArgs{CrawlID: id}); err != nil {
|
|
t.Fatalf("crawl job: %v", err)
|
|
}
|
|
}
|
|
|
|
// crawlOnce starts a crawl and runs it against the site.
|
|
func (e *testEnv) crawlOnce(t *testing.T, token string, ctx context.Context) uint {
|
|
t.Helper()
|
|
id := e.startCrawl(t, token)
|
|
e.runCrawl(t, ctx, id)
|
|
return id
|
|
}
|
|
|
|
// apiIssue is one issue of GET /_seo/api/v1/issues.
|
|
type apiIssue struct {
|
|
ID uint `json:"id"`
|
|
Kind string `json:"kind"`
|
|
URL string `json:"url"`
|
|
Locale string `json:"locale"`
|
|
Details map[string]any `json:"details"`
|
|
Tool string `json:"tool"`
|
|
Severity string `json:"severity"`
|
|
Source string `json:"source"`
|
|
Fix string `json:"fix"`
|
|
Ignored bool `json:"ignored"`
|
|
CrawlID *uint `json:"crawl_id"`
|
|
}
|
|
|
|
// issues lists the audit's issues through the API (query appended).
|
|
func (e *testEnv) issues(t *testing.T, token, query string) []apiIssue {
|
|
t.Helper()
|
|
path := "/_seo/api/v1/issues"
|
|
if query != "" {
|
|
path += "?" + query
|
|
}
|
|
rec := e.expect(t, http.StatusOK, http.MethodGet, path, token, nil)
|
|
var out struct {
|
|
Data []apiIssue `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out.Data
|
|
}
|
|
|
|
// issueSet is "kind url" for every issue of source, sorted.
|
|
func issueSet(list []apiIssue, source string) []string {
|
|
var out []string
|
|
for _, i := range list {
|
|
if source == "" || i.Source == source {
|
|
out = append(out, i.Kind+" "+i.URL)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// findIssue returns the one issue of kind at url, failing when there is
|
|
// not exactly one.
|
|
func findIssue(t *testing.T, list []apiIssue, kind, url string) apiIssue {
|
|
t.Helper()
|
|
var found []apiIssue
|
|
for _, i := range list {
|
|
if i.Kind == kind && i.URL == url {
|
|
found = append(found, i)
|
|
}
|
|
}
|
|
if len(found) != 1 {
|
|
t.Fatalf("%d %s issues for %s, want 1; all: %v", len(found), kind, url, issueSet(list, ""))
|
|
}
|
|
return found[0]
|
|
}
|
|
|
|
// plantedPage is one page of the audit site: the document (or a raw
|
|
// answer), whether a sitemap lists it, whether the home page links to it
|
|
// and the issues it plants ("kind" each, at its own address).
|
|
type plantedPage struct {
|
|
doc doc
|
|
status int
|
|
header []string
|
|
raw string
|
|
listed bool
|
|
linked bool
|
|
kinds []string
|
|
}
|
|
|
|
// auditPages is the planted audit site: one page per kind of UI-SPEC C23
|
|
// and the clean control page /clean.
|
|
func auditPages() []plantedPage {
|
|
jsonld := func(v string) string { return `<script type="application/ld+json">` + v + `</script>` }
|
|
alt := func(lang, href string) string {
|
|
return `<link rel="alternate" hreflang="` + lang + `" href="` + siteURL + href + `">`
|
|
}
|
|
page := func(d doc, listed, linked bool, kinds ...string) plantedPage {
|
|
return plantedPage{doc: d, listed: listed, linked: linked, kinds: kinds}
|
|
}
|
|
return []plantedPage{
|
|
page(doc{path: "/clean", head: jsonld(`{"@context":"https://schema.org","@graph":[{"@type":"FAQPage","@id":"#faq"}]}`),
|
|
body: `<p>Plain text of the control page.</p><img src="/deco.png" alt="" width="10" height="10"><img src="/photo.jpg" alt="A photo" width="10" height="10">`}, true, true),
|
|
// Statuses and sitemaps.
|
|
{doc: doc{path: "/p/status-500"}, status: 500, linked: true, kinds: []string{"page_status"}},
|
|
{doc: doc{path: "/p/sitemap-404"}, status: 404, listed: true, kinds: []string{"sitemap_error"}},
|
|
page(doc{path: "/p/noindex", robots: "noindex, follow"}, true, true, "sitemap_noindex"),
|
|
page(doc{path: "/p/blocked"}, true, true, "sitemap_blocked"),
|
|
page(doc{path: "/p/unlisted"}, false, true, "sitemap_missing"),
|
|
// Head.
|
|
page(doc{path: "/p/no-canonical", noCanonical: true}, true, true, "canonical_missing"),
|
|
page(doc{path: "/p/bad-canonical", canonical: siteURL + "/p/canonical-404"}, true, true, "canonical_error"),
|
|
page(doc{path: "/p/no-title", noTitle: true, h1: []string{"No title page"}}, true, true, "missing_title"),
|
|
page(doc{path: "/p/dup-title-a", title: "Same title"}, true, true, "duplicate_title"),
|
|
page(doc{path: "/p/dup-title-b", title: "Same title"}, true, true, "duplicate_title"),
|
|
page(doc{path: "/p/no-desc", noDesc: true}, true, true, "missing_description"),
|
|
page(doc{path: "/p/dup-desc-a", description: "Same description"}, true, true, "duplicate_description"),
|
|
page(doc{path: "/p/dup-desc-b", description: "Same description"}, true, true, "duplicate_description"),
|
|
// Headings.
|
|
page(doc{path: "/p/no-h1", h1: []string{}, body: "<h2>Only a subheading</h2>"}, true, true, "h1_missing"),
|
|
page(doc{path: "/p/two-h1", h1: []string{"Page /p/two-h1", "Second"}}, true, true, "h1_multiple"),
|
|
page(doc{path: "/p/skip", body: "<h3>Deep</h3>"}, true, true, "heading_skipped"),
|
|
page(doc{path: "/p/empty-heading", body: "<h2></h2><p>Text</p>"}, true, true, "heading_empty"),
|
|
page(doc{path: "/p/mismatch", title: "Alpha page", h1: []string{"Completely different"}}, true, true, "h1_title_mismatch"),
|
|
// JSON-LD.
|
|
page(doc{path: "/p/jsonld-bad", head: jsonld(`{"@context": "https://schema.org", bad`)}, true, true, "jsonld_invalid"),
|
|
page(doc{path: "/p/jsonld-two", head: jsonld(`{"@context":"https://schema.org","@type":"WebPage","name":"A"}`) + jsonld(`{"@context":"https://schema.org","@type":"WebPage","name":"B"}`)}, true, true, "jsonld_multiple"),
|
|
page(doc{path: "/p/jsonld-req", head: jsonld(`{"@context":"https://schema.org","@graph":[{"@type":"BreadcrumbList","@id":"#crumbs"}]}`)}, true, true, "jsonld_required"),
|
|
page(doc{path: "/p/jsonld-rec", head: jsonld(`{"@context":"https://schema.org","@type":"Organization","name":"Acme","url":"https://www.example.test/"}`)}, true, true, "jsonld_recommended"),
|
|
// hreflang.
|
|
page(doc{path: "/en/x", head: alt("en", "/en/x") + alt("pl", "/pl/x")}, true, true, "hreflang_reciprocal"),
|
|
page(doc{path: "/pl/x", lang: "pl", head: alt("pl", "/pl/x")}, true, true),
|
|
page(doc{path: "/p/hreflang-err", head: alt("en", "/p/hreflang-err") + alt("de", "/de/missing")}, true, true, "hreflang_error"),
|
|
// Links.
|
|
page(doc{path: "/p/generic", body: `<p><a href="/clean">kliknij tutaj</a> and <a href="/clean">Click here</a></p>`}, true, true, "generic_anchor"),
|
|
{doc: doc{path: "/p/gone"}, status: 404, linked: true, kinds: []string{"broken_link"}},
|
|
{doc: doc{path: "/p/moved"}, status: 301, header: []string{"Location", "/clean"}, linked: true, kinds: []string{"redirected_link"}},
|
|
// Images.
|
|
page(doc{path: "/p/img-noalt", body: `<img src="/a.jpg" width="10" height="10">`}, true, true, "image_alt"),
|
|
page(doc{path: "/p/img-nosize", body: `<img src="/b.jpg" alt="B">`}, true, true, "image_size"),
|
|
page(doc{path: "/p/lazy", head: `<meta property="og:image" content="` + siteURL + `/hero.jpg">`, body: `<img src="/hero.jpg" alt="Hero" width="10" height="10" loading="lazy">`}, true, true, "featured_lazy"),
|
|
// Raw HTML against the CMS records (content_js, check cms).
|
|
page(doc{path: "/c/js-h1", title: "Template heading"}, true, true, "content_js"),
|
|
page(doc{path: "/c/js-words", title: "Words page", body: "<p>Just a few words here.</p>"}, true, true, "content_js"),
|
|
page(doc{path: "/c/js-links", title: "Linked post", links: []string{"/clean"}}, true, true, "content_js"),
|
|
page(doc{path: "/c/parent", title: "Parent post"}, true, true),
|
|
// Raw against rendered (content_js, check rendered).
|
|
page(doc{path: "/p/render", body: "<p>Short raw text.</p>"}, true, true, "content_js"),
|
|
page(doc{path: "/p/render-h1", h1: []string{}}, true, true, "h1_missing", "content_js"),
|
|
// Orphans.
|
|
page(doc{path: "/p/orphan", title: "Orphan topic"}, true, false, "orphan_page"),
|
|
page(doc{path: "/p/mentions", body: "<p>We wrote about the orphan topic before.</p>"}, true, true),
|
|
}
|
|
}
|
|
|
|
// auditRecords are the CMS records behind the /c/ pages.
|
|
func auditRecords() map[string][]crawlRecord {
|
|
return map[string][]crawlRecord{crawlPostType: {
|
|
{id: "1", path: "/c/js-h1", title: "Record heading"},
|
|
{id: "2", path: "/c/js-words", title: "Words page", words: 400},
|
|
{id: "3", path: "/c/js-links", title: "Linked post", parents: []pact.ContentRef{{Type: crawlPostType, ID: "4"}}},
|
|
{id: "4", path: "/c/parent", title: "Parent post"},
|
|
}}
|
|
}
|
|
|
|
// plantAudit serves the audit site and returns the expected crawl issues
|
|
// ("kind url", sorted).
|
|
func plantAudit(site *fakeSite) []string {
|
|
var listed, linked, want []string
|
|
listed = append(listed, "/")
|
|
for _, p := range auditPages() {
|
|
switch {
|
|
case p.raw != "":
|
|
site.set(p.doc.path, p.status, p.raw, p.header...)
|
|
case p.status != 0:
|
|
site.set(p.doc.path, p.status, p.doc.render(), p.header...)
|
|
default:
|
|
site.plant(p.doc)
|
|
}
|
|
if p.listed {
|
|
listed = append(listed, p.doc.path)
|
|
}
|
|
if p.linked {
|
|
linked = append(linked, p.doc.path)
|
|
}
|
|
for _, k := range p.kinds {
|
|
want = append(want, k+" "+p.doc.path)
|
|
}
|
|
}
|
|
home := doc{path: "/", title: "Home", links: linked, body: `<p><a href="https://evil.example.com/x">Partner</a> <a href="http://169.254.169.254/latest">Metadata</a> <a href="http://www.example.test/plain">Plain http</a></p>`}
|
|
site.plant(home)
|
|
site.sitemap(listed...)
|
|
site.set("/robots.txt", http.StatusOK, "User-agent: *\nDisallow: /p/blocked\n", "Content-Type", "text/plain")
|
|
sort.Strings(want)
|
|
return want
|
|
}
|
|
|
|
// fakeRenderer is the test renderer of the comparison: the DOM planted
|
|
// for a path, else the raw page (an identical rendering).
|
|
type fakeRenderer struct {
|
|
site *fakeSite
|
|
dom map[string]string
|
|
fail error
|
|
mu sync.Mutex
|
|
calls int
|
|
closed bool
|
|
}
|
|
|
|
func (r *fakeRenderer) Render(_ context.Context, pageURL string) (string, error) {
|
|
r.mu.Lock()
|
|
r.calls++
|
|
r.mu.Unlock()
|
|
if r.fail != nil {
|
|
return "", r.fail
|
|
}
|
|
u, err := url.Parse(pageURL)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if d, ok := r.dom[u.Path]; ok {
|
|
return d, nil
|
|
}
|
|
r.site.mu.Lock()
|
|
defer r.site.mu.Unlock()
|
|
return r.site.pages[u.Path].body, nil
|
|
}
|
|
|
|
func (r *fakeRenderer) Close() error {
|
|
r.closed = true
|
|
return nil
|
|
}
|
|
|
|
// auditRenderer renders /p/render with a link and 80 more words, and
|
|
// /p/render-h1 with an H1.
|
|
func auditRenderer(site *fakeSite) *fakeRenderer {
|
|
words := strings.Repeat("rendered word ", 40)
|
|
return &fakeRenderer{site: site, dom: map[string]string{
|
|
"/p/render": doc{path: "/p/render", body: `<p>Short raw text.</p><p>` + words + `</p><a href="/p/js-only">Only after JavaScript</a>`}.render(),
|
|
"/p/render-h1": doc{path: "/p/render-h1"}.render(),
|
|
}}
|
|
}
|
|
|
|
// callCtx runs one request with ctx (a crawler transport for a recheck).
|
|
func (e *testEnv) callCtx(t *testing.T, ctx context.Context, method, path, token string, body any) *httptest.ResponseRecorder {
|
|
t.Helper()
|
|
var reader io.Reader
|
|
if body != nil {
|
|
raw, err := json.Marshal(body)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reader = bytes.NewReader(raw)
|
|
}
|
|
req := httptest.NewRequest(method, path, reader).WithContext(ctx)
|
|
req.RemoteAddr = fmt.Sprintf("198.51.100.%d:4000", seq.Add(1)%250+1)
|
|
req.Header.Set("Accept-Language", "en")
|
|
if body != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
if token != "" {
|
|
req.Header.Set("Authorization", "Bearer "+token)
|
|
}
|
|
rec := httptest.NewRecorder()
|
|
e.h.ServeHTTP(rec, req)
|
|
return rec
|
|
}
|