c2b61bf8e9
- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37) - classes/crawl: fetchguard client on the site host and its www twin, sitemap index and robots.txt seeds, two workers with a delay, limits - h1_missing and h1_multiple with the P33-38 fix route - POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues answers the stored issues with id, severity, source, fix, found_at, ignored and crawl_id
418 lines
12 KiB
Go
418 lines
12 KiB
Go
package seo_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"html"
|
|
"io"
|
|
"net/http"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
|
|
"git.golem15.com/golem15/summercms/modules/backpack"
|
|
"git.golem15.com/golem15/summercms/modules/conga"
|
|
"git.golem15.com/golem15/summercms/modules/fetchguard"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"git.golem15.com/golem15/summercms/modules/party"
|
|
"git.golem15.com/golem15/summercms/modules/signpost"
|
|
)
|
|
|
|
// The test site the crawler fetches: a fake http.RoundTripper handed to
|
|
// the crawler through fetchguard.WithTransport, so the crawler's real
|
|
// requests (URL checks, User-Agent, no redirects followed) are asserted
|
|
// offline. Paths it does not know answer 404.
|
|
|
|
// sitePage is one answer of the test site.
|
|
type sitePage struct {
|
|
status int
|
|
body string
|
|
header map[string]string
|
|
}
|
|
|
|
// fakeSite serves the test site and records every request URL.
|
|
type fakeSite struct {
|
|
mu sync.Mutex
|
|
pages map[string]sitePage
|
|
requests []string
|
|
agents []string
|
|
}
|
|
|
|
func newFakeSite() *fakeSite { return &fakeSite{pages: map[string]sitePage{}} }
|
|
|
|
// set serves body with status at path (path with query).
|
|
func (s *fakeSite) set(path string, status int, body string, header ...string) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
h := map[string]string{"Content-Type": "text/html; charset=utf-8"}
|
|
for i := 0; i+1 < len(header); i += 2 {
|
|
h[header[i]] = header[i+1]
|
|
}
|
|
s.pages[path] = sitePage{status: status, body: body, header: h}
|
|
}
|
|
|
|
// html serves a 200 HTML page.
|
|
func (s *fakeSite) html(path, body string) { s.set(path, http.StatusOK, body) }
|
|
|
|
func (s *fakeSite) RoundTrip(r *http.Request) (*http.Response, error) {
|
|
s.mu.Lock()
|
|
s.requests = append(s.requests, r.URL.String())
|
|
s.agents = append(s.agents, r.Header.Get("User-Agent"))
|
|
key := r.URL.EscapedPath()
|
|
if r.URL.RawQuery != "" {
|
|
key += "?" + r.URL.RawQuery
|
|
}
|
|
p, ok := s.pages[key]
|
|
s.mu.Unlock()
|
|
if !ok || !strings.EqualFold(r.URL.Hostname(), "www.example.test") {
|
|
p = sitePage{status: http.StatusNotFound, body: "<html><body>Not found</body></html>", header: map[string]string{"Content-Type": "text/html"}}
|
|
}
|
|
header := http.Header{}
|
|
for k, v := range p.header {
|
|
header.Set(k, v)
|
|
}
|
|
return &http.Response{StatusCode: p.status, Header: header, Body: io.NopCloser(strings.NewReader(p.body)), Request: r}, nil
|
|
}
|
|
|
|
// requested reports whether any request went to an URL containing part.
|
|
func (s *fakeSite) requested(part string) bool {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
for _, u := range s.requests {
|
|
if strings.Contains(u, part) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// sitemap serves /sitemap.xml as an index of one file listing paths.
|
|
func (s *fakeSite) sitemap(paths ...string) {
|
|
s.set("/sitemap.xml", http.StatusOK, `<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>`+siteURL+`/sitemap-pages-1.xml</loc></sitemap></sitemapindex>`, "Content-Type", "application/xml")
|
|
var b strings.Builder
|
|
b.WriteString(`<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`)
|
|
for _, p := range paths {
|
|
b.WriteString("<url><loc>" + siteURL + html.EscapeString(p) + "</loc></url>")
|
|
}
|
|
b.WriteString("</urlset>")
|
|
s.set("/sitemap-pages-1.xml", http.StatusOK, b.String(), "Content-Type", "application/xml")
|
|
}
|
|
|
|
// doc is one planted page: a complete, clean document (unique title and
|
|
// description, a self canonical, one H1 matching the title) that a test
|
|
// mutates for the kind it plants.
|
|
type doc struct {
|
|
path string
|
|
lang string
|
|
title string
|
|
noTitle bool
|
|
description string
|
|
noDesc bool
|
|
canonical string
|
|
noCanonical bool
|
|
robots string
|
|
head string
|
|
h1 []string
|
|
body string
|
|
links []string
|
|
}
|
|
|
|
// render writes the document's HTML.
|
|
func (d doc) render() string {
|
|
var b strings.Builder
|
|
lang := d.lang
|
|
if lang == "" {
|
|
lang = "en"
|
|
}
|
|
b.WriteString(`<!doctype html><html lang="` + lang + `"><head><meta charset="utf-8">`)
|
|
title := d.title
|
|
if title == "" {
|
|
title = "Page " + d.path
|
|
}
|
|
if !d.noTitle {
|
|
b.WriteString("<title>" + html.EscapeString(title) + "</title>")
|
|
}
|
|
if !d.noDesc {
|
|
desc := d.description
|
|
if desc == "" {
|
|
desc = "About " + d.path
|
|
}
|
|
b.WriteString(`<meta name="description" content="` + html.EscapeString(desc) + `">`)
|
|
}
|
|
if !d.noCanonical {
|
|
c := d.canonical
|
|
if c == "" {
|
|
c = siteURL + d.path
|
|
}
|
|
b.WriteString(`<link rel="canonical" href="` + html.EscapeString(c) + `">`)
|
|
}
|
|
if d.robots != "" {
|
|
b.WriteString(`<meta name="robots" content="` + d.robots + `">`)
|
|
}
|
|
b.WriteString(d.head + "</head><body><main>")
|
|
h1 := d.h1
|
|
if h1 == nil {
|
|
h1 = []string{title}
|
|
}
|
|
for _, h := range h1 {
|
|
b.WriteString("<h1>" + html.EscapeString(h) + "</h1>")
|
|
}
|
|
b.WriteString(d.body)
|
|
for _, l := range d.links {
|
|
b.WriteString(`<p><a href="` + l + `">Read about ` + html.EscapeString(strings.Trim(l, "/")) + `</a></p>`)
|
|
}
|
|
b.WriteString("</main></body></html>")
|
|
return b.String()
|
|
}
|
|
|
|
// plant serves the documents on the site.
|
|
func (s *fakeSite) plant(docs ...doc) {
|
|
for _, d := range docs {
|
|
s.html(d.path, d.render())
|
|
}
|
|
}
|
|
|
|
// The fake content plugin acme.crawl: in-memory records behind two types
|
|
// with descriptor facts and fixing tools, so the audit's content routes
|
|
// (P33-38) are exercised. acme-crawl-post (/c/:slug) names edit, headings
|
|
// and image_alt tools; acme-crawl-category (/cat/:slug) names only a
|
|
// headings tool.
|
|
const (
|
|
crawlPostType = "acme-crawl-post"
|
|
crawlCategoryType = "acme-crawl-category"
|
|
)
|
|
|
|
// crawlRecord is one in-memory record.
|
|
type crawlRecord struct {
|
|
id string
|
|
path string
|
|
title string
|
|
words int
|
|
headings []pact.ContentHeading
|
|
images []pact.ContentImage
|
|
topHeading int
|
|
parents []pact.ContentRef
|
|
}
|
|
|
|
var (
|
|
crawlRecordsMu sync.Mutex
|
|
crawlRecords = map[string][]crawlRecord{}
|
|
)
|
|
|
|
// setCrawlRecords replaces the records of the fake types for one test.
|
|
func setCrawlRecords(t *testing.T, byType map[string][]crawlRecord) {
|
|
t.Helper()
|
|
crawlRecordsMu.Lock()
|
|
crawlRecords = byType
|
|
crawlRecordsMu.Unlock()
|
|
t.Cleanup(func() {
|
|
crawlRecordsMu.Lock()
|
|
crawlRecords = map[string][]crawlRecord{}
|
|
crawlRecordsMu.Unlock()
|
|
})
|
|
}
|
|
|
|
func crawlRecordsOf(typeKey string) []crawlRecord {
|
|
crawlRecordsMu.Lock()
|
|
defer crawlRecordsMu.Unlock()
|
|
return slices.Clone(crawlRecords[typeKey])
|
|
}
|
|
|
|
func (r crawlRecord) content() pact.ContentRecord {
|
|
words := r.words
|
|
var wp *int
|
|
if words > 0 {
|
|
wp = &words
|
|
}
|
|
return pact.ContentRecord{
|
|
ID: r.id, Published: true, TopHeading: r.topHeading, Parents: r.parents,
|
|
Locales: []pact.ContentLocale{{Locale: "en", URL: r.path, Title: r.title, Words: wp, Headings: r.headings, Images: r.images}},
|
|
}
|
|
}
|
|
|
|
type crawlContentPlugin struct{}
|
|
|
|
func (crawlContentPlugin) ID() string { return "acme.crawl" }
|
|
func (crawlContentPlugin) Requires() []string { return nil }
|
|
func (crawlContentPlugin) Register(*backpack.App) error { return nil }
|
|
func (crawlContentPlugin) Boot(*backpack.App) error { return nil }
|
|
func (crawlContentPlugin) MenuItemTypes() []pact.MenuItemType {
|
|
return []pact.MenuItemType{
|
|
crawlType(crawlPostType, "/c/:slug", map[string]string{
|
|
pact.ContentToolEdit: "acme_edit_post", pact.ContentToolHeadings: "acme_fix_headings", pact.ContentToolImageAlt: "acme_set_alt",
|
|
}),
|
|
crawlType(crawlCategoryType, "/cat/:slug", map[string]string{pact.ContentToolHeadings: "acme_category_headings"}),
|
|
}
|
|
}
|
|
|
|
// crawlType is one fake type over the in-memory records.
|
|
func crawlType(key, pattern string, tools map[string]string) pact.MenuItemType {
|
|
find := func(match func(crawlRecord) bool) *crawlRecord {
|
|
for _, r := range crawlRecordsOf(key) {
|
|
if match(r) {
|
|
return &r
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
return pact.MenuItemType{
|
|
Key: key, Label: key, DefaultPattern: pattern, Placeholders: []string{"slug"},
|
|
Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { return nil, nil },
|
|
Content: &pact.ContentDescriptor{
|
|
Model: "acme_crawl_" + strings.ReplaceAll(key, "-", "_"),
|
|
Tools: tools,
|
|
Describe: func(_ context.Context, in pact.ContentDescribe) (*pact.ContentRecord, error) {
|
|
r := find(func(r crawlRecord) bool { return r.id == in.ID })
|
|
if r == nil {
|
|
return nil, nil
|
|
}
|
|
rec := r.content()
|
|
return &rec, nil
|
|
},
|
|
Records: func(_ context.Context, in pact.ContentList) (pact.ContentPage, error) {
|
|
var out pact.ContentPage
|
|
if in.After != "" {
|
|
return out, nil
|
|
}
|
|
for _, r := range crawlRecordsOf(key) {
|
|
out.Records = append(out.Records, r.content())
|
|
}
|
|
return out, nil
|
|
},
|
|
Match: func(_ context.Context, in pact.ContentMatch) (*pact.ContentMatchResult, error) {
|
|
if _, ok := signpost.MatchPattern(in.Pattern, in.Path); !ok {
|
|
return nil, nil
|
|
}
|
|
r := find(func(r crawlRecord) bool { return r.path == in.Path })
|
|
if r == nil {
|
|
return nil, nil
|
|
}
|
|
return &pact.ContentMatchResult{ID: r.id, Locale: "en", Exact: true}, nil
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
var _ pact.HasMenuItemTypes = crawlContentPlugin{}
|
|
|
|
func init() {
|
|
party.Register(crawlContentPlugin{})
|
|
}
|
|
|
|
// bootCrawlEnv boots acme.pages, acme.crawl and golem15.seo.
|
|
func bootCrawlEnv(t *testing.T, extraEnv ...string) *testEnv {
|
|
t.Helper()
|
|
return bootPlugins(t, []string{"acme.pages", "acme.crawl", "golem15.seo"}, extraEnv...)
|
|
}
|
|
|
|
// crawlJob returns the SEO plugin's crawl job.
|
|
func (e *testEnv) crawlJob(t *testing.T) pact.Job {
|
|
t.Helper()
|
|
for _, j := range e.seoPlugin(t).Jobs() {
|
|
if info, ok := conga.Describe(j); ok && info.Kind == crawl.CrawlKind {
|
|
return j
|
|
}
|
|
}
|
|
t.Fatal("the SEO plugin registers no crawl job")
|
|
return nil
|
|
}
|
|
|
|
// siteCtx is a context whose crawler requests go to site.
|
|
func siteCtx(t *testing.T, site *fakeSite) context.Context {
|
|
return fetchguard.WithTransport(t.Context(), site)
|
|
}
|
|
|
|
// startCrawl starts a crawl through the API and returns its id.
|
|
func (e *testEnv) startCrawl(t *testing.T, token string) uint {
|
|
t.Helper()
|
|
rec := e.expect(t, http.StatusAccepted, http.MethodPost, "/_seo/api/v1/crawls", token, map[string]any{})
|
|
var out struct {
|
|
Data struct {
|
|
ID uint `json:"id"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil || out.Data.ID == 0 {
|
|
t.Fatalf("start crawl: %s (%v)", rec.Body.String(), err)
|
|
}
|
|
return out.Data.ID
|
|
}
|
|
|
|
// runCrawl runs the crawl job of id against the site.
|
|
func (e *testEnv) runCrawl(t *testing.T, ctx context.Context, id uint) {
|
|
t.Helper()
|
|
if err := e.crawlJob(t).Work(ctx, crawl.CrawlArgs{CrawlID: id}); err != nil {
|
|
t.Fatalf("crawl job: %v", err)
|
|
}
|
|
}
|
|
|
|
// crawlOnce starts a crawl and runs it against the site.
|
|
func (e *testEnv) crawlOnce(t *testing.T, token string, ctx context.Context) uint {
|
|
t.Helper()
|
|
id := e.startCrawl(t, token)
|
|
e.runCrawl(t, ctx, id)
|
|
return id
|
|
}
|
|
|
|
// apiIssue is one issue of GET /_seo/api/v1/issues.
|
|
type apiIssue struct {
|
|
ID uint `json:"id"`
|
|
Kind string `json:"kind"`
|
|
URL string `json:"url"`
|
|
Locale string `json:"locale"`
|
|
Details map[string]any `json:"details"`
|
|
Tool string `json:"tool"`
|
|
Severity string `json:"severity"`
|
|
Source string `json:"source"`
|
|
Fix string `json:"fix"`
|
|
Ignored bool `json:"ignored"`
|
|
CrawlID *uint `json:"crawl_id"`
|
|
}
|
|
|
|
// issues lists the audit's issues through the API (query appended).
|
|
func (e *testEnv) issues(t *testing.T, token, query string) []apiIssue {
|
|
t.Helper()
|
|
path := "/_seo/api/v1/issues"
|
|
if query != "" {
|
|
path += "?" + query
|
|
}
|
|
rec := e.expect(t, http.StatusOK, http.MethodGet, path, token, nil)
|
|
var out struct {
|
|
Data []apiIssue `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out.Data
|
|
}
|
|
|
|
// issueSet is "kind url" for every issue of source, sorted.
|
|
func issueSet(list []apiIssue, source string) []string {
|
|
var out []string
|
|
for _, i := range list {
|
|
if source == "" || i.Source == source {
|
|
out = append(out, i.Kind+" "+i.URL)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// findIssue returns the one issue of kind at url, failing when there is
|
|
// not exactly one.
|
|
func findIssue(t *testing.T, list []apiIssue, kind, url string) apiIssue {
|
|
t.Helper()
|
|
var found []apiIssue
|
|
for _, i := range list {
|
|
if i.Kind == kind && i.URL == url {
|
|
found = append(found, i)
|
|
}
|
|
}
|
|
if len(found) != 1 {
|
|
t.Fatalf("%d %s issues for %s, want 1; all: %v", len(found), kind, url, issueSet(list, ""))
|
|
}
|
|
return found[0]
|
|
}
|