package seo_test import ( "context" "encoding/json" "html" "io" "net/http" "slices" "sort" "strings" "sync" "testing" "git.golem15.com/golem15/sm-seo-plugin/classes/crawl" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/conga" "git.golem15.com/golem15/summercms/modules/fetchguard" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/party" "git.golem15.com/golem15/summercms/modules/signpost" ) // The test site the crawler fetches: a fake http.RoundTripper handed to // the crawler through fetchguard.WithTransport, so the crawler's real // requests (URL checks, User-Agent, no redirects followed) are asserted // offline. Paths it does not know answer 404. // sitePage is one answer of the test site. type sitePage struct { status int body string header map[string]string } // fakeSite serves the test site and records every request URL. type fakeSite struct { mu sync.Mutex pages map[string]sitePage requests []string agents []string } func newFakeSite() *fakeSite { return &fakeSite{pages: map[string]sitePage{}} } // set serves body with status at path (path with query). func (s *fakeSite) set(path string, status int, body string, header ...string) { s.mu.Lock() defer s.mu.Unlock() h := map[string]string{"Content-Type": "text/html; charset=utf-8"} for i := 0; i+1 < len(header); i += 2 { h[header[i]] = header[i+1] } s.pages[path] = sitePage{status: status, body: body, header: h} } // html serves a 200 HTML page. func (s *fakeSite) html(path, body string) { s.set(path, http.StatusOK, body) } func (s *fakeSite) RoundTrip(r *http.Request) (*http.Response, error) { s.mu.Lock() s.requests = append(s.requests, r.URL.String()) s.agents = append(s.agents, r.Header.Get("User-Agent")) key := r.URL.EscapedPath() if r.URL.RawQuery != "" { key += "?" + r.URL.RawQuery } p, ok := s.pages[key] s.mu.Unlock() if !ok || !strings.EqualFold(r.URL.Hostname(), "www.example.test") { p = sitePage{status: http.StatusNotFound, body: "Not found", header: map[string]string{"Content-Type": "text/html"}} } header := http.Header{} for k, v := range p.header { header.Set(k, v) } return &http.Response{StatusCode: p.status, Header: header, Body: io.NopCloser(strings.NewReader(p.body)), Request: r}, nil } // requested reports whether any request went to an URL containing part. func (s *fakeSite) requested(part string) bool { s.mu.Lock() defer s.mu.Unlock() for _, u := range s.requests { if strings.Contains(u, part) { return true } } return false } // sitemap serves /sitemap.xml as an index of one file listing paths. func (s *fakeSite) sitemap(paths ...string) { s.set("/sitemap.xml", http.StatusOK, ``+siteURL+`/sitemap-pages-1.xml`, "Content-Type", "application/xml") var b strings.Builder b.WriteString(``) for _, p := range paths { b.WriteString("" + siteURL + html.EscapeString(p) + "") } b.WriteString("") s.set("/sitemap-pages-1.xml", http.StatusOK, b.String(), "Content-Type", "application/xml") } // doc is one planted page: a complete, clean document (unique title and // description, a self canonical, one H1 matching the title) that a test // mutates for the kind it plants. type doc struct { path string lang string title string noTitle bool description string noDesc bool canonical string noCanonical bool robots string head string h1 []string body string links []string } // render writes the document's HTML. func (d doc) render() string { var b strings.Builder lang := d.lang if lang == "" { lang = "en" } b.WriteString(``) title := d.title if title == "" { title = "Page " + d.path } if !d.noTitle { b.WriteString("" + html.EscapeString(title) + "") } if !d.noDesc { desc := d.description if desc == "" { desc = "About " + d.path } b.WriteString(``) } if !d.noCanonical { c := d.canonical if c == "" { c = siteURL + d.path } b.WriteString(``) } if d.robots != "" { b.WriteString(``) } b.WriteString(d.head + "
") h1 := d.h1 if h1 == nil { h1 = []string{title} } for _, h := range h1 { b.WriteString("

" + html.EscapeString(h) + "

") } b.WriteString(d.body) for _, l := range d.links { b.WriteString(`

Read about ` + html.EscapeString(strings.Trim(l, "/")) + `

`) } b.WriteString("
") return b.String() } // plant serves the documents on the site. func (s *fakeSite) plant(docs ...doc) { for _, d := range docs { s.html(d.path, d.render()) } } // The fake content plugin acme.crawl: in-memory records behind two types // with descriptor facts and fixing tools, so the audit's content routes // (P33-38) are exercised. acme-crawl-post (/c/:slug) names edit, headings // and image_alt tools; acme-crawl-category (/cat/:slug) names only a // headings tool. const ( crawlPostType = "acme-crawl-post" crawlCategoryType = "acme-crawl-category" ) // crawlRecord is one in-memory record. type crawlRecord struct { id string path string title string words int headings []pact.ContentHeading images []pact.ContentImage topHeading int parents []pact.ContentRef } var ( crawlRecordsMu sync.Mutex crawlRecords = map[string][]crawlRecord{} ) // setCrawlRecords replaces the records of the fake types for one test. func setCrawlRecords(t *testing.T, byType map[string][]crawlRecord) { t.Helper() crawlRecordsMu.Lock() crawlRecords = byType crawlRecordsMu.Unlock() t.Cleanup(func() { crawlRecordsMu.Lock() crawlRecords = map[string][]crawlRecord{} crawlRecordsMu.Unlock() }) } func crawlRecordsOf(typeKey string) []crawlRecord { crawlRecordsMu.Lock() defer crawlRecordsMu.Unlock() return slices.Clone(crawlRecords[typeKey]) } func (r crawlRecord) content() pact.ContentRecord { words := r.words var wp *int if words > 0 { wp = &words } return pact.ContentRecord{ ID: r.id, Published: true, TopHeading: r.topHeading, Parents: r.parents, Locales: []pact.ContentLocale{{Locale: "en", URL: r.path, Title: r.title, Words: wp, Headings: r.headings, Images: r.images}}, } } type crawlContentPlugin struct{} func (crawlContentPlugin) ID() string { return "acme.crawl" } func (crawlContentPlugin) Requires() []string { return nil } func (crawlContentPlugin) Register(*backpack.App) error { return nil } func (crawlContentPlugin) Boot(*backpack.App) error { return nil } func (crawlContentPlugin) MenuItemTypes() []pact.MenuItemType { return []pact.MenuItemType{ crawlType(crawlPostType, "/c/:slug", map[string]string{ pact.ContentToolEdit: "acme_edit_post", pact.ContentToolHeadings: "acme_fix_headings", pact.ContentToolImageAlt: "acme_set_alt", }), crawlType(crawlCategoryType, "/cat/:slug", map[string]string{pact.ContentToolHeadings: "acme_category_headings"}), } } // crawlType is one fake type over the in-memory records. func crawlType(key, pattern string, tools map[string]string) pact.MenuItemType { find := func(match func(crawlRecord) bool) *crawlRecord { for _, r := range crawlRecordsOf(key) { if match(r) { return &r } } return nil } return pact.MenuItemType{ Key: key, Label: key, DefaultPattern: pattern, Placeholders: []string{"slug"}, Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { return nil, nil }, Content: &pact.ContentDescriptor{ Model: "acme_crawl_" + strings.ReplaceAll(key, "-", "_"), Tools: tools, Describe: func(_ context.Context, in pact.ContentDescribe) (*pact.ContentRecord, error) { r := find(func(r crawlRecord) bool { return r.id == in.ID }) if r == nil { return nil, nil } rec := r.content() return &rec, nil }, Records: func(_ context.Context, in pact.ContentList) (pact.ContentPage, error) { var out pact.ContentPage if in.After != "" { return out, nil } for _, r := range crawlRecordsOf(key) { out.Records = append(out.Records, r.content()) } return out, nil }, Match: func(_ context.Context, in pact.ContentMatch) (*pact.ContentMatchResult, error) { if _, ok := signpost.MatchPattern(in.Pattern, in.Path); !ok { return nil, nil } r := find(func(r crawlRecord) bool { return r.path == in.Path }) if r == nil { return nil, nil } return &pact.ContentMatchResult{ID: r.id, Locale: "en", Exact: true}, nil }, }, } } var _ pact.HasMenuItemTypes = crawlContentPlugin{} func init() { party.Register(crawlContentPlugin{}) } // bootCrawlEnv boots acme.pages, acme.crawl and golem15.seo. func bootCrawlEnv(t *testing.T, extraEnv ...string) *testEnv { t.Helper() return bootPlugins(t, []string{"acme.pages", "acme.crawl", "golem15.seo"}, extraEnv...) } // crawlJob returns the SEO plugin's crawl job. func (e *testEnv) crawlJob(t *testing.T) pact.Job { t.Helper() for _, j := range e.seoPlugin(t).Jobs() { if info, ok := conga.Describe(j); ok && info.Kind == crawl.CrawlKind { return j } } t.Fatal("the SEO plugin registers no crawl job") return nil } // siteCtx is a context whose crawler requests go to site. func siteCtx(t *testing.T, site *fakeSite) context.Context { return fetchguard.WithTransport(t.Context(), site) } // startCrawl starts a crawl through the API and returns its id. func (e *testEnv) startCrawl(t *testing.T, token string) uint { t.Helper() rec := e.expect(t, http.StatusAccepted, http.MethodPost, "/_seo/api/v1/crawls", token, map[string]any{}) var out struct { Data struct { ID uint `json:"id"` } `json:"data"` } if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil || out.Data.ID == 0 { t.Fatalf("start crawl: %s (%v)", rec.Body.String(), err) } return out.Data.ID } // runCrawl runs the crawl job of id against the site. func (e *testEnv) runCrawl(t *testing.T, ctx context.Context, id uint) { t.Helper() if err := e.crawlJob(t).Work(ctx, crawl.CrawlArgs{CrawlID: id}); err != nil { t.Fatalf("crawl job: %v", err) } } // crawlOnce starts a crawl and runs it against the site. func (e *testEnv) crawlOnce(t *testing.T, token string, ctx context.Context) uint { t.Helper() id := e.startCrawl(t, token) e.runCrawl(t, ctx, id) return id } // apiIssue is one issue of GET /_seo/api/v1/issues. type apiIssue struct { ID uint `json:"id"` Kind string `json:"kind"` URL string `json:"url"` Locale string `json:"locale"` Details map[string]any `json:"details"` Tool string `json:"tool"` Severity string `json:"severity"` Source string `json:"source"` Fix string `json:"fix"` Ignored bool `json:"ignored"` CrawlID *uint `json:"crawl_id"` } // issues lists the audit's issues through the API (query appended). func (e *testEnv) issues(t *testing.T, token, query string) []apiIssue { t.Helper() path := "/_seo/api/v1/issues" if query != "" { path += "?" + query } rec := e.expect(t, http.StatusOK, http.MethodGet, path, token, nil) var out struct { Data []apiIssue `json:"data"` } if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { t.Fatal(err) } return out.Data } // issueSet is "kind url" for every issue of source, sorted. func issueSet(list []apiIssue, source string) []string { var out []string for _, i := range list { if source == "" || i.Source == source { out = append(out, i.Kind+" "+i.URL) } } sort.Strings(out) return out } // findIssue returns the one issue of kind at url, failing when there is // not exactly one. func findIssue(t *testing.T, list []apiIssue, kind, url string) apiIssue { t.Helper() var found []apiIssue for _, i := range list { if i.Kind == kind && i.URL == url { found = append(found, i) } } if len(found) != 1 { t.Fatalf("%d %s issues for %s, want 1; all: %v", len(found), kind, url, issueSet(list, "")) } return found[0] }