From c2b61bf8e9e42603c02403add6b5da19d1fde5a6 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Wed, 4 Nov 2026 15:05:45 +0100 Subject: [PATCH] feat: crawler finds a page with two H1 headings and stores the issue (tracer) - golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37) - classes/crawl: fetchguard client on the site host and its www twin, sitemap index and robots.txt seeds, two workers with a delay, limits - h1_missing and h1_multiple with the P33-38 fix route - POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues answers the stored issues with id, severity, source, fix, found_at, ignored and crawl_id --- classes/crawl/checks.go | 151 +++++++ classes/crawl/crawler.go | 425 ++++++++++++++++++ classes/crawl/fetch.go | 191 ++++++++ classes/crawl/fix.go | 145 ++++++ classes/crawl/job.go | 361 +++++++++++++++ classes/crawl/parse.go | 293 ++++++++++++ classes/crawl/render.go | 43 ++ classes/crawl/store.go | 248 ++++++++++ classes/issues.go | 9 + config/config.yaml | 7 + controllers/api/crawls.go | 90 ++++ controllers/api/issues.go | 87 +++- crawl_site_test.go | 417 +++++++++++++++++ crawler_test.go | 48 ++ go.mod | 2 +- jobs.go | 16 +- lang/en/lang.yaml | 5 + lang/pl/lang.yaml | 5 + models/crawl.go | 103 +++++ models/crawl_page.go | 66 +++ models/issue.go | 156 +++++++ routes.go | 1 + .../202610280005_create_golem15_seo_crawls.go | 111 +++++ 23 files changed, 2967 insertions(+), 13 deletions(-) create mode 100644 classes/crawl/checks.go create mode 100644 classes/crawl/crawler.go create mode 100644 classes/crawl/fetch.go create mode 100644 classes/crawl/fix.go create mode 100644 classes/crawl/job.go create mode 100644 classes/crawl/parse.go create mode 100644 classes/crawl/render.go create mode 100644 classes/crawl/store.go create mode 100644 controllers/api/crawls.go create mode 100644 crawl_site_test.go create mode 100644 crawler_test.go create mode 100644 models/crawl.go create mode 100644 models/crawl_page.go create mode 100644 models/issue.go create mode 100644 updates/202610280005_create_golem15_seo_crawls.go diff --git a/classes/crawl/checks.go b/classes/crawl/checks.go new file mode 100644 index 0000000..6150f30 --- /dev/null +++ b/classes/crawl/checks.go @@ -0,0 +1,151 @@ +package crawl + +import ( + "git.golem15.com/golem15/sm-seo-plugin/models" + "git.golem15.com/golem15/summercms/modules/pact" +) + +// The crawl kinds of UI-SPEC C23. +const ( + KindPageStatus = "page_status" + KindSitemapError = "sitemap_error" + KindSitemapNoindex = "sitemap_noindex" + KindSitemapBlocked = "sitemap_blocked" + KindSitemapMissing = "sitemap_missing" + KindBrokenLink = "broken_link" + KindRedirectedLink = "redirected_link" + KindCanonicalMissing = "canonical_missing" + KindCanonicalError = "canonical_error" + KindMissingTitle = "missing_title" + KindDuplicateTitle = "duplicate_title" + KindMissingDesc = "missing_description" + KindDuplicateDesc = "duplicate_description" + KindH1Missing = "h1_missing" + KindH1Multiple = "h1_multiple" + KindHeadingSkipped = "heading_skipped" + KindHeadingEmpty = "heading_empty" + KindH1TitleMismatch = "h1_title_mismatch" + KindJSONLDInvalid = "jsonld_invalid" + KindJSONLDMultiple = "jsonld_multiple" + KindJSONLDRequired = "jsonld_required" + KindJSONLDRecommended = "jsonld_recommended" + KindHreflangRecip = "hreflang_reciprocal" + KindHreflangError = "hreflang_error" + KindGenericAnchor = "generic_anchor" + KindImageAlt = "image_alt" + KindImageSize = "image_size" + KindFeaturedLazy = "featured_lazy" + KindContentJS = "content_js" + KindOrphanPage = "orphan_page" + // KindH1InContent is the CMS-data check of a record whose body holds + // an H1 although its page renders the title as the H1. + KindH1InContent = "h1_in_content" +) + +// severities are the default severities of the kinds table (UI-SPEC C23), +// the Phase 32 CMS kinds included. +var severities = map[string]string{ + KindPageStatus: models.SeverityError, + KindSitemapError: models.SeverityError, + KindSitemapNoindex: models.SeverityError, + KindSitemapBlocked: models.SeverityError, + KindSitemapMissing: models.SeverityNotice, + KindBrokenLink: models.SeverityError, + KindRedirectedLink: models.SeverityNotice, + KindCanonicalMissing: models.SeverityWarning, + KindCanonicalError: models.SeverityError, + "canonical_noindex": models.SeverityError, + "canonical_redirected": models.SeverityWarning, + KindMissingTitle: models.SeverityWarning, + KindDuplicateTitle: models.SeverityWarning, + KindMissingDesc: models.SeverityNotice, + KindDuplicateDesc: models.SeverityNotice, + "rule_overrides_record": models.SeverityNotice, + "redirect_chain": models.SeverityWarning, + "noindex_blocked": models.SeverityError, + KindH1Missing: models.SeverityError, + KindH1Multiple: models.SeverityError, + KindH1InContent: models.SeverityWarning, + KindHeadingSkipped: models.SeverityWarning, + KindHeadingEmpty: models.SeverityWarning, + KindH1TitleMismatch: models.SeverityNotice, + KindJSONLDInvalid: models.SeverityError, + KindJSONLDMultiple: models.SeverityWarning, + KindJSONLDRequired: models.SeverityError, + KindJSONLDRecommended: models.SeverityNotice, + KindHreflangRecip: models.SeverityWarning, + KindHreflangError: models.SeverityError, + KindGenericAnchor: models.SeverityWarning, + KindImageAlt: models.SeverityWarning, + KindImageSize: models.SeverityNotice, + KindFeaturedLazy: models.SeverityNotice, + KindContentJS: models.SeverityWarning, + KindOrphanPage: models.SeverityWarning, +} + +// Severity returns a kind's severity (warning for an unknown kind). +func Severity(kind string) string { + if s, ok := severities[kind]; ok { + return s + } + return models.SeverityWarning +} + +// Kinds lists every kind the audit stores, in the kinds table order. +func Kinds() []string { + return []string{ + KindPageStatus, KindSitemapError, KindSitemapNoindex, KindSitemapBlocked, + KindSitemapMissing, KindBrokenLink, KindRedirectedLink, KindCanonicalMissing, + KindCanonicalError, "canonical_noindex", "canonical_redirected", KindMissingTitle, + KindDuplicateTitle, KindMissingDesc, KindDuplicateDesc, "rule_overrides_record", + "redirect_chain", "noindex_blocked", KindH1Missing, KindH1Multiple, KindH1InContent, + KindHeadingSkipped, KindHeadingEmpty, KindH1TitleMismatch, KindJSONLDInvalid, + KindJSONLDMultiple, KindJSONLDRequired, KindJSONLDRecommended, KindHreflangRecip, + KindHreflangError, KindGenericAnchor, KindImageAlt, KindImageSize, KindFeaturedLazy, + KindContentJS, KindOrphanPage, + } +} + +// Env is what the checks know besides the crawl: the record behind an +// address. +type Env struct { + // Lookup returns the CMS record of an address, nil when none. + Lookup func(key string) *Record +} + +func (e *Env) record(key string) *Record { + if e == nil || e.Lookup == nil { + return nil + } + return e.Lookup(key) +} + +// Check runs the per-page checks over a crawl's pages. +func Check(res *Result, env *Env) []Finding { + var out []Finding + for _, key := range res.Order { + p := res.Pages[key] + if !p.OK() { + continue + } + out = append(out, checkHeadings(p, env.record(key))...) + } + return out +} + +// checkHeadings reports h1_missing and h1_multiple. An H1 inside the +// record's own content is fixed through the record's headings tool. +func checkHeadings(p *Page, rec *Record) []Finding { + h1 := p.Facts.H1() + switch { + case len(h1) == 0: + return []Finding{{Kind: KindH1Missing, URL: p.Key, Record: rec}} + case len(h1) > 1: + f := Finding{Kind: KindH1Multiple, URL: p.Key, Record: rec, Details: map[string]any{"count": len(h1), "h1": h1}} + if rec.hasHeading(1, "") { + f.Content = []string{pact.ContentToolHeadings, pact.ContentToolEdit} + } + return []Finding{f} + } + return nil +} diff --git a/classes/crawl/crawler.go b/classes/crawl/crawler.go new file mode 100644 index 0000000..c0cd74a --- /dev/null +++ b/classes/crawl/crawler.go @@ -0,0 +1,425 @@ +package crawl + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "io" + "net/url" + "strings" + "sync" + "time" +) + +// ErrNoSiteURL is a crawl started without a site address +// (golem15.seo.site_url, else app.url). +var ErrNoSiteURL = errors.New("golem15.seo: the site address for crawls is not configured") + +// The paths the crawler seeds from (RESEARCH Pattern 7). +const ( + HomePath = "/" + SitemapPath = "/sitemap.xml" + RobotsPath = "/robots.txt" +) + +// maxSitemapFiles caps the sitemap files read from an index. +const maxSitemapFiles = 500 + +// How a page entered the frontier. +const ( + viaHome = "home" + viaSitemap = "sitemap" + viaLink = "link" + viaRedirect = "redirect" + viaCanonical = "canonical" + viaAlternate = "alternate" +) + +// Page is one fetched address of a crawl: the answer, the facts of its raw +// HTML, the facts of its JavaScript-rendered DOM when the comparison ran, +// its depth from the home page and the sitemap file that lists it. +type Page struct { + Key string + Depth int + Via string + Resp *Response + Facts *PageFacts + Rendered *PageFacts + InSitemap bool + SitemapFile string + FetchedAt time.Time +} + +// OK reports whether the page answered 200 with HTML. +func (p *Page) OK() bool { + return p != nil && p.Resp != nil && p.Resp.Err == "" && p.Resp.Status == 200 && p.Facts != nil +} + +// SitemapFile is one sitemap the crawl read: its address, status and the +// page addresses it lists. +type SitemapFile struct { + Key string + Status int + URLs []string +} + +// RenderState is the outcome of the rendered comparison (P33-55). +type RenderState struct { + Status string + Pages int + Note string +} + +// Result is what one crawl fetched, in frontier order. +type Result struct { + Pages map[string]*Page + Order []string + Sitemaps []SitemapFile + // Listed maps a page address to the sitemap file listing it. + Listed map[string]string + Robots string + RobotsStatus int + LimitHit bool + Canceled bool + Render RenderState + fetcher *Fetcher +} + +// Page returns the fetched page at key, nil when none. +func (r *Result) Page(key string) *Page { + if r == nil { + return nil + } + return r.Pages[key] +} + +// Options are the limits and seams of one crawl. +type Options struct { + // MaxPages caps the fetched addresses (D-14, default 5000). + MaxPages int + // Delay is the pause between two requests (default 250 ms). + Delay time.Duration + // Renderer renders pages for the comparison; nil skips it. + Renderer Renderer + // RenderMax caps the rendered pages (default 200, crawl order). + RenderMax int + // Canceled is asked between requests; true stops the crawl and keeps + // what was fetched. + Canceled func(ctx context.Context) bool + // Progress is told the number of fetched pages. + Progress func(ctx context.Context, fetched int) +} + +// item is one frontier entry. +type item struct { + key string + depth int + via string +} + +// pacer spaces requests at least delay apart across the workers. +type pacer struct { + mu sync.Mutex + next time.Time + delay time.Duration +} + +func (p *pacer) wait(ctx context.Context) error { + p.mu.Lock() + now := time.Now() + at := p.next + if at.Before(now) { + at = now + } + p.next = at.Add(p.delay) + p.mu.Unlock() + if d := time.Until(at); d > 0 { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + } + } + return nil +} + +// Crawl fetches the site through f, a fetchguard client limited to the +// site host and its www twin (fetchguard.AllowHostsMode, HTTPS, no +// redirects followed; TrustedMode only from Go config): robots.txt, the +// sitemap index and its files, then breadth-first from the home page and +// the sitemap addresses over same-host links, redirect targets, canonicals +// and hreflang alternates, two requests at a time with the delay between +// requests, up to MaxPages. robots.txt rules are read for the report, not +// obeyed (P33-41). Cancelling keeps what was fetched. After each HTML page +// answering 200, a configured Renderer renders it (one tab at a time, up +// to RenderMax pages) for the raw versus rendered comparison. +func Crawl(ctx context.Context, f *Fetcher, opt Options) (*Result, error) { + if opt.MaxPages <= 0 { + opt.MaxPages = 5000 + } + if opt.RenderMax <= 0 { + opt.RenderMax = DefaultRenderMax + } + res := &Result{Pages: map[string]*Page{}, Listed: map[string]string{}, Render: RenderState{Status: renderStatusSkipped}, fetcher: f} + if opt.Renderer != nil { + res.Render.Status = renderStatusDone + } + pace := &pacer{delay: opt.Delay} + canceled := func() bool { + if ctx.Err() != nil { + return true + } + return opt.Canceled != nil && opt.Canceled(ctx) + } + + // robots.txt, for the sitemap_blocked check. + if err := pace.wait(ctx); err != nil { + return res, err + } + if r := f.Get(ctx, RobotsPath); r.Err == "" { + res.RobotsStatus = r.Status + if r.Status == 200 { + res.Robots = string(r.Body) + } + } + + // The sitemap index and its files. + if err := readSitemaps(ctx, f, pace, res); err != nil { + return res, err + } + + seen := map[string]bool{} + var queue []item + push := func(key string, depth int, via string) { + if key == "" || seen[key] { + return + } + seen[key] = true + queue = append(queue, item{key: key, depth: depth, via: via}) + } + push(HomePath, 0, viaHome) + for _, sm := range res.Sitemaps { + for _, key := range sm.URLs { + push(key, 1, viaSitemap) + } + } + + var renderMu sync.Mutex + rendered := 0 + renderOff := opt.Renderer == nil + for len(queue) > 0 { + if canceled() { + res.Canceled = true + break + } + if len(res.Order) >= opt.MaxPages { + res.LimitHit = true + break + } + n := Workers + if room := opt.MaxPages - len(res.Order); n > room { + n = room + } + if n > len(queue) { + n = len(queue) + } + batch := queue[:n] + queue = queue[n:] + pages := make([]*Page, len(batch)) + var wg sync.WaitGroup + for i, it := range batch { + wg.Add(1) + go func(i int, it item) { + defer wg.Done() + p := &Page{Key: it.key, Depth: it.depth, Via: it.via} + if err := pace.wait(ctx); err != nil { + p.Resp = &Response{Err: "canceled"} + pages[i] = p + return + } + p.Resp = f.Get(ctx, it.key) + p.FetchedAt = time.Now().UTC() + if p.Resp.Err == "" && p.Resp.Status == 200 && p.Resp.HTML() { + base, _ := url.Parse(f.Absolute(it.key)) + p.Facts = Parse(base, p.Resp.Body) + renderMu.Lock() + if !renderOff && rendered < opt.RenderMax && ctx.Err() == nil { + rendered++ + dom, err := opt.Renderer.Render(ctx, f.Absolute(it.key)) + if err != nil { + renderOff = true + rendered-- + res.Render = RenderState{Status: renderStatusFailed, Note: err.Error()} + } else { + p.Rendered = Parse(base, []byte(dom)) + } + } + renderMu.Unlock() + } + pages[i] = p + }(i, it) + } + wg.Wait() + for _, p := range pages { + if file, ok := res.Listed[p.Key]; ok { + p.InSitemap, p.SitemapFile = true, file + } + res.Pages[p.Key] = p + res.Order = append(res.Order, p.Key) + p.Resp.Body = nil + for _, next := range discovered(f, p) { + push(next.key, p.Depth+1, next.via) + } + } + if opt.Progress != nil { + opt.Progress(ctx, len(res.Order)) + } + } + if len(queue) > 0 && !res.Canceled && len(res.Order) >= opt.MaxPages { + res.LimitHit = true + } + res.Render.Pages = rendered + return res, nil +} + +// discovered lists the same-host addresses a fetched page leads to: its +// redirect target, its links, canonical and hreflang alternates. +func discovered(f *Fetcher, p *Page) []item { + var out []item + add := func(raw string, via string) { + u, err := url.Parse(raw) + if err != nil { + return + } + u.Fragment = "" + if key := f.Key(u); key != "" { + out = append(out, item{key: key, via: via}) + } + } + if p.Resp != nil && p.Resp.Location != "" { + add(p.Resp.Location, viaRedirect) + } + if p.Facts == nil { + return out + } + for _, l := range p.Facts.Links { + if l.URL != nil { + add(l.URL.String(), viaLink) + } + } + if p.Facts.Canonical != "" { + add(p.Facts.Canonical, viaCanonical) + } + for _, a := range p.Facts.Alternates { + if a.URL != nil { + add(a.URL.String(), viaAlternate) + } + } + return out +} + +// readSitemaps reads /sitemap.xml: an index's files (same host only) or a +// single urlset. +func readSitemaps(ctx context.Context, f *Fetcher, pace *pacer, res *Result) error { + if err := pace.wait(ctx); err != nil { + return err + } + r := f.Get(ctx, SitemapPath) + if r.Err != "" || r.Status != 200 { + res.Sitemaps = append(res.Sitemaps, SitemapFile{Key: SitemapPath, Status: r.Status}) + return nil + } + index, locs := parseSitemap(r.Body) + if !index { + res.addSitemap(f, SitemapFile{Key: SitemapPath, Status: 200}, locs) + return nil + } + for i, loc := range locs { + if i >= maxSitemapFiles { + break + } + u, err := url.Parse(loc) + if err != nil { + continue + } + key := f.Key(u) + if key == "" { + continue + } + if err := pace.wait(ctx); err != nil { + return err + } + sr := f.Get(ctx, key) + file := SitemapFile{Key: key, Status: sr.Status} + if sr.Err != "" || sr.Status != 200 { + res.Sitemaps = append(res.Sitemaps, file) + continue + } + _, urls := parseSitemap(sr.Body) + res.addSitemap(f, file, urls) + } + return nil +} + +func (res *Result) addSitemap(f *Fetcher, file SitemapFile, locs []string) { + for _, loc := range locs { + u, err := url.Parse(loc) + if err != nil { + continue + } + key := f.Key(u) + if key == "" { + continue + } + if _, dup := res.Listed[key]; !dup { + res.Listed[key] = file.Key + } + file.URLs = append(file.URLs, key) + } + res.Sitemaps = append(res.Sitemaps, file) +} + +// parseSitemap reads the values of a sitemap document; index is true +// for a . +func parseSitemap(body []byte) (index bool, locs []string) { + dec := xml.NewDecoder(io.LimitReader(bytes.NewReader(body), MaxBody)) + depth := 0 + inLoc := false + var parent string + var stack []string + for { + tok, err := dec.Token() + if err != nil { + break + } + switch t := tok.(type) { + case xml.StartElement: + depth++ + name := strings.ToLower(t.Name.Local) + if depth == 1 && name == "sitemapindex" { + index = true + } + stack = append(stack, name) + if name == "loc" && len(stack) >= 2 { + parent = stack[len(stack)-2] + inLoc = parent == "url" || parent == "sitemap" + } + case xml.EndElement: + depth-- + if len(stack) > 0 { + stack = stack[:len(stack)-1] + } + inLoc = false + case xml.CharData: + if inLoc { + if s := strings.TrimSpace(string(t)); s != "" { + locs = append(locs, s) + } + } + } + } + return index, locs +} diff --git a/classes/crawl/fetch.go b/classes/crawl/fetch.go new file mode 100644 index 0000000..3d7b238 --- /dev/null +++ b/classes/crawl/fetch.go @@ -0,0 +1,191 @@ +// Package crawl is the site audit's crawler (D-14): it fetches the +// rendered site like a search engine, checks every page (UI-SPEC C23), +// stores the findings with the CMS-data issues and names how each one is +// fixed (P33-38). +package crawl + +import ( + "context" + "net/http" + "net/url" + "strings" + "time" + + "git.golem15.com/golem15/summercms/modules/compass" + "git.golem15.com/golem15/summercms/modules/fetchguard" +) + +// The config keys of the crawler. +const ( + // TrustedConfigKey lets the crawler fetch plain http and private + // addresses (staging, local sites). Only Go config can set it. + TrustedConfigKey = "golem15.seo.crawl_trusted" +) + +// The crawler's limits (D-14, RESEARCH Pattern 7). +const ( + // MaxBody caps one answer's body. + MaxBody = 5 << 20 + // FetchTimeout is one request's timeout. + FetchTimeout = 10 * time.Second + // Workers is the number of requests at a time. + Workers = 2 + // MaxLinksPerPage caps the links stored and followed per page. + MaxLinksPerPage = 1000 +) + +// UserAgent is the crawler's identity (P33-41). +func UserAgent(site string) string { + return "SummerCMS-SEO-Crawler/1 (+" + site + ")" +} + +// Response is one fetched address: its status, Location, content type, +// X-Robots-Tag and body, or the fetch error. +type Response struct { + Status int + Location string + ContentType string + XRobots string + Body []byte + Err string +} + +// HTML reports whether the answer is an HTML page. +func (r *Response) HTML() bool { + ct := strings.ToLower(r.ContentType) + return ct == "" || strings.Contains(ct, "text/html") || strings.Contains(ct, "application/xhtml") +} + +// Fetcher fetches the site's own addresses through a fetchguard client +// limited to the site host and its www twin: HTTPS only, private +// addresses refused at dial time and redirects never followed. Trusted +// mode (golem15.seo.crawl_trusted, Go config only) allows plain http and +// private addresses, still only on those hosts. +type Fetcher struct { + client *fetchguard.Client + site *url.URL + hosts []string + trusted bool + ua string +} + +// SiteHosts returns the host of site and its www twin. +func SiteHosts(host string) []string { + host = strings.ToLower(strings.TrimSpace(host)) + if host == "" { + return nil + } + if bare, ok := strings.CutPrefix(host, "www."); ok { + return []string{host, bare} + } + return []string{host, "www." + host} +} + +// Trusted reports whether config turns trusted mode on. +func Trusted(cfg *compass.Config) bool { + return cfg != nil && cfg.Bool(TrustedConfigKey) +} + +// NewFetcher builds the fetcher of site (an absolute origin). +func NewFetcher(site string, cfg *compass.Config) (*Fetcher, error) { + u, err := url.Parse(strings.TrimRight(site, "/")) + if err != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") { + return nil, ErrNoSiteURL + } + f := &Fetcher{site: u, hosts: SiteHosts(u.Hostname()), trusted: Trusted(cfg), ua: UserAgent(u.Scheme + "://" + u.Host)} + policy := fetchguard.Policy{Mode: fetchguard.AllowHostsMode, AllowHosts: f.hosts, MaxBytes: MaxBody, Timeout: FetchTimeout} + if f.trusted { + policy.Mode = fetchguard.TrustedMode + } + f.client, err = fetchguard.NewClient(policy, cfg) + if err != nil { + return nil, err + } + return f, nil +} + +// Hosts returns the site host and its www twin. +func (f *Fetcher) Hosts() []string { return f.hosts } + +// IsTrusted reports whether the fetcher runs in trusted mode. +func (f *Fetcher) IsTrusted() bool { return f.trusted } + +// Site returns the site origin. +func (f *Fetcher) Site() *url.URL { return f.site } + +// UA returns the crawler's User-Agent. +func (f *Fetcher) UA() string { return f.ua } + +// Allowed reports whether u may be fetched: https (http only in trusted +// mode) on the site host or its www twin, without credentials. In trusted +// mode the host list is checked here, since fetchguard checks none then. +func (f *Fetcher) Allowed(u *url.URL) bool { + if u == nil || u.User != nil { + return false + } + switch strings.ToLower(u.Scheme) { + case "https": + case "http": + if !f.trusted { + return false + } + default: + return false + } + host := strings.ToLower(u.Hostname()) + for _, h := range f.hosts { + if host == h { + return true + } + } + return false +} + +// Key is the address of u on the site: its path with the query, or "" +// when u is not on the site. +func (f *Fetcher) Key(u *url.URL) string { + if !f.Allowed(u) { + return "" + } + p := u.EscapedPath() + if p == "" { + p = "/" + } + if u.RawQuery != "" { + p += "?" + u.RawQuery + } + return p +} + +// Absolute returns the site URL of an address key. +func (f *Fetcher) Absolute(key string) string { + return f.site.Scheme + "://" + f.site.Host + key +} + +// Get fetches one address key of the site. +func (f *Fetcher) Get(ctx context.Context, key string) *Response { + header := http.Header{} + header.Set("User-Agent", f.ua) + header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.5") + res, err := f.client.Get(ctx, f.Absolute(key), header) + if err != nil { + reason := "network_error" + if fe, ok := err.(*fetchguard.Error); ok { + reason = string(fe.Reason) + } + return &Response{Err: reason} + } + out := &Response{ + Status: res.StatusCode, + ContentType: res.Header.Get("Content-Type"), + XRobots: strings.Join(res.Header.Values("X-Robots-Tag"), ", "), + Body: res.Body, + } + if loc := res.Header.Get("Location"); loc != "" { + base, _ := url.Parse(f.Absolute(key)) + if ref, err := url.Parse(loc); err == nil && base != nil { + out.Location = base.ResolveReference(ref).String() + } + } + return out +} diff --git a/classes/crawl/fix.go b/classes/crawl/fix.go new file mode 100644 index 0000000..0574c38 --- /dev/null +++ b/classes/crawl/fix.go @@ -0,0 +1,145 @@ +package crawl + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + + "git.golem15.com/golem15/sm-seo-plugin/models" + "git.golem15.com/golem15/summercms/modules/pact" +) + +// The SEO tools an issue may name (pier prefixes them with seo_). +const ( + ToolSetURLSEO = "seo_set_url_seo" + ToolUpdateRobots = "seo_update_robots" + ToolCreateRedirect = "seo_create_redirect" + ToolUpdatePage = "seo_update_page" + ToolCreatePage = "seo_create_page" + ToolUpdateSettings = "seo_update_settings" +) + +// Record is the CMS record behind a crawled page: its content type, id +// and locale, the descriptor facts the checks compare the page with +// (title, word count, headings, images, the highest heading level) and the +// fixing tools its type names (pact.ContentDescriptor.Tools). +type Record struct { + Type string + ID string + Locale string + Title string + Words *int + Headings []pact.ContentHeading + Images []pact.ContentImage + TopHeading int + // Links are the record's CMS-known links: the address of its primary + // parent, which the page's breadcrumb trail links to. + Links []string + Tools map[string]string + // CustomJSONLD is true when the record's SEO tab stores custom JSON-LD. + CustomJSONLD bool +} + +// tool returns the record's tool for the first operation of ops it +// names, "" when it names none. +func (r *Record) tool(ops ...string) string { + if r == nil { + return "" + } + for _, op := range ops { + if t := strings.TrimSpace(r.Tools[op]); t != "" { + return t + } + } + return "" +} + +// hasHeading reports whether the record's own headings hold one of level +// (any level when level is 0) whose text is text (any text when empty). +func (r *Record) hasHeading(level int, text string) bool { + if r == nil { + return false + } + for _, h := range r.Headings { + if level != 0 && h.Level != level { + continue + } + if text == "" || normal(h.Text) == normal(text) { + return true + } + } + return false +} + +// Finding is one problem a check found on a page, before it is stored: +// the kind, the address and locale, the details of the kind, and how it +// is fixed (P33-38). Tool is a fixed CMS tool (seo_*); Content lists the +// record operations (pact.ContentTool*) that fix a finding inside the +// record's own content, tried in order; What names the template element +// when it is fixed in code. +type Finding struct { + Kind string + URL string + Locale string + Details map[string]any + Tool string + Content []string + Record *Record + What string + // Check tells the two content_js findings apart (cms, rendered). + Check string +} + +// Route applies P33-38: a finding inside the record's own content names +// the record type's fitting tool (H1 findings prefer Tools[headings], +// other content findings Tools[edit]); else a fixed CMS tool; else fix +// code with the template element in details.what. Every finding gets one +// of the two. +func Route(f Finding) (fix, tool, what string) { + if len(f.Content) > 0 { + if t := f.Record.tool(f.Content...); t != "" { + return models.FixTool, t, "" + } + } + if f.Tool != "" { + return models.FixTool, f.Tool, "" + } + what = f.What + if what == "" { + what = defaultWhat[f.Kind] + } + if what == "" { + what = "the page template" + } + return models.FixCode, "", what +} + +// defaultWhat names the template element of a kind fixed in code. +var defaultWhat = map[string]string{ + KindCanonicalMissing: ` in the page head`, + KindH1Missing: "the

of the page template", + KindH1Multiple: "the

headings of the page template", + KindHeadingSkipped: "the heading levels of the page template", + KindHeadingEmpty: "the empty heading of the page template", + KindJSONLDInvalid: `the