feat: crawler finds a page with two H1 headings and stores the issue (tracer)

- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37)
- classes/crawl: fetchguard client on the site host and its www twin,
  sitemap index and robots.txt seeds, two workers with a delay, limits
- h1_missing and h1_multiple with the P33-38 fix route
- POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues
  answers the stored issues with id, severity, source, fix, found_at,
  ignored and crawl_id

classes/crawl/fetch.go

@@ -0,0 +1,191 @@
+// Package crawl is the site audit's crawler (D-14): it fetches the
+// rendered site like a search engine, checks every page (UI-SPEC C23),
+// stores the findings with the CMS-data issues and names how each one is
+// fixed (P33-38).
+package crawl
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"git.golem15.com/golem15/summercms/modules/compass"
+	"git.golem15.com/golem15/summercms/modules/fetchguard"
+)
+
+// The config keys of the crawler.
+const (
+	// TrustedConfigKey lets the crawler fetch plain http and private
+	// addresses (staging, local sites). Only Go config can set it.
+	TrustedConfigKey = "golem15.seo.crawl_trusted"
+)
+
+// The crawler's limits (D-14, RESEARCH Pattern 7).
+const (
+	// MaxBody caps one answer's body.
+	MaxBody = 5 << 20
+	// FetchTimeout is one request's timeout.
+	FetchTimeout = 10 * time.Second
+	// Workers is the number of requests at a time.
+	Workers = 2
+	// MaxLinksPerPage caps the links stored and followed per page.
+	MaxLinksPerPage = 1000
+)
+
+// UserAgent is the crawler's identity (P33-41).
+func UserAgent(site string) string {
+	return "SummerCMS-SEO-Crawler/1 (+" + site + ")"
+}
+
+// Response is one fetched address: its status, Location, content type,
+// X-Robots-Tag and body, or the fetch error.
+type Response struct {
+	Status      int
+	Location    string
+	ContentType string
+	XRobots     string
+	Body        []byte
+	Err         string
+}
+
+// HTML reports whether the answer is an HTML page.
+func (r *Response) HTML() bool {
+	ct := strings.ToLower(r.ContentType)
+	return ct == "" || strings.Contains(ct, "text/html") || strings.Contains(ct, "application/xhtml")
+}
+
+// Fetcher fetches the site's own addresses through a fetchguard client
+// limited to the site host and its www twin: HTTPS only, private
+// addresses refused at dial time and redirects never followed. Trusted
+// mode (golem15.seo.crawl_trusted, Go config only) allows plain http and
+// private addresses, still only on those hosts.
+type Fetcher struct {
+	client  *fetchguard.Client
+	site    *url.URL
+	hosts   []string
+	trusted bool
+	ua      string
+}
+
+// SiteHosts returns the host of site and its www twin.
+func SiteHosts(host string) []string {
+	host = strings.ToLower(strings.TrimSpace(host))
+	if host == "" {
+		return nil
+	}
+	if bare, ok := strings.CutPrefix(host, "www."); ok {
+		return []string{host, bare}
+	}
+	return []string{host, "www." + host}
+}
+
+// Trusted reports whether config turns trusted mode on.
+func Trusted(cfg *compass.Config) bool {
+	return cfg != nil && cfg.Bool(TrustedConfigKey)
+}
+
+// NewFetcher builds the fetcher of site (an absolute origin).
+func NewFetcher(site string, cfg *compass.Config) (*Fetcher, error) {
+	u, err := url.Parse(strings.TrimRight(site, "/"))
+	if err != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") {
+		return nil, ErrNoSiteURL
+	}
+	f := &Fetcher{site: u, hosts: SiteHosts(u.Hostname()), trusted: Trusted(cfg), ua: UserAgent(u.Scheme + "://" + u.Host)}
+	policy := fetchguard.Policy{Mode: fetchguard.AllowHostsMode, AllowHosts: f.hosts, MaxBytes: MaxBody, Timeout: FetchTimeout}
+	if f.trusted {
+		policy.Mode = fetchguard.TrustedMode
+	}
+	f.client, err = fetchguard.NewClient(policy, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
+// Hosts returns the site host and its www twin.
+func (f *Fetcher) Hosts() []string { return f.hosts }
+
+// IsTrusted reports whether the fetcher runs in trusted mode.
+func (f *Fetcher) IsTrusted() bool { return f.trusted }
+
+// Site returns the site origin.
+func (f *Fetcher) Site() *url.URL { return f.site }
+
+// UA returns the crawler's User-Agent.
+func (f *Fetcher) UA() string { return f.ua }
+
+// Allowed reports whether u may be fetched: https (http only in trusted
+// mode) on the site host or its www twin, without credentials. In trusted
+// mode the host list is checked here, since fetchguard checks none then.
+func (f *Fetcher) Allowed(u *url.URL) bool {
+	if u == nil || u.User != nil {
+		return false
+	}
+	switch strings.ToLower(u.Scheme) {
+	case "https":
+	case "http":
+		if !f.trusted {
+			return false
+		}
+	default:
+		return false
+	}
+	host := strings.ToLower(u.Hostname())
+	for _, h := range f.hosts {
+		if host == h {
+			return true
+		}
+	}
+	return false
+}
+
+// Key is the address of u on the site: its path with the query, or ""
+// when u is not on the site.
+func (f *Fetcher) Key(u *url.URL) string {
+	if !f.Allowed(u) {
+		return ""
+	}
+	p := u.EscapedPath()
+	if p == "" {
+		p = "/"
+	}
+	if u.RawQuery != "" {
+		p += "?" + u.RawQuery
+	}
+	return p
+}
+
+// Absolute returns the site URL of an address key.
+func (f *Fetcher) Absolute(key string) string {
+	return f.site.Scheme + "://" + f.site.Host + key
+}
+
+// Get fetches one address key of the site.
+func (f *Fetcher) Get(ctx context.Context, key string) *Response {
+	header := http.Header{}
+	header.Set("User-Agent", f.ua)
+	header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.5")
+	res, err := f.client.Get(ctx, f.Absolute(key), header)
+	if err != nil {
+		reason := "network_error"
+		if fe, ok := err.(*fetchguard.Error); ok {
+			reason = string(fe.Reason)
+		}
+		return &Response{Err: reason}
+	}
+	out := &Response{
+		Status:      res.StatusCode,
+		ContentType: res.Header.Get("Content-Type"),
+		XRobots:     strings.Join(res.Header.Values("X-Robots-Tag"), ", "),
+		Body:        res.Body,
+	}
+	if loc := res.Header.Get("Location"); loc != "" {
+		base, _ := url.Parse(f.Absolute(key))
+		if ref, err := url.Parse(loc); err == nil && base != nil {
+			out.Location = base.ResolveReference(ref).String()
+		}
+	}
+	return out
+}