sm-seo-plugin/classes/crawl/fetch.go

// Package crawl is the site audit's crawler (D-14): it fetches the
// rendered site like a search engine, checks every page (UI-SPEC C23),
// stores the findings with the CMS-data issues and names how each one is
// fixed (P33-38).
package crawl

import (
	"context"
	"net/http"
	"net/url"
	"strings"
	"time"

	"git.golem15.com/golem15/summercms/modules/compass"
	"git.golem15.com/golem15/summercms/modules/fetchguard"
)

// The config keys of the crawler.
const (
	// TrustedConfigKey lets the crawler fetch plain http and private
	// addresses (staging, local sites). Only Go config can set it.
	TrustedConfigKey = "golem15.seo.crawl_trusted"
)

// The crawler's limits (D-14, RESEARCH Pattern 7).
const (
	// MaxBody caps one answer's body.
	MaxBody = 5 << 20
	// FetchTimeout is one request's timeout.
	FetchTimeout = 10 * time.Second
	// Workers is the number of requests at a time.
	Workers = 2
	// MaxLinksPerPage caps the links stored and followed per page.
	MaxLinksPerPage = 1000
)

// UserAgent is the crawler's identity (P33-41).
func UserAgent(site string) string {
	return "SummerCMS-SEO-Crawler/1 (+" + site + ")"
}

// Response is one fetched address: its status, Location, content type,
// X-Robots-Tag and body, or the fetch error.
type Response struct {
	Status      int
	Location    string
	ContentType string
	XRobots     string
	Body        []byte
	Err         string
}

// HTML reports whether the answer is an HTML page.
func (r *Response) HTML() bool {
	ct := strings.ToLower(r.ContentType)
	return ct == "" || strings.Contains(ct, "text/html") || strings.Contains(ct, "application/xhtml")
}

// Fetcher fetches the site's own addresses through a fetchguard client
// limited to the site host and its www twin: HTTPS only, private
// addresses refused at dial time and redirects never followed. Trusted
// mode (golem15.seo.crawl_trusted, Go config only) allows plain http and
// private addresses, still only on those hosts.
type Fetcher struct {
	client  *fetchguard.Client
	site    *url.URL
	hosts   []string
	trusted bool
	ua      string
}

// SiteHosts returns the host of site and its www twin.
func SiteHosts(host string) []string {
	host = strings.ToLower(strings.TrimSpace(host))
	if host == "" {
		return nil
	}
	if bare, ok := strings.CutPrefix(host, "www."); ok {
		return []string{host, bare}
	}
	return []string{host, "www." + host}
}

// Trusted reports whether config turns trusted mode on.
func Trusted(cfg *compass.Config) bool {
	return cfg != nil && cfg.Bool(TrustedConfigKey)
}

// NewFetcher builds the fetcher of site (an absolute origin).
func NewFetcher(site string, cfg *compass.Config) (*Fetcher, error) {
	u, err := url.Parse(strings.TrimRight(site, "/"))
	if err != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") {
		return nil, ErrNoSiteURL
	}
	f := &Fetcher{site: u, hosts: SiteHosts(u.Hostname()), trusted: Trusted(cfg), ua: UserAgent(u.Scheme + "://" + u.Host)}
	policy := fetchguard.Policy{Mode: fetchguard.AllowHostsMode, AllowHosts: f.hosts, MaxBytes: MaxBody, Timeout: FetchTimeout}
	if f.trusted {
		policy.Mode = fetchguard.TrustedMode
	}
	f.client, err = fetchguard.NewClient(policy, cfg)
	if err != nil {
		return nil, err
	}
	return f, nil
}

// Hosts returns the site host and its www twin.
func (f *Fetcher) Hosts() []string { return f.hosts }

// IsTrusted reports whether the fetcher runs in trusted mode.
func (f *Fetcher) IsTrusted() bool { return f.trusted }

// Site returns the site origin.
func (f *Fetcher) Site() *url.URL { return f.site }

// UA returns the crawler's User-Agent.
func (f *Fetcher) UA() string { return f.ua }

// Allowed reports whether u may be fetched: https (http only in trusted
// mode) on the site host or its www twin, without credentials. In trusted
// mode the host list is checked here, since fetchguard checks none then.
func (f *Fetcher) Allowed(u *url.URL) bool {
	if u == nil || u.User != nil {
		return false
	}
	switch strings.ToLower(u.Scheme) {
	case "https":
	case "http":
		if !f.trusted {
			return false
		}
	default:
		return false
	}
	host := strings.ToLower(u.Hostname())
	for _, h := range f.hosts {
		if host == h {
			return true
		}
	}
	return false
}

// Key is the address of u on the site: its path with the query, or ""
// when u is not on the site.
func (f *Fetcher) Key(u *url.URL) string {
	if !f.Allowed(u) {
		return ""
	}
	p := u.EscapedPath()
	if p == "" {
		p = "/"
	}
	if u.RawQuery != "" {
		p += "?" + u.RawQuery
	}
	return p
}

// Absolute returns the site URL of an address key.
func (f *Fetcher) Absolute(key string) string {
	return f.site.Scheme + "://" + f.site.Host + key
}

// Get fetches one address key of the site.
func (f *Fetcher) Get(ctx context.Context, key string) *Response {
	header := http.Header{}
	header.Set("User-Agent", f.ua)
	header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.5")
	res, err := f.client.Get(ctx, f.Absolute(key), header)
	if err != nil {
		reason := "network_error"
		if fe, ok := err.(*fetchguard.Error); ok {
			reason = string(fe.Reason)
		}
		return &Response{Err: reason}
	}
	out := &Response{
		Status:      res.StatusCode,
		ContentType: res.Header.Get("Content-Type"),
		XRobots:     strings.Join(res.Header.Values("X-Robots-Tag"), ", "),
		Body:        res.Body,
	}
	if loc := res.Header.Get("Location"); loc != "" {
		base, _ := url.Parse(f.Absolute(key))
		if ref, err := url.Parse(loc); err == nil && base != nil {
			out.Location = base.ResolveReference(ref).String()
		}
	}
	return out
}