sm-seo-plugin/classes/crawl/render.go

package crawl

import (
	"context"
	"fmt"
	"net/netip"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"git.golem15.com/golem15/summercms/modules/compass"
)

// Renderer renders a page of the site in a browser for the raw versus
// rendered comparison (D-70, P33-55): Render answers the page's DOM after
// JavaScript ran, Close releases the browser. A crawl without a Renderer
// records the comparison as skipped; a Render error records it as failed
// with the error and never fails the crawl.
type Renderer interface {
	Render(ctx context.Context, pageURL string) (string, error)
	Close() error
}

// DefaultRenderMax is the number of pages rendered per crawl when
// golem15.seo.crawl_render_max is unset.
const DefaultRenderMax = 200

// The render states of a crawl (models.RenderSkipped and the others).
const (
	renderStatusSkipped = "skipped"
	renderStatusDone    = "done"
	renderStatusFailed  = "failed"
)

type rendererKey struct{}

// WithRenderer returns a context whose crawls use r instead of the
// renderer built from config. It is the test seam of the comparison,
// settable only from Go code (like fetchguard.WithTransport): no config
// key, setting, request or MCP call reaches it.
func WithRenderer(ctx context.Context, r Renderer) context.Context {
	return context.WithValue(ctx, rendererKey{}, r)
}

// rendererFrom returns the renderer of ctx, nil when none.
func rendererFrom(ctx context.Context) Renderer {
	if ctx == nil {
		return nil
	}
	r, _ := ctx.Value(rendererKey{}).(Renderer)
	return r
}

// The config keys of the rendered comparison (P33-55). The executable
// path is Go config only (environment-overridable like every key), never
// an admin setting or MCP value: an editable path would let any holder of
// golem15.seo.manage make the server run an arbitrary binary.
const (
	// ChromePathConfigKey names an operator-installed Chrome or Chromium
	// binary; empty skips the comparison.
	ChromePathConfigKey = "golem15.seo.crawl_chrome_path"
	// RenderMaxConfigKey caps the pages rendered per crawl.
	RenderMaxConfigKey = "golem15.seo.crawl_render_max"
)

// The limits of one rendered page.
const (
	RenderTimeout = 20 * time.Second
	MaxDOM        = 5 << 20
)

// RenderMax is the configured page cap of the comparison (default 200).
func RenderMax(cfg *compass.Config) int {
	if cfg != nil {
		if n := cfg.Int(RenderMaxConfigKey); n > 0 {
			return n
		}
	}
	return DefaultRenderMax
}

// NewRenderer builds the renderer of a crawl of site from config: nil
// without error when golem15.seo.crawl_chrome_path is empty (the
// comparison is skipped); an error, recorded as the comparison's failure
// note, when the path is not absolute, not an executable regular file, the
// site host resolves to a private address outside trusted mode, or Chrome
// does not start. ua is the crawler's User-Agent.
func NewRenderer(ctx context.Context, cfg *compass.Config, site *url.URL, trusted bool, ua string) (Renderer, error) {
	if cfg == nil || site == nil {
		return nil, nil
	}
	path := strings.TrimSpace(cfg.String(ChromePathConfigKey))
	if path == "" {
		return nil, nil
	}
	if err := checkExecutable(path); err != nil {
		return nil, err
	}
	hosts, err := resolveHosts(ctx, SiteHosts(site.Hostname()), trusted)
	if err != nil {
		return nil, err
	}
	r, err := newChromeRenderer(path, hosts, trusted, ua)
	if err != nil {
		return nil, err
	}
	return r, nil
}

// checkExecutable refuses a Chrome path that is not absolute or not an
// executable regular file.
func checkExecutable(path string) error {
	if !filepath.IsAbs(path) {
		return fmt.Errorf("the Chrome path %q is not absolute", path)
	}
	info, err := os.Stat(path)
	if err != nil {
		return fmt.Errorf("the Chrome path %q cannot be read", path)
	}
	if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 {
		return fmt.Errorf("the Chrome path %q is not an executable file", path)
	}
	return nil
}

// resolverRules is Chrome's --host-resolver-rules value: one MAP rule per
// allowed host to one of its checked addresses (IPv4 first), then
// MAP * ~NOTFOUND for every other host. Chrome applies the first rule
// that matches a host, so the catch-all comes last.
func resolverRules(hosts map[string][]netip.Addr) string {
	names := make([]string, 0, len(hosts))
	for h := range hosts {
		names = append(names, h)
	}
	sort.Strings(names)
	var rules []string
	for _, h := range names {
		addrs := hosts[h]
		if len(addrs) == 0 {
			continue
		}
		// One address per host: the first IPv4 one, else the first.
		a := addrs[0]
		for _, c := range addrs {
			if c.Is4() {
				a = c
				break
			}
		}
		target := a.String()
		if a.Is6() {
			target = "[" + target + "]"
		}
		rules = append(rules, "MAP "+h+" "+target)
	}
	rules = append(rules, "MAP * ~NOTFOUND")
	return strings.Join(rules, ", ")
}

// allowRequest reports whether the headless browser may load rawURL: https
// (http only in trusted mode) to one of hosts, or a data: or blob: URL.
// Every other request, third-party scripts included, is failed.
func allowRequest(rawURL string, hosts []string, trusted bool) bool {
	u, err := url.Parse(rawURL)
	if err != nil {
		return false
	}
	switch strings.ToLower(u.Scheme) {
	case "data", "blob":
		return true
	case "https":
	case "http":
		if !trusted {
			return false
		}
	default:
		return false
	}
	if u.User != nil {
		return false
	}
	host := strings.ToLower(u.Hostname())
	for _, h := range hosts {
		if host == strings.ToLower(strings.Trim(h, "[]")) {
			return true
		}
	}
	return false
}