sm-seo-plugin/classes/crawl/render_chrome.go

package crawl

import (
	"context"
	"fmt"
	"net"
	"net/netip"
	"os"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"git.golem15.com/golem15/summercms/modules/fetchguard"
	"github.com/chromedp/cdproto/browser"
	"github.com/chromedp/cdproto/cdp"
	"github.com/chromedp/cdproto/fetch"
	"github.com/chromedp/cdproto/network"
	"github.com/chromedp/chromedp"
)

// chromeRenderer renders pages in an operator-installed headless Chrome
// or Chromium through github.com/chromedp/chromedp (D-70, P33-55), held to
// the crawler's fetchguard limits:
//
//   - Chrome resolves only the site host and its www twin, pinned through
//     --host-resolver-rules to the addresses resolveHosts checked with
//     fetchguard.IsPrivateAddr; every other host resolves to NOTFOUND;
//   - every request is paused through the Fetch domain and continued only
//     when allowRequest passes (https, http only in trusted mode, to an
//     allowed host on the site's port, or data: and blob:); the others fail with
//     BlockedByClient, so third-party scripts are never loaded;
//   - a fresh temporary profile per crawl, removed by Close; no cookies,
//     credentials or extensions; downloads denied;
//   - one tab at a time (the crawler serialises Render), at most
//     TabOpenTimeout (5 s) to open the tab and RenderTimeout (20 s) for the
//     page, both under one deadline, so a hung Chrome never holds the
//     crawl's render lock longer; the DOM capped at 5 MiB; Close stops
//     Chrome.
//
// chromedp's default allocator flags are used as they are: nothing is
// added that weakens Chrome's sandbox.
type chromeRenderer struct {
	allowed []string
	port    string
	trusted bool
	profile string

	allocCancel   context.CancelFunc
	browserCtx    context.Context
	browserCancel context.CancelFunc

	// newTab and startTab open a tab: chromedp.NewContext and chromedp.Do
	// unless a test replaces them.
	newTab   func(parent context.Context) (context.Context, context.CancelFunc)
	startTab func(tab context.Context) error

	mu      sync.Mutex
	blocked []string
	closed  bool
}

// TabOpenTimeout bounds opening a Chrome tab (P33-60, L3 WR-06); Render
// bounds opening the tab and rendering the page with one deadline of
// TabOpenTimeout plus RenderTimeout.
var TabOpenTimeout = 5 * time.Second

// seconds writes d as whole seconds ("5 s") when it is one, else as
// time.Duration does.
func seconds(d time.Duration) string {
	if d%time.Second == 0 {
		return strconv.Itoa(int(d/time.Second)) + " s"
	}
	return d.String()
}

// newChromeRenderer starts Chrome at path for the hosts and their checked
// addresses, on the site's effective port.
func newChromeRenderer(path string, hosts map[string][]netip.Addr, port string, trusted bool, ua string) (*chromeRenderer, error) {
	profile, err := os.MkdirTemp("", "summercms-seo-chrome-")
	if err != nil {
		return nil, fmt.Errorf("the Chrome profile cannot be created: %w", err)
	}
	r := &chromeRenderer{port: port, trusted: trusted, profile: profile}
	for h := range hosts {
		r.allowed = append(r.allowed, h)
	}
	sort.Strings(r.allowed)
	opts := append(chromedp.DefaultExecAllocatorOptions[:],
		chromedp.ExecPath(path),
		chromedp.UserDataDir(profile),
		chromedp.Flag("host-resolver-rules", resolverRules(hosts)),
		chromedp.UserAgent(ua),
	)
	allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...)
	r.allocCancel = allocCancel
	r.browserCtx, r.browserCancel = chromedp.NewContext(allocCtx)
	// The first Run starts the browser; it runs without a timeout, since a
	// timeout on this context would stop the whole browser.
	if err := chromedp.Do(r.browserCtx); err != nil {
		_ = r.Close()
		return nil, fmt.Errorf("Chrome did not start: %w", err)
	}
	if _, err := chromedp.CallBrowser(r.browserCtx, browser.SetDownloadBehavior, browser.SetDownloadBehaviorParams{Behavior: browser.SetDownloadBehaviorBehaviorDeny}); err != nil {
		_ = r.Close()
		return nil, fmt.Errorf("Chrome downloads cannot be denied: %w", err)
	}
	return r, nil
}

// Render loads pageURL in a new tab and answers the DOM after JavaScript
// ran (the outer HTML of the document, capped at 5 MiB).
func (r *chromeRenderer) Render(ctx context.Context, pageURL string) (string, error) {
	r.mu.Lock()
	closed := r.closed
	r.mu.Unlock()
	if closed {
		return "", fmt.Errorf("Chrome is closed")
	}
	if !allowRequest(pageURL, r.allowed, r.port, r.trusted) {
		return "", fmt.Errorf("the address %s is outside the site", pageURL)
	}
	// One deadline bounds opening the tab and rendering the page; the tab
	// ends with it or with the crawl's context (a cancelled crawl).
	rctx, rcancel := context.WithTimeout(ctx, TabOpenTimeout+RenderTimeout)
	defer rcancel()
	newTab, startTab := r.newTab, r.startTab
	if newTab == nil {
		newTab = func(parent context.Context) (context.Context, context.CancelFunc) { return chromedp.NewContext(parent) }
	}
	if startTab == nil {
		startTab = func(tab context.Context) error { return chromedp.Do(tab) }
	}
	tab, closeTab := newTab(r.browserCtx)
	defer closeTab()
	stop := context.AfterFunc(rctx, closeTab)
	defer stop()
	opened := make(chan error, 1)
	go func() { opened <- startTab(tab) }()
	openTimer := time.NewTimer(TabOpenTimeout)
	defer openTimer.Stop()
	select {
	case err := <-opened:
		if err != nil {
			if ctx.Err() != nil {
				return "", ctx.Err()
			}
			return "", fmt.Errorf("Chrome cannot open a tab: %w", err)
		}
	case <-openTimer.C:
		closeTab()
		return "", fmt.Errorf("Chrome did not open a tab within %s", seconds(TabOpenTimeout))
	case <-rctx.Done():
		closeTab()
		if ctx.Err() != nil {
			return "", ctx.Err()
		}
		return "", fmt.Errorf("Chrome did not open a tab within %s", seconds(TabOpenTimeout))
	}
	page, cancel := context.WithTimeout(tab, RenderTimeout)
	defer cancel()
	target := chromedp.FromContext(tab).Target
	paused := chromedp.Events(page, fetch.RequestPaused)
	if _, err := chromedp.Call(page, fetch.Enable, fetch.EnableParams{Patterns: []*fetch.RequestPattern{{URLPattern: "*"}}}); err != nil {
		return "", fmt.Errorf("Chrome cannot intercept requests: %w", err)
	}
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for ev, err := range paused {
			if err != nil {
				return
			}
			r.decide(page, target, ev)
		}
	}()
	html, err := chromedp.Run(page, func(ctx context.Context, t *chromedp.Target) (string, error) {
		if _, err := chromedp.Navigate(pageURL)(ctx, t); err != nil {
			return "", err
		}
		if _, err := chromedp.WaitReady(chromedp.CSS("body"))(ctx, t); err != nil {
			return "", err
		}
		return chromedp.OuterHTML(chromedp.CSS("html"))(ctx, t)
	})
	cancel()
	closeTab()
	wg.Wait()
	if err != nil {
		if ctx.Err() != nil {
			return "", ctx.Err()
		}
		return "", fmt.Errorf("Chrome could not render %s: %w", pageURL, err)
	}
	if len(html) > MaxDOM {
		html = truncateUTF8(html, MaxDOM)
	}
	return html, nil
}

// decide continues a paused request that allowRequest passes and fails
// every other one with BlockedByClient, keeping its address.
func (r *chromeRenderer) decide(ctx context.Context, target cdp.Session, ev fetch.EventRequestPaused) {
	u := ""
	if ev.Request != nil {
		u = ev.Request.URL
	}
	if allowRequest(u, r.allowed, r.port, r.trusted) {
		_, _ = cdp.Call(ctx, target, fetch.ContinueRequest, fetch.ContinueRequestParams{RequestID: ev.RequestID})
		return
	}
	r.mu.Lock()
	r.blocked = append(r.blocked, u)
	r.mu.Unlock()
	_, _ = cdp.Call(ctx, target, fetch.FailRequest, fetch.FailRequestParams{RequestID: ev.RequestID, ErrorReason: network.ErrorReasonBlockedByClient})
}

// Blocked lists the addresses the browser was kept from loading.
func (r *chromeRenderer) Blocked() []string {
	r.mu.Lock()
	defer r.mu.Unlock()
	return append([]string(nil), r.blocked...)
}

// ProfileDir is the temporary profile of this crawl's browser.
func (r *chromeRenderer) ProfileDir() string { return r.profile }

// Close stops Chrome and removes the temporary profile.
func (r *chromeRenderer) Close() error {
	r.mu.Lock()
	if r.closed {
		r.mu.Unlock()
		return nil
	}
	r.closed = true
	r.mu.Unlock()
	if r.browserCancel != nil {
		r.browserCancel()
	}
	if r.allocCancel != nil {
		r.allocCancel()
	}
	return os.RemoveAll(r.profile)
}

// resolveHosts resolves the site host and its www twin. A private or
// reserved address (fetchguard.IsPrivateAddr) refuses outside trusted
// mode; a twin that does not resolve is left out.
func resolveHosts(ctx context.Context, hosts []string, trusted bool) (map[string][]netip.Addr, error) {
	out := map[string][]netip.Addr{}
	for i, host := range hosts {
		if addr, err := netip.ParseAddr(strings.Trim(host, "[]")); err == nil {
			if !trusted && fetchguard.IsPrivateAddr(addr) {
				return nil, fmt.Errorf("the site host %s is a private address", addr)
			}
			out[host] = []netip.Addr{addr.Unmap()}
			continue
		}
		addrs, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host)
		if err != nil || len(addrs) == 0 {
			if i == 0 {
				return nil, fmt.Errorf("the site host %s does not resolve", host)
			}
			continue
		}
		for _, a := range addrs {
			a = a.Unmap()
			if !trusted && fetchguard.IsPrivateAddr(a) {
				return nil, fmt.Errorf("the site host %s resolves to the private address %s", host, a)
			}
			out[host] = append(out[host], a)
		}
	}
	return out, nil
}