sm-seo-plugin/classes/crawl/render_chrome.go

package crawl

import (
	"context"
	"fmt"
	"net"
	"net/netip"
	"os"
	"sort"
	"strings"
	"sync"

	"git.golem15.com/golem15/summercms/modules/fetchguard"
	"github.com/chromedp/cdproto/browser"
	"github.com/chromedp/cdproto/cdp"
	"github.com/chromedp/cdproto/fetch"
	"github.com/chromedp/cdproto/network"
	"github.com/chromedp/chromedp"
)

// chromeRenderer renders pages in an operator-installed headless Chrome
// or Chromium through github.com/chromedp/chromedp (D-70, P33-55), held to
// the crawler's fetchguard limits:
//
//   - Chrome resolves only the site host and its www twin, pinned through
//     --host-resolver-rules to the addresses resolveHosts checked with
//     fetchguard.IsPrivateAddr; every other host resolves to NOTFOUND;
//   - every request is paused through the Fetch domain and continued only
//     when allowRequest passes (https, http only in trusted mode, to an
//     allowed host, or data: and blob:); the others fail with
//     BlockedByClient, so third-party scripts are never loaded;
//   - a fresh temporary profile per crawl, removed by Close; no cookies,
//     credentials or extensions; downloads denied;
//   - one tab at a time (the crawler serialises Render), 20 s per page, the
//     DOM capped at 5 MiB; Close stops Chrome.
//
// chromedp's default allocator flags are used as they are: nothing is
// added that weakens Chrome's sandbox.
type chromeRenderer struct {
	allowed []string
	trusted bool
	profile string

	allocCancel   context.CancelFunc
	browserCtx    context.Context
	browserCancel context.CancelFunc

	mu      sync.Mutex
	blocked []string
	closed  bool
}

// newChromeRenderer starts Chrome at path for the hosts and their checked
// addresses.
func newChromeRenderer(path string, hosts map[string][]netip.Addr, trusted bool, ua string) (*chromeRenderer, error) {
	profile, err := os.MkdirTemp("", "summercms-seo-chrome-")
	if err != nil {
		return nil, fmt.Errorf("the Chrome profile cannot be created: %w", err)
	}
	r := &chromeRenderer{trusted: trusted, profile: profile}
	for h := range hosts {
		r.allowed = append(r.allowed, h)
	}
	sort.Strings(r.allowed)
	opts := append(chromedp.DefaultExecAllocatorOptions[:],
		chromedp.ExecPath(path),
		chromedp.UserDataDir(profile),
		chromedp.Flag("host-resolver-rules", resolverRules(hosts)),
		chromedp.UserAgent(ua),
	)
	allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...)
	r.allocCancel = allocCancel
	r.browserCtx, r.browserCancel = chromedp.NewContext(allocCtx)
	// The first Run starts the browser; it runs without a timeout, since a
	// timeout on this context would stop the whole browser.
	if err := chromedp.Do(r.browserCtx); err != nil {
		_ = r.Close()
		return nil, fmt.Errorf("Chrome did not start: %w", err)
	}
	if _, err := chromedp.CallBrowser(r.browserCtx, browser.SetDownloadBehavior, browser.SetDownloadBehaviorParams{Behavior: browser.SetDownloadBehaviorBehaviorDeny}); err != nil {
		_ = r.Close()
		return nil, fmt.Errorf("Chrome downloads cannot be denied: %w", err)
	}
	return r, nil
}

// Render loads pageURL in a new tab and answers the DOM after JavaScript
// ran (the outer HTML of the document, capped at 5 MiB).
func (r *chromeRenderer) Render(ctx context.Context, pageURL string) (string, error) {
	r.mu.Lock()
	closed := r.closed
	r.mu.Unlock()
	if closed {
		return "", fmt.Errorf("Chrome is closed")
	}
	if !allowRequest(pageURL, r.allowed, r.trusted) {
		return "", fmt.Errorf("the address %s is outside the site", pageURL)
	}
	tab, closeTab := chromedp.NewContext(r.browserCtx)
	defer closeTab()
	// The tab ends with the crawl's context (a cancelled crawl) or after
	// the page's timeout.
	stop := context.AfterFunc(ctx, closeTab)
	defer stop()
	if err := chromedp.Do(tab); err != nil {
		return "", fmt.Errorf("Chrome cannot open a tab: %w", err)
	}
	page, cancel := context.WithTimeout(tab, RenderTimeout)
	defer cancel()
	target := chromedp.FromContext(tab).Target
	paused := chromedp.Events(page, fetch.RequestPaused)
	if _, err := chromedp.Call(page, fetch.Enable, fetch.EnableParams{Patterns: []*fetch.RequestPattern{{URLPattern: "*"}}}); err != nil {
		return "", fmt.Errorf("Chrome cannot intercept requests: %w", err)
	}
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for ev, err := range paused {
			if err != nil {
				return
			}
			r.decide(page, target, ev)
		}
	}()
	html, err := chromedp.Run(page, func(ctx context.Context, t *chromedp.Target) (string, error) {
		if _, err := chromedp.Navigate(pageURL)(ctx, t); err != nil {
			return "", err
		}
		if _, err := chromedp.WaitReady(chromedp.CSS("body"))(ctx, t); err != nil {
			return "", err
		}
		return chromedp.OuterHTML(chromedp.CSS("html"))(ctx, t)
	})
	cancel()
	closeTab()
	wg.Wait()
	if err != nil {
		if ctx.Err() != nil {
			return "", ctx.Err()
		}
		return "", fmt.Errorf("Chrome could not render %s: %w", pageURL, err)
	}
	if len(html) > MaxDOM {
		html = truncateUTF8(html, MaxDOM)
	}
	return html, nil
}

// decide continues a paused request that allowRequest passes and fails
// every other one with BlockedByClient, keeping its address.
func (r *chromeRenderer) decide(ctx context.Context, target cdp.Session, ev fetch.EventRequestPaused) {
	u := ""
	if ev.Request != nil {
		u = ev.Request.URL
	}
	if allowRequest(u, r.allowed, r.trusted) {
		_, _ = cdp.Call(ctx, target, fetch.ContinueRequest, fetch.ContinueRequestParams{RequestID: ev.RequestID})
		return
	}
	r.mu.Lock()
	r.blocked = append(r.blocked, u)
	r.mu.Unlock()
	_, _ = cdp.Call(ctx, target, fetch.FailRequest, fetch.FailRequestParams{RequestID: ev.RequestID, ErrorReason: network.ErrorReasonBlockedByClient})
}

// Blocked lists the addresses the browser was kept from loading.
func (r *chromeRenderer) Blocked() []string {
	r.mu.Lock()
	defer r.mu.Unlock()
	return append([]string(nil), r.blocked...)
}

// ProfileDir is the temporary profile of this crawl's browser.
func (r *chromeRenderer) ProfileDir() string { return r.profile }

// Close stops Chrome and removes the temporary profile.
func (r *chromeRenderer) Close() error {
	r.mu.Lock()
	if r.closed {
		r.mu.Unlock()
		return nil
	}
	r.closed = true
	r.mu.Unlock()
	if r.browserCancel != nil {
		r.browserCancel()
	}
	if r.allocCancel != nil {
		r.allocCancel()
	}
	return os.RemoveAll(r.profile)
}

// resolveHosts resolves the site host and its www twin. A private or
// reserved address (fetchguard.IsPrivateAddr) refuses outside trusted
// mode; a twin that does not resolve is left out.
func resolveHosts(ctx context.Context, hosts []string, trusted bool) (map[string][]netip.Addr, error) {
	out := map[string][]netip.Addr{}
	for i, host := range hosts {
		if addr, err := netip.ParseAddr(strings.Trim(host, "[]")); err == nil {
			if !trusted && fetchguard.IsPrivateAddr(addr) {
				return nil, fmt.Errorf("the site host %s is a private address", addr)
			}
			out[host] = []netip.Addr{addr.Unmap()}
			continue
		}
		addrs, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host)
		if err != nil || len(addrs) == 0 {
			if i == 0 {
				return nil, fmt.Errorf("the site host %s does not resolve", host)
			}
			continue
		}
		for _, a := range addrs {
			a = a.Unmap()
			if !trusted && fetchguard.IsPrivateAddr(a) {
				return nil, fmt.Errorf("the site host %s resolves to the private address %s", host, a)
			}
			out[host] = append(out[host], a)
		}
	}
	return out, nil
}