cfeb3dff12
Render bounds opening the tab and rendering the page with one deadline (TabOpenTimeout 5 s plus RenderTimeout 20 s), so a hung Chrome no longer holds the crawl's render lock until the job timeout.
277 lines
9.0 KiB
Go
277 lines
9.0 KiB
Go
package crawl
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"net/netip"
|
|
"os"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/fetchguard"
|
|
"github.com/chromedp/cdproto/browser"
|
|
"github.com/chromedp/cdproto/cdp"
|
|
"github.com/chromedp/cdproto/fetch"
|
|
"github.com/chromedp/cdproto/network"
|
|
"github.com/chromedp/chromedp"
|
|
)
|
|
|
|
// chromeRenderer renders pages in an operator-installed headless Chrome
|
|
// or Chromium through github.com/chromedp/chromedp (D-70, P33-55), held to
|
|
// the crawler's fetchguard limits:
|
|
//
|
|
// - Chrome resolves only the site host and its www twin, pinned through
|
|
// --host-resolver-rules to the addresses resolveHosts checked with
|
|
// fetchguard.IsPrivateAddr; every other host resolves to NOTFOUND;
|
|
// - every request is paused through the Fetch domain and continued only
|
|
// when allowRequest passes (https, http only in trusted mode, to an
|
|
// allowed host, or data: and blob:); the others fail with
|
|
// BlockedByClient, so third-party scripts are never loaded;
|
|
// - a fresh temporary profile per crawl, removed by Close; no cookies,
|
|
// credentials or extensions; downloads denied;
|
|
// - one tab at a time (the crawler serialises Render), at most
|
|
// TabOpenTimeout (5 s) to open the tab and RenderTimeout (20 s) for the
|
|
// page, both under one deadline, so a hung Chrome never holds the
|
|
// crawl's render lock longer; the DOM capped at 5 MiB; Close stops
|
|
// Chrome.
|
|
//
|
|
// chromedp's default allocator flags are used as they are: nothing is
|
|
// added that weakens Chrome's sandbox.
|
|
type chromeRenderer struct {
|
|
allowed []string
|
|
trusted bool
|
|
profile string
|
|
|
|
allocCancel context.CancelFunc
|
|
browserCtx context.Context
|
|
browserCancel context.CancelFunc
|
|
|
|
// newTab and startTab open a tab: chromedp.NewContext and chromedp.Do
|
|
// unless a test replaces them.
|
|
newTab func(parent context.Context) (context.Context, context.CancelFunc)
|
|
startTab func(tab context.Context) error
|
|
|
|
mu sync.Mutex
|
|
blocked []string
|
|
closed bool
|
|
}
|
|
|
|
// TabOpenTimeout bounds opening a Chrome tab (P33-60, L3 WR-06); Render
|
|
// bounds opening the tab and rendering the page with one deadline of
|
|
// TabOpenTimeout plus RenderTimeout.
|
|
var TabOpenTimeout = 5 * time.Second
|
|
|
|
// seconds writes d as whole seconds ("5 s") when it is one, else as
|
|
// time.Duration does.
|
|
func seconds(d time.Duration) string {
|
|
if d%time.Second == 0 {
|
|
return strconv.Itoa(int(d/time.Second)) + " s"
|
|
}
|
|
return d.String()
|
|
}
|
|
|
|
// newChromeRenderer starts Chrome at path for the hosts and their checked
|
|
// addresses.
|
|
func newChromeRenderer(path string, hosts map[string][]netip.Addr, trusted bool, ua string) (*chromeRenderer, error) {
|
|
profile, err := os.MkdirTemp("", "summercms-seo-chrome-")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the Chrome profile cannot be created: %w", err)
|
|
}
|
|
r := &chromeRenderer{trusted: trusted, profile: profile}
|
|
for h := range hosts {
|
|
r.allowed = append(r.allowed, h)
|
|
}
|
|
sort.Strings(r.allowed)
|
|
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
|
chromedp.ExecPath(path),
|
|
chromedp.UserDataDir(profile),
|
|
chromedp.Flag("host-resolver-rules", resolverRules(hosts)),
|
|
chromedp.UserAgent(ua),
|
|
)
|
|
allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...)
|
|
r.allocCancel = allocCancel
|
|
r.browserCtx, r.browserCancel = chromedp.NewContext(allocCtx)
|
|
// The first Run starts the browser; it runs without a timeout, since a
|
|
// timeout on this context would stop the whole browser.
|
|
if err := chromedp.Do(r.browserCtx); err != nil {
|
|
_ = r.Close()
|
|
return nil, fmt.Errorf("Chrome did not start: %w", err)
|
|
}
|
|
if _, err := chromedp.CallBrowser(r.browserCtx, browser.SetDownloadBehavior, browser.SetDownloadBehaviorParams{Behavior: browser.SetDownloadBehaviorBehaviorDeny}); err != nil {
|
|
_ = r.Close()
|
|
return nil, fmt.Errorf("Chrome downloads cannot be denied: %w", err)
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
// Render loads pageURL in a new tab and answers the DOM after JavaScript
|
|
// ran (the outer HTML of the document, capped at 5 MiB).
|
|
func (r *chromeRenderer) Render(ctx context.Context, pageURL string) (string, error) {
|
|
r.mu.Lock()
|
|
closed := r.closed
|
|
r.mu.Unlock()
|
|
if closed {
|
|
return "", fmt.Errorf("Chrome is closed")
|
|
}
|
|
if !allowRequest(pageURL, r.allowed, r.trusted) {
|
|
return "", fmt.Errorf("the address %s is outside the site", pageURL)
|
|
}
|
|
// One deadline bounds opening the tab and rendering the page; the tab
|
|
// ends with it or with the crawl's context (a cancelled crawl).
|
|
rctx, rcancel := context.WithTimeout(ctx, TabOpenTimeout+RenderTimeout)
|
|
defer rcancel()
|
|
newTab, startTab := r.newTab, r.startTab
|
|
if newTab == nil {
|
|
newTab = func(parent context.Context) (context.Context, context.CancelFunc) { return chromedp.NewContext(parent) }
|
|
}
|
|
if startTab == nil {
|
|
startTab = func(tab context.Context) error { return chromedp.Do(tab) }
|
|
}
|
|
tab, closeTab := newTab(r.browserCtx)
|
|
defer closeTab()
|
|
stop := context.AfterFunc(rctx, closeTab)
|
|
defer stop()
|
|
opened := make(chan error, 1)
|
|
go func() { opened <- startTab(tab) }()
|
|
openTimer := time.NewTimer(TabOpenTimeout)
|
|
defer openTimer.Stop()
|
|
select {
|
|
case err := <-opened:
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return "", ctx.Err()
|
|
}
|
|
return "", fmt.Errorf("Chrome cannot open a tab: %w", err)
|
|
}
|
|
case <-openTimer.C:
|
|
closeTab()
|
|
return "", fmt.Errorf("Chrome did not open a tab within %s", seconds(TabOpenTimeout))
|
|
case <-rctx.Done():
|
|
closeTab()
|
|
if ctx.Err() != nil {
|
|
return "", ctx.Err()
|
|
}
|
|
return "", fmt.Errorf("Chrome did not open a tab within %s", seconds(TabOpenTimeout))
|
|
}
|
|
page, cancel := context.WithTimeout(tab, RenderTimeout)
|
|
defer cancel()
|
|
target := chromedp.FromContext(tab).Target
|
|
paused := chromedp.Events(page, fetch.RequestPaused)
|
|
if _, err := chromedp.Call(page, fetch.Enable, fetch.EnableParams{Patterns: []*fetch.RequestPattern{{URLPattern: "*"}}}); err != nil {
|
|
return "", fmt.Errorf("Chrome cannot intercept requests: %w", err)
|
|
}
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for ev, err := range paused {
|
|
if err != nil {
|
|
return
|
|
}
|
|
r.decide(page, target, ev)
|
|
}
|
|
}()
|
|
html, err := chromedp.Run(page, func(ctx context.Context, t *chromedp.Target) (string, error) {
|
|
if _, err := chromedp.Navigate(pageURL)(ctx, t); err != nil {
|
|
return "", err
|
|
}
|
|
if _, err := chromedp.WaitReady(chromedp.CSS("body"))(ctx, t); err != nil {
|
|
return "", err
|
|
}
|
|
return chromedp.OuterHTML(chromedp.CSS("html"))(ctx, t)
|
|
})
|
|
cancel()
|
|
closeTab()
|
|
wg.Wait()
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return "", ctx.Err()
|
|
}
|
|
return "", fmt.Errorf("Chrome could not render %s: %w", pageURL, err)
|
|
}
|
|
if len(html) > MaxDOM {
|
|
html = truncateUTF8(html, MaxDOM)
|
|
}
|
|
return html, nil
|
|
}
|
|
|
|
// decide continues a paused request that allowRequest passes and fails
|
|
// every other one with BlockedByClient, keeping its address.
|
|
func (r *chromeRenderer) decide(ctx context.Context, target cdp.Session, ev fetch.EventRequestPaused) {
|
|
u := ""
|
|
if ev.Request != nil {
|
|
u = ev.Request.URL
|
|
}
|
|
if allowRequest(u, r.allowed, r.trusted) {
|
|
_, _ = cdp.Call(ctx, target, fetch.ContinueRequest, fetch.ContinueRequestParams{RequestID: ev.RequestID})
|
|
return
|
|
}
|
|
r.mu.Lock()
|
|
r.blocked = append(r.blocked, u)
|
|
r.mu.Unlock()
|
|
_, _ = cdp.Call(ctx, target, fetch.FailRequest, fetch.FailRequestParams{RequestID: ev.RequestID, ErrorReason: network.ErrorReasonBlockedByClient})
|
|
}
|
|
|
|
// Blocked lists the addresses the browser was kept from loading.
|
|
func (r *chromeRenderer) Blocked() []string {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return append([]string(nil), r.blocked...)
|
|
}
|
|
|
|
// ProfileDir is the temporary profile of this crawl's browser.
|
|
func (r *chromeRenderer) ProfileDir() string { return r.profile }
|
|
|
|
// Close stops Chrome and removes the temporary profile.
|
|
func (r *chromeRenderer) Close() error {
|
|
r.mu.Lock()
|
|
if r.closed {
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
r.closed = true
|
|
r.mu.Unlock()
|
|
if r.browserCancel != nil {
|
|
r.browserCancel()
|
|
}
|
|
if r.allocCancel != nil {
|
|
r.allocCancel()
|
|
}
|
|
return os.RemoveAll(r.profile)
|
|
}
|
|
|
|
// resolveHosts resolves the site host and its www twin. A private or
|
|
// reserved address (fetchguard.IsPrivateAddr) refuses outside trusted
|
|
// mode; a twin that does not resolve is left out.
|
|
func resolveHosts(ctx context.Context, hosts []string, trusted bool) (map[string][]netip.Addr, error) {
|
|
out := map[string][]netip.Addr{}
|
|
for i, host := range hosts {
|
|
if addr, err := netip.ParseAddr(strings.Trim(host, "[]")); err == nil {
|
|
if !trusted && fetchguard.IsPrivateAddr(addr) {
|
|
return nil, fmt.Errorf("the site host %s is a private address", addr)
|
|
}
|
|
out[host] = []netip.Addr{addr.Unmap()}
|
|
continue
|
|
}
|
|
addrs, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host)
|
|
if err != nil || len(addrs) == 0 {
|
|
if i == 0 {
|
|
return nil, fmt.Errorf("the site host %s does not resolve", host)
|
|
}
|
|
continue
|
|
}
|
|
for _, a := range addrs {
|
|
a = a.Unmap()
|
|
if !trusted && fetchguard.IsPrivateAddr(a) {
|
|
return nil, fmt.Errorf("the site host %s resolves to the private address %s", host, a)
|
|
}
|
|
out[host] = append(out[host], a)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|