feat: rendered comparison through headless Chrome
- github.com/chromedp/chromedp v0.21.0 (decision note 2026-10-28-chromedp-rendered-crawl.md): with golem15.seo.crawl_chrome_path the crawl renders up to golem15.seo.crawl_render_max pages (200) and compares them with the raw HTML; without a path it records skipped - guard: site host and www twin resolved and checked with fetchguard.IsPrivateAddr, pinned through --host-resolver-rules with a final MAP * ~NOTFOUND; Fetch-domain interception fails every request outside the site (third-party scripts never load); fresh temporary profile removed on Close; downloads denied; one tab, 20 s, 5 MiB DOM - a relative or non-executable path, a private address or a Chrome that does not start records render_status failed; the crawl still finishes - config keys documented; README Rendered comparison section - TestChromeRenderer (runs with a local Chromium) and TestRendererGuards
This commit is contained in:
Jakub Zych
@@ -104,6 +104,9 @@ type Options struct {
|
||||
Renderer Renderer
|
||||
// RenderMax caps the rendered pages (default 200, crawl order).
|
||||
RenderMax int
|
||||
// RenderErr is why the renderer could not be built: the comparison is
|
||||
// recorded as failed with it, and the raw checks run.
|
||||
RenderErr error
|
||||
// Canceled is asked between requests; true stops the crawl and keeps
|
||||
// what was fetched.
|
||||
Canceled func(ctx context.Context) bool
|
||||
@@ -164,8 +167,11 @@ func Crawl(ctx context.Context, f *Fetcher, opt Options) (*Result, error) {
|
||||
opt.RenderMax = DefaultRenderMax
|
||||
}
|
||||
res := &Result{Pages: map[string]*Page{}, Listed: map[string]string{}, Render: RenderState{Status: renderStatusSkipped}, fetcher: f}
|
||||
if opt.Renderer != nil {
|
||||
switch {
|
||||
case opt.Renderer != nil:
|
||||
res.Render.Status = renderStatusDone
|
||||
case opt.RenderErr != nil:
|
||||
res.Render = RenderState{Status: renderStatusFailed, Note: opt.RenderErr.Error()}
|
||||
}
|
||||
pace := &pacer{delay: opt.Delay}
|
||||
canceled := func() bool {
|
||||
|
||||
@@ -196,10 +196,22 @@ func run(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.Craw
|
||||
if app := svc.App(); app != nil {
|
||||
m, _ = conga.From(app)
|
||||
}
|
||||
// The rendered comparison: the test seam's renderer, else Chrome from
|
||||
// golem15.seo.crawl_chrome_path (none: skipped). Chrome is closed when
|
||||
// the crawl finishes, fails or is cancelled.
|
||||
renderer, renderErr := rendererFrom(ctx), error(nil)
|
||||
if renderer == nil {
|
||||
renderer, renderErr = NewRenderer(ctx, svc.Config(), f.Site(), f.IsTrusted(), f.UA())
|
||||
if renderer != nil {
|
||||
defer func() { _ = renderer.Close() }()
|
||||
}
|
||||
}
|
||||
opt := Options{
|
||||
MaxPages: limits.MaxPages,
|
||||
Delay: limits.Delay,
|
||||
Renderer: rendererFrom(ctx),
|
||||
MaxPages: limits.MaxPages,
|
||||
Delay: limits.Delay,
|
||||
Renderer: renderer,
|
||||
RenderErr: renderErr,
|
||||
RenderMax: RenderMax(svc.Config()),
|
||||
Canceled: func(ctx context.Context) bool {
|
||||
var status string
|
||||
if err := db.Raw(`SELECT status FROM golem15_seo_crawls WHERE id = ?`, row.ID).Scan(&status).Error; err == nil && status == models.CrawlCancelled {
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
package crawl
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/compass"
|
||||
)
|
||||
|
||||
// Renderer renders a page of the site in a browser for the raw versus
|
||||
// rendered comparison (D-70, P33-55): Render answers the page's DOM after
|
||||
@@ -41,3 +53,140 @@ func rendererFrom(ctx context.Context) Renderer {
|
||||
r, _ := ctx.Value(rendererKey{}).(Renderer)
|
||||
return r
|
||||
}
|
||||
|
||||
// The config keys of the rendered comparison (P33-55). The executable
|
||||
// path is Go config only (environment-overridable like every key), never
|
||||
// an admin setting or MCP value: an editable path would let any holder of
|
||||
// golem15.seo.manage make the server run an arbitrary binary.
|
||||
const (
|
||||
// ChromePathConfigKey names an operator-installed Chrome or Chromium
|
||||
// binary; empty skips the comparison.
|
||||
ChromePathConfigKey = "golem15.seo.crawl_chrome_path"
|
||||
// RenderMaxConfigKey caps the pages rendered per crawl.
|
||||
RenderMaxConfigKey = "golem15.seo.crawl_render_max"
|
||||
)
|
||||
|
||||
// The limits of one rendered page.
|
||||
const (
|
||||
RenderTimeout = 20 * time.Second
|
||||
MaxDOM = 5 << 20
|
||||
)
|
||||
|
||||
// RenderMax is the configured page cap of the comparison (default 200).
|
||||
func RenderMax(cfg *compass.Config) int {
|
||||
if cfg != nil {
|
||||
if n := cfg.Int(RenderMaxConfigKey); n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return DefaultRenderMax
|
||||
}
|
||||
|
||||
// NewRenderer builds the renderer of a crawl of site from config: nil
|
||||
// without error when golem15.seo.crawl_chrome_path is empty (the
|
||||
// comparison is skipped); an error, recorded as the comparison's failure
|
||||
// note, when the path is not absolute, not an executable regular file, the
|
||||
// site host resolves to a private address outside trusted mode, or Chrome
|
||||
// does not start. ua is the crawler's User-Agent.
|
||||
func NewRenderer(ctx context.Context, cfg *compass.Config, site *url.URL, trusted bool, ua string) (Renderer, error) {
|
||||
if cfg == nil || site == nil {
|
||||
return nil, nil
|
||||
}
|
||||
path := strings.TrimSpace(cfg.String(ChromePathConfigKey))
|
||||
if path == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if err := checkExecutable(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hosts, err := resolveHosts(ctx, SiteHosts(site.Hostname()), trusted)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := newChromeRenderer(path, hosts, trusted, ua)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// checkExecutable refuses a Chrome path that is not absolute or not an
|
||||
// executable regular file.
|
||||
func checkExecutable(path string) error {
|
||||
if !filepath.IsAbs(path) {
|
||||
return fmt.Errorf("the Chrome path %q is not absolute", path)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the Chrome path %q cannot be read", path)
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 {
|
||||
return fmt.Errorf("the Chrome path %q is not an executable file", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolverRules is Chrome's --host-resolver-rules value: one MAP rule per
|
||||
// allowed host to one of its checked addresses (IPv4 first), then
|
||||
// MAP * ~NOTFOUND for every other host. Chrome applies the first rule
|
||||
// that matches a host, so the catch-all comes last.
|
||||
func resolverRules(hosts map[string][]netip.Addr) string {
|
||||
names := make([]string, 0, len(hosts))
|
||||
for h := range hosts {
|
||||
names = append(names, h)
|
||||
}
|
||||
sort.Strings(names)
|
||||
var rules []string
|
||||
for _, h := range names {
|
||||
addrs := hosts[h]
|
||||
if len(addrs) == 0 {
|
||||
continue
|
||||
}
|
||||
// One address per host: the first IPv4 one, else the first.
|
||||
a := addrs[0]
|
||||
for _, c := range addrs {
|
||||
if c.Is4() {
|
||||
a = c
|
||||
break
|
||||
}
|
||||
}
|
||||
target := a.String()
|
||||
if a.Is6() {
|
||||
target = "[" + target + "]"
|
||||
}
|
||||
rules = append(rules, "MAP "+h+" "+target)
|
||||
}
|
||||
rules = append(rules, "MAP * ~NOTFOUND")
|
||||
return strings.Join(rules, ", ")
|
||||
}
|
||||
|
||||
// allowRequest reports whether the headless browser may load rawURL: https
|
||||
// (http only in trusted mode) to one of hosts, or a data: or blob: URL.
|
||||
// Every other request, third-party scripts included, is failed.
|
||||
func allowRequest(rawURL string, hosts []string, trusted bool) bool {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "data", "blob":
|
||||
return true
|
||||
case "https":
|
||||
case "http":
|
||||
if !trusted {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
if u.User != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
for _, h := range hosts {
|
||||
if host == strings.ToLower(strings.Trim(h, "[]")) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
224
classes/crawl/render_chrome.go
Normal file
224
classes/crawl/render_chrome.go
Normal file
@@ -0,0 +1,224 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/fetchguard"
|
||||
"github.com/chromedp/cdproto/browser"
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/cdproto/fetch"
|
||||
"github.com/chromedp/cdproto/network"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
// chromeRenderer renders pages in an operator-installed headless Chrome
|
||||
// or Chromium through github.com/chromedp/chromedp (D-70, P33-55), held to
|
||||
// the crawler's fetchguard limits:
|
||||
//
|
||||
// - Chrome resolves only the site host and its www twin, pinned through
|
||||
// --host-resolver-rules to the addresses resolveHosts checked with
|
||||
// fetchguard.IsPrivateAddr; every other host resolves to NOTFOUND;
|
||||
// - every request is paused through the Fetch domain and continued only
|
||||
// when allowRequest passes (https, http only in trusted mode, to an
|
||||
// allowed host, or data: and blob:); the others fail with
|
||||
// BlockedByClient, so third-party scripts are never loaded;
|
||||
// - a fresh temporary profile per crawl, removed by Close; no cookies,
|
||||
// credentials or extensions; downloads denied;
|
||||
// - one tab at a time (the crawler serialises Render), 20 s per page, the
|
||||
// DOM capped at 5 MiB; Close stops Chrome.
|
||||
//
|
||||
// chromedp's default allocator flags are used as they are: nothing is
|
||||
// added that weakens Chrome's sandbox.
|
||||
type chromeRenderer struct {
|
||||
allowed []string
|
||||
trusted bool
|
||||
profile string
|
||||
|
||||
allocCancel context.CancelFunc
|
||||
browserCtx context.Context
|
||||
browserCancel context.CancelFunc
|
||||
|
||||
mu sync.Mutex
|
||||
blocked []string
|
||||
closed bool
|
||||
}
|
||||
|
||||
// newChromeRenderer starts Chrome at path for the hosts and their checked
|
||||
// addresses.
|
||||
func newChromeRenderer(path string, hosts map[string][]netip.Addr, trusted bool, ua string) (*chromeRenderer, error) {
|
||||
profile, err := os.MkdirTemp("", "summercms-seo-chrome-")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the Chrome profile cannot be created: %w", err)
|
||||
}
|
||||
r := &chromeRenderer{trusted: trusted, profile: profile}
|
||||
for h := range hosts {
|
||||
r.allowed = append(r.allowed, h)
|
||||
}
|
||||
sort.Strings(r.allowed)
|
||||
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
||||
chromedp.ExecPath(path),
|
||||
chromedp.UserDataDir(profile),
|
||||
chromedp.Flag("host-resolver-rules", resolverRules(hosts)),
|
||||
chromedp.UserAgent(ua),
|
||||
)
|
||||
allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...)
|
||||
r.allocCancel = allocCancel
|
||||
r.browserCtx, r.browserCancel = chromedp.NewContext(allocCtx)
|
||||
// The first Run starts the browser; it runs without a timeout, since a
|
||||
// timeout on this context would stop the whole browser.
|
||||
if err := chromedp.Do(r.browserCtx); err != nil {
|
||||
_ = r.Close()
|
||||
return nil, fmt.Errorf("Chrome did not start: %w", err)
|
||||
}
|
||||
if _, err := chromedp.CallBrowser(r.browserCtx, browser.SetDownloadBehavior, browser.SetDownloadBehaviorParams{Behavior: browser.SetDownloadBehaviorBehaviorDeny}); err != nil {
|
||||
_ = r.Close()
|
||||
return nil, fmt.Errorf("Chrome downloads cannot be denied: %w", err)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Render loads pageURL in a new tab and answers the DOM after JavaScript
|
||||
// ran (the outer HTML of the document, capped at 5 MiB).
|
||||
func (r *chromeRenderer) Render(ctx context.Context, pageURL string) (string, error) {
|
||||
r.mu.Lock()
|
||||
closed := r.closed
|
||||
r.mu.Unlock()
|
||||
if closed {
|
||||
return "", fmt.Errorf("Chrome is closed")
|
||||
}
|
||||
if !allowRequest(pageURL, r.allowed, r.trusted) {
|
||||
return "", fmt.Errorf("the address %s is outside the site", pageURL)
|
||||
}
|
||||
tab, closeTab := chromedp.NewContext(r.browserCtx)
|
||||
defer closeTab()
|
||||
// The tab ends with the crawl's context (a cancelled crawl) or after
|
||||
// the page's timeout.
|
||||
stop := context.AfterFunc(ctx, closeTab)
|
||||
defer stop()
|
||||
if err := chromedp.Do(tab); err != nil {
|
||||
return "", fmt.Errorf("Chrome cannot open a tab: %w", err)
|
||||
}
|
||||
page, cancel := context.WithTimeout(tab, RenderTimeout)
|
||||
defer cancel()
|
||||
target := chromedp.FromContext(tab).Target
|
||||
paused := chromedp.Events(page, fetch.RequestPaused)
|
||||
if _, err := chromedp.Call(page, fetch.Enable, fetch.EnableParams{Patterns: []*fetch.RequestPattern{{URLPattern: "*"}}}); err != nil {
|
||||
return "", fmt.Errorf("Chrome cannot intercept requests: %w", err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for ev, err := range paused {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
r.decide(page, target, ev)
|
||||
}
|
||||
}()
|
||||
html, err := chromedp.Run(page, func(ctx context.Context, t *chromedp.Target) (string, error) {
|
||||
if _, err := chromedp.Navigate(pageURL)(ctx, t); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := chromedp.WaitReady(chromedp.CSS("body"))(ctx, t); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return chromedp.OuterHTML(chromedp.CSS("html"))(ctx, t)
|
||||
})
|
||||
cancel()
|
||||
closeTab()
|
||||
wg.Wait()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return "", ctx.Err()
|
||||
}
|
||||
return "", fmt.Errorf("Chrome could not render %s: %w", pageURL, err)
|
||||
}
|
||||
if len(html) > MaxDOM {
|
||||
html = truncateUTF8(html, MaxDOM)
|
||||
}
|
||||
return html, nil
|
||||
}
|
||||
|
||||
// decide continues a paused request that allowRequest passes and fails
|
||||
// every other one with BlockedByClient, keeping its address.
|
||||
func (r *chromeRenderer) decide(ctx context.Context, target cdp.Session, ev fetch.EventRequestPaused) {
|
||||
u := ""
|
||||
if ev.Request != nil {
|
||||
u = ev.Request.URL
|
||||
}
|
||||
if allowRequest(u, r.allowed, r.trusted) {
|
||||
_, _ = cdp.Call(ctx, target, fetch.ContinueRequest, fetch.ContinueRequestParams{RequestID: ev.RequestID})
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.blocked = append(r.blocked, u)
|
||||
r.mu.Unlock()
|
||||
_, _ = cdp.Call(ctx, target, fetch.FailRequest, fetch.FailRequestParams{RequestID: ev.RequestID, ErrorReason: network.ErrorReasonBlockedByClient})
|
||||
}
|
||||
|
||||
// Blocked lists the addresses the browser was kept from loading.
|
||||
func (r *chromeRenderer) Blocked() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return append([]string(nil), r.blocked...)
|
||||
}
|
||||
|
||||
// ProfileDir is the temporary profile of this crawl's browser.
|
||||
func (r *chromeRenderer) ProfileDir() string { return r.profile }
|
||||
|
||||
// Close stops Chrome and removes the temporary profile.
|
||||
func (r *chromeRenderer) Close() error {
|
||||
r.mu.Lock()
|
||||
if r.closed {
|
||||
r.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
r.closed = true
|
||||
r.mu.Unlock()
|
||||
if r.browserCancel != nil {
|
||||
r.browserCancel()
|
||||
}
|
||||
if r.allocCancel != nil {
|
||||
r.allocCancel()
|
||||
}
|
||||
return os.RemoveAll(r.profile)
|
||||
}
|
||||
|
||||
// resolveHosts resolves the site host and its www twin. A private or
|
||||
// reserved address (fetchguard.IsPrivateAddr) refuses outside trusted
|
||||
// mode; a twin that does not resolve is left out.
|
||||
func resolveHosts(ctx context.Context, hosts []string, trusted bool) (map[string][]netip.Addr, error) {
|
||||
out := map[string][]netip.Addr{}
|
||||
for i, host := range hosts {
|
||||
if addr, err := netip.ParseAddr(strings.Trim(host, "[]")); err == nil {
|
||||
if !trusted && fetchguard.IsPrivateAddr(addr) {
|
||||
return nil, fmt.Errorf("the site host %s is a private address", addr)
|
||||
}
|
||||
out[host] = []netip.Addr{addr.Unmap()}
|
||||
continue
|
||||
}
|
||||
addrs, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host)
|
||||
if err != nil || len(addrs) == 0 {
|
||||
if i == 0 {
|
||||
return nil, fmt.Errorf("the site host %s does not resolve", host)
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, a := range addrs {
|
||||
a = a.Unmap()
|
||||
if !trusted && fetchguard.IsPrivateAddr(a) {
|
||||
return nil, fmt.Errorf("the site host %s resolves to the private address %s", host, a)
|
||||
}
|
||||
out[host] = append(out[host], a)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user