feat: rendered comparison through headless Chrome
- github.com/chromedp/chromedp v0.21.0 (decision note 2026-10-28-chromedp-rendered-crawl.md): with golem15.seo.crawl_chrome_path the crawl renders up to golem15.seo.crawl_render_max pages (200) and compares them with the raw HTML; without a path it records skipped - guard: site host and www twin resolved and checked with fetchguard.IsPrivateAddr, pinned through --host-resolver-rules with a final MAP * ~NOTFOUND; Fetch-domain interception fails every request outside the site (third-party scripts never load); fresh temporary profile removed on Close; downloads denied; one tab, 20 s, 5 MiB DOM - a relative or non-executable path, a private address or a Chrome that does not start records render_status failed; the crawl still finishes - config keys documented; README Rendered comparison section - TestChromeRenderer (runs with a local Chromium) and TestRendererGuards
This commit is contained in:
@@ -1,6 +1,18 @@
|
||||
package crawl
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/compass"
|
||||
)
|
||||
|
||||
// Renderer renders a page of the site in a browser for the raw versus
|
||||
// rendered comparison (D-70, P33-55): Render answers the page's DOM after
|
||||
@@ -41,3 +53,140 @@ func rendererFrom(ctx context.Context) Renderer {
|
||||
r, _ := ctx.Value(rendererKey{}).(Renderer)
|
||||
return r
|
||||
}
|
||||
|
||||
// The config keys of the rendered comparison (P33-55). The executable
|
||||
// path is Go config only (environment-overridable like every key), never
|
||||
// an admin setting or MCP value: an editable path would let any holder of
|
||||
// golem15.seo.manage make the server run an arbitrary binary.
|
||||
const (
|
||||
// ChromePathConfigKey names an operator-installed Chrome or Chromium
|
||||
// binary; empty skips the comparison.
|
||||
ChromePathConfigKey = "golem15.seo.crawl_chrome_path"
|
||||
// RenderMaxConfigKey caps the pages rendered per crawl.
|
||||
RenderMaxConfigKey = "golem15.seo.crawl_render_max"
|
||||
)
|
||||
|
||||
// The limits of one rendered page.
|
||||
const (
|
||||
RenderTimeout = 20 * time.Second
|
||||
MaxDOM = 5 << 20
|
||||
)
|
||||
|
||||
// RenderMax is the configured page cap of the comparison (default 200).
|
||||
func RenderMax(cfg *compass.Config) int {
|
||||
if cfg != nil {
|
||||
if n := cfg.Int(RenderMaxConfigKey); n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return DefaultRenderMax
|
||||
}
|
||||
|
||||
// NewRenderer builds the renderer of a crawl of site from config: nil
|
||||
// without error when golem15.seo.crawl_chrome_path is empty (the
|
||||
// comparison is skipped); an error, recorded as the comparison's failure
|
||||
// note, when the path is not absolute, not an executable regular file, the
|
||||
// site host resolves to a private address outside trusted mode, or Chrome
|
||||
// does not start. ua is the crawler's User-Agent.
|
||||
func NewRenderer(ctx context.Context, cfg *compass.Config, site *url.URL, trusted bool, ua string) (Renderer, error) {
|
||||
if cfg == nil || site == nil {
|
||||
return nil, nil
|
||||
}
|
||||
path := strings.TrimSpace(cfg.String(ChromePathConfigKey))
|
||||
if path == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if err := checkExecutable(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hosts, err := resolveHosts(ctx, SiteHosts(site.Hostname()), trusted)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := newChromeRenderer(path, hosts, trusted, ua)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// checkExecutable refuses a Chrome path that is not absolute or not an
|
||||
// executable regular file.
|
||||
func checkExecutable(path string) error {
|
||||
if !filepath.IsAbs(path) {
|
||||
return fmt.Errorf("the Chrome path %q is not absolute", path)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the Chrome path %q cannot be read", path)
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 {
|
||||
return fmt.Errorf("the Chrome path %q is not an executable file", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolverRules is Chrome's --host-resolver-rules value: one MAP rule per
|
||||
// allowed host to one of its checked addresses (IPv4 first), then
|
||||
// MAP * ~NOTFOUND for every other host. Chrome applies the first rule
|
||||
// that matches a host, so the catch-all comes last.
|
||||
func resolverRules(hosts map[string][]netip.Addr) string {
|
||||
names := make([]string, 0, len(hosts))
|
||||
for h := range hosts {
|
||||
names = append(names, h)
|
||||
}
|
||||
sort.Strings(names)
|
||||
var rules []string
|
||||
for _, h := range names {
|
||||
addrs := hosts[h]
|
||||
if len(addrs) == 0 {
|
||||
continue
|
||||
}
|
||||
// One address per host: the first IPv4 one, else the first.
|
||||
a := addrs[0]
|
||||
for _, c := range addrs {
|
||||
if c.Is4() {
|
||||
a = c
|
||||
break
|
||||
}
|
||||
}
|
||||
target := a.String()
|
||||
if a.Is6() {
|
||||
target = "[" + target + "]"
|
||||
}
|
||||
rules = append(rules, "MAP "+h+" "+target)
|
||||
}
|
||||
rules = append(rules, "MAP * ~NOTFOUND")
|
||||
return strings.Join(rules, ", ")
|
||||
}
|
||||
|
||||
// allowRequest reports whether the headless browser may load rawURL: https
|
||||
// (http only in trusted mode) to one of hosts, or a data: or blob: URL.
|
||||
// Every other request, third-party scripts included, is failed.
|
||||
func allowRequest(rawURL string, hosts []string, trusted bool) bool {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "data", "blob":
|
||||
return true
|
||||
case "https":
|
||||
case "http":
|
||||
if !trusted {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
if u.User != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
for _, h := range hosts {
|
||||
if host == strings.ToLower(strings.Trim(h, "[]")) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user