package crawl import ( "context" "fmt" "net/netip" "net/url" "os" "path/filepath" "sort" "strings" "time" "git.golem15.com/golem15/summercms/modules/compass" ) // Renderer renders a page of the site in a browser for the raw versus // rendered comparison (D-70, P33-55): Render answers the page's DOM after // JavaScript ran, Close releases the browser. A crawl without a Renderer // records the comparison as skipped; a Render error records it as failed // with the error and never fails the crawl. type Renderer interface { Render(ctx context.Context, pageURL string) (string, error) Close() error } // DefaultRenderMax is the number of pages rendered per crawl when // golem15.seo.crawl_render_max is unset. const DefaultRenderMax = 200 // The render states of a crawl (models.RenderSkipped and the others). const ( renderStatusSkipped = "skipped" renderStatusDone = "done" renderStatusFailed = "failed" ) type rendererKey struct{} // WithRenderer returns a context whose crawls use r instead of the // renderer built from config. It is the test seam of the comparison, // settable only from Go code (like fetchguard.WithTransport): no config // key, setting, request or MCP call reaches it. func WithRenderer(ctx context.Context, r Renderer) context.Context { return context.WithValue(ctx, rendererKey{}, r) } // rendererFrom returns the renderer of ctx, nil when none. func rendererFrom(ctx context.Context) Renderer { if ctx == nil { return nil } r, _ := ctx.Value(rendererKey{}).(Renderer) return r } // The config keys of the rendered comparison (P33-55). The executable // path is Go config only (environment-overridable like every key), never // an admin setting or MCP value: an editable path would let any holder of // golem15.seo.manage make the server run an arbitrary binary. const ( // ChromePathConfigKey names an operator-installed Chrome or Chromium // binary; empty skips the comparison. ChromePathConfigKey = "golem15.seo.crawl_chrome_path" // RenderMaxConfigKey caps the pages rendered per crawl. RenderMaxConfigKey = "golem15.seo.crawl_render_max" ) // The limits of one rendered page. const ( RenderTimeout = 20 * time.Second MaxDOM = 5 << 20 ) // RenderMax is the configured page cap of the comparison (default 200). func RenderMax(cfg *compass.Config) int { if cfg != nil { if n := cfg.Int(RenderMaxConfigKey); n > 0 { return n } } return DefaultRenderMax } // NewRenderer builds the renderer of a crawl of site from config: nil // without error when golem15.seo.crawl_chrome_path is empty (the // comparison is skipped); an error, recorded as the comparison's failure // note, when the path is not absolute, not an executable regular file, the // site host resolves to a private address outside trusted mode, or Chrome // does not start. ua is the crawler's User-Agent. func NewRenderer(ctx context.Context, cfg *compass.Config, site *url.URL, trusted bool, ua string) (Renderer, error) { if cfg == nil || site == nil { return nil, nil } path := strings.TrimSpace(cfg.String(ChromePathConfigKey)) if path == "" { return nil, nil } if err := checkExecutable(path); err != nil { return nil, err } hosts, err := resolveHosts(ctx, SiteHosts(site.Hostname()), trusted) if err != nil { return nil, err } r, err := newChromeRenderer(path, hosts, effectivePort(site), trusted, ua) if err != nil { return nil, err } return r, nil } // checkExecutable refuses a Chrome path that is not absolute or not an // executable regular file. func checkExecutable(path string) error { if !filepath.IsAbs(path) { return fmt.Errorf("the Chrome path %q is not absolute", path) } info, err := os.Stat(path) if err != nil { return fmt.Errorf("the Chrome path %q cannot be read", path) } if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { return fmt.Errorf("the Chrome path %q is not an executable file", path) } return nil } // resolverRules is Chrome's --host-resolver-rules value: one MAP rule per // allowed host to one of its checked addresses (IPv4 first), then // MAP * ~NOTFOUND for every other host. Chrome applies the first rule // that matches a host, so the catch-all comes last. func resolverRules(hosts map[string][]netip.Addr) string { names := make([]string, 0, len(hosts)) for h := range hosts { names = append(names, h) } sort.Strings(names) var rules []string for _, h := range names { addrs := hosts[h] if len(addrs) == 0 { continue } // One address per host: the first IPv4 one, else the first. a := addrs[0] for _, c := range addrs { if c.Is4() { a = c break } } target := a.String() if a.Is6() { target = "[" + target + "]" } rules = append(rules, "MAP "+h+" "+target) } rules = append(rules, "MAP * ~NOTFOUND") return strings.Join(rules, ", ") } // allowRequest reports whether the headless browser may load rawURL: https // (http only in trusted mode) to one of hosts on the site's effective port // (port; the request's explicit port, else 443 for https and 80 for http), // or a data: or blob: URL. Every other request, third-party scripts and // other services on the site host's address included, is failed. func allowRequest(rawURL string, hosts []string, port string, trusted bool) bool { u, err := url.Parse(rawURL) if err != nil { return false } switch strings.ToLower(u.Scheme) { case "data", "blob": return true case "https": case "http": if !trusted { return false } default: return false } if u.User != nil || effectivePort(u) != port { return false } host := strings.ToLower(u.Hostname()) for _, h := range hosts { if host == strings.ToLower(strings.Trim(h, "[]")) { return true } } return false }