3fdb44bce2
Fetcher.Allowed and Key, the rendered comparison and the Chrome request guard accept an address only on the site's effective port (explicit, else 443 for https and 80 for http), so page code cannot make headless Chrome reach other services on the site host's address and a link to another port is not reported as a page of the site.
87 lines
2.4 KiB
Go
87 lines
2.4 KiB
Go
package crawl
|
|
|
|
import (
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// The thresholds of the rendered comparison (P33-55): text counts as
|
|
// added by JavaScript when the rendered page has at least growthWords more
|
|
// words and growthShare more than the raw HTML.
|
|
const (
|
|
growthWords = 50
|
|
growthShare = 0.2
|
|
maxJSOnlyURLs = 5
|
|
)
|
|
|
|
// compareRendered compares a page's raw HTML with its JavaScript-rendered
|
|
// DOM (D-70, P33-55; pure): same-site links, an H1 or a substantial amount
|
|
// of text that appear only after JavaScript are one content_js finding
|
|
// with details.check rendered and details.what naming them. key is the
|
|
// crawl's address of a link (Fetcher.Key: "" off the site, another port
|
|
// included). An identical rendering finds nothing.
|
|
func compareRendered(raw, rendered *PageFacts, key func(*url.URL) string) []Finding {
|
|
if raw == nil || rendered == nil {
|
|
return nil
|
|
}
|
|
same := func(u *url.URL) (string, bool) {
|
|
if u == nil || key == nil {
|
|
return "", false
|
|
}
|
|
k := key(u)
|
|
return k, k != ""
|
|
}
|
|
before := map[string]bool{}
|
|
for _, l := range raw.Links {
|
|
if k, ok := same(l.URL); ok {
|
|
before[k] = true
|
|
}
|
|
}
|
|
var added []string
|
|
seen := map[string]bool{}
|
|
for _, l := range rendered.Links {
|
|
k, ok := same(l.URL)
|
|
if !ok || before[k] || seen[k] {
|
|
continue
|
|
}
|
|
seen[k] = true
|
|
added = append(added, k)
|
|
}
|
|
details := map[string]any{}
|
|
var what []string
|
|
if len(added) > 0 {
|
|
shown := added
|
|
if len(shown) > maxJSOnlyURLs {
|
|
shown = shown[:maxJSOnlyURLs]
|
|
}
|
|
details["links"] = shown
|
|
what = append(what, "links "+strings.Join(shown, ", "))
|
|
}
|
|
if len(raw.H1()) == 0 {
|
|
if h1 := rendered.H1(); len(h1) > 0 {
|
|
details["h1"] = h1[0]
|
|
what = append(what, "H1 “"+h1[0]+"”")
|
|
}
|
|
}
|
|
if grow := rendered.Words - raw.Words; grow >= growthWords && float64(grow) >= growthShare*float64(raw.Words) {
|
|
details["words_raw"] = raw.Words
|
|
details["words_rendered"] = rendered.Words
|
|
what = append(what, strconv.Itoa(grow)+" words of text")
|
|
}
|
|
if len(what) == 0 {
|
|
return nil
|
|
}
|
|
return []Finding{{Kind: KindContentJS, Check: checkRendered, Details: details, What: strings.Join(what, "; ")}}
|
|
}
|
|
|
|
// The two content_js checks: the raw HTML against the CMS record, and the
|
|
// raw HTML against the JavaScript-rendered DOM.
|
|
const (
|
|
checkCMS = "cms"
|
|
checkRendered = "rendered"
|
|
)
|
|
|
|
// parseURL parses an absolute or site-relative address.
|
|
func parseURL(raw string) (*url.URL, error) { return url.Parse(strings.TrimSpace(raw)) }
|