8c632331cc
- jsonld_invalid, jsonld_multiple, and jsonld_required/recommended from schema.Validate (the E9 rules); publisher nodes route to settings - hreflang_error and hreflang_reciprocal over the fetched alternates - broken_link and redirected_link with the linking pages, generic_anchor against the en and pl seed lists - image_alt, image_size and featured_lazy - content_js check cms: the record title in the H1, half its words and its primary parent's link in the raw HTML - compareRendered (pure): links, an H1 or text only after JavaScript as content_js check rendered; the Renderer seam and render bookkeeping - TestRenderSkipped: skipped without a renderer, failed with its note
98 lines
2.5 KiB
Go
98 lines
2.5 KiB
Go
package crawl
|
|
|
|
import (
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// The thresholds of the rendered comparison (P33-55): text counts as
|
|
// added by JavaScript when the rendered page has at least growthWords more
|
|
// words and growthShare more than the raw HTML.
|
|
const (
|
|
growthWords = 50
|
|
growthShare = 0.2
|
|
maxJSOnlyURLs = 5
|
|
)
|
|
|
|
// compareRendered compares a page's raw HTML with its JavaScript-rendered
|
|
// DOM (D-70, P33-55; pure): same-host links, an H1 or a substantial amount
|
|
// of text that appear only after JavaScript are one content_js finding
|
|
// with details.check rendered and details.what naming them. hosts are the
|
|
// site host and its www twin. An identical rendering finds nothing.
|
|
func compareRendered(raw, rendered *PageFacts, hosts []string) []Finding {
|
|
if raw == nil || rendered == nil {
|
|
return nil
|
|
}
|
|
same := func(u *url.URL) (string, bool) {
|
|
if u == nil {
|
|
return "", false
|
|
}
|
|
host := strings.ToLower(u.Hostname())
|
|
for _, h := range hosts {
|
|
if host == h {
|
|
p := u.EscapedPath()
|
|
if p == "" {
|
|
p = "/"
|
|
}
|
|
if u.RawQuery != "" {
|
|
p += "?" + u.RawQuery
|
|
}
|
|
return p, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
before := map[string]bool{}
|
|
for _, l := range raw.Links {
|
|
if k, ok := same(l.URL); ok {
|
|
before[k] = true
|
|
}
|
|
}
|
|
var added []string
|
|
seen := map[string]bool{}
|
|
for _, l := range rendered.Links {
|
|
k, ok := same(l.URL)
|
|
if !ok || before[k] || seen[k] {
|
|
continue
|
|
}
|
|
seen[k] = true
|
|
added = append(added, k)
|
|
}
|
|
details := map[string]any{}
|
|
var what []string
|
|
if len(added) > 0 {
|
|
shown := added
|
|
if len(shown) > maxJSOnlyURLs {
|
|
shown = shown[:maxJSOnlyURLs]
|
|
}
|
|
details["links"] = shown
|
|
what = append(what, "links "+strings.Join(shown, ", "))
|
|
}
|
|
if len(raw.H1()) == 0 {
|
|
if h1 := rendered.H1(); len(h1) > 0 {
|
|
details["h1"] = h1[0]
|
|
what = append(what, "H1 “"+h1[0]+"”")
|
|
}
|
|
}
|
|
if grow := rendered.Words - raw.Words; grow >= growthWords && float64(grow) >= growthShare*float64(raw.Words) {
|
|
details["words_raw"] = raw.Words
|
|
details["words_rendered"] = rendered.Words
|
|
what = append(what, strconv.Itoa(grow)+" words of text")
|
|
}
|
|
if len(what) == 0 {
|
|
return nil
|
|
}
|
|
return []Finding{{Kind: KindContentJS, Check: checkRendered, Details: details, What: strings.Join(what, "; ")}}
|
|
}
|
|
|
|
// The two content_js checks: the raw HTML against the CMS record, and the
|
|
// raw HTML against the JavaScript-rendered DOM.
|
|
const (
|
|
checkCMS = "cms"
|
|
checkRendered = "rendered"
|
|
)
|
|
|
|
// parseURL parses an absolute or site-relative address.
|
|
func parseURL(raw string) (*url.URL, error) { return url.Parse(strings.TrimSpace(raw)) }
|