8c632331cc
- jsonld_invalid, jsonld_multiple, and jsonld_required/recommended from schema.Validate (the E9 rules); publisher nodes route to settings - hreflang_error and hreflang_reciprocal over the fetched alternates - broken_link and redirected_link with the linking pages, generic_anchor against the en and pl seed lists - image_alt, image_size and featured_lazy - content_js check cms: the record title in the H1, half its words and its primary parent's link in the raw HTML - compareRendered (pure): links, an H1 or text only after JavaScript as content_js check rendered; the Renderer seam and render bookkeeping - TestRenderSkipped: skipped without a renderer, failed with its note
103 lines
3.1 KiB
Go
103 lines
3.1 KiB
Go
package crawl
|
|
|
|
import (
|
|
"strings"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
)
|
|
|
|
// DefaultGenericAnchors are the generic link texts of the en and pl seed
|
|
// lists (UI-SPEC C27, Q-1): the Site audit settings start with them.
|
|
var DefaultGenericAnchors = []string{
|
|
"click here", "here", "more", "read more", "learn more", "this link", "link",
|
|
"kliknij tutaj", "tutaj", "tu", "więcej", "czytaj więcej", "czytaj dalej", "ten link",
|
|
}
|
|
|
|
// ParseAnchors splits the generic anchors setting (one per line).
|
|
func ParseAnchors(text string) []string {
|
|
var out []string
|
|
for _, line := range strings.Split(text, "\n") {
|
|
if v := normal(line); v != "" {
|
|
out = append(out, v)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// maxLinkedFrom caps the linking pages a link finding keeps.
|
|
const maxLinkedFrom = 10
|
|
|
|
// checkLinks reports, per linked address, broken_link (404 or 410) and
|
|
// redirected_link (3xx) with the pages linking to it.
|
|
func checkLinks(res *Result, env *Env) []Finding {
|
|
var out []Finding
|
|
links := inbound(res)
|
|
for _, key := range res.Order {
|
|
p := res.Pages[key]
|
|
from := links[key]
|
|
if len(from) == 0 || p.Resp == nil || p.Resp.Err != "" {
|
|
continue
|
|
}
|
|
kept := from
|
|
if len(kept) > maxLinkedFrom {
|
|
kept = kept[:maxLinkedFrom]
|
|
}
|
|
rec := env.record(key)
|
|
switch s := p.Resp.Status; {
|
|
case s == 404 || s == 410:
|
|
out = append(out, Finding{Kind: KindBrokenLink, URL: key, Locale: locale(rec), Record: rec, Tool: ToolCreateRedirect, Details: map[string]any{"status": s, "count": len(from), "from": kept}})
|
|
case s >= 300 && s < 400:
|
|
target := p.Resp.Location
|
|
if u, err := parseURL(target); err == nil && res.fetcher != nil {
|
|
if k := res.fetcher.Key(u); k != "" {
|
|
target = k
|
|
}
|
|
}
|
|
// The link sits on the linking pages: their record's edit tool
|
|
// fixes it, else the template code.
|
|
linking := env.record(from[0])
|
|
f := Finding{Kind: KindRedirectedLink, URL: key, Locale: locale(rec), Record: linking, Details: map[string]any{"status": s, "target": target, "count": len(from), "from": kept}}
|
|
if linking != nil {
|
|
f.Content = []string{pact.ContentToolEdit}
|
|
}
|
|
out = append(out, f)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// checkGenericAnchors reports a page whose links have a generic text
|
|
// (D-81): the whole text, any case, is one of the generic anchors.
|
|
func checkGenericAnchors(res *Result, p *Page, rec *Record, anchors []string) []Finding {
|
|
if len(anchors) == 0 {
|
|
return nil
|
|
}
|
|
set := map[string]bool{}
|
|
for _, a := range anchors {
|
|
set[normal(a)] = true
|
|
}
|
|
var found []map[string]any
|
|
for _, l := range p.Facts.Links {
|
|
if !set[normal(l.Text)] {
|
|
continue
|
|
}
|
|
target := l.URL.String()
|
|
if res.fetcher != nil {
|
|
if k := res.fetcher.Key(l.URL); k != "" {
|
|
target = k
|
|
}
|
|
}
|
|
found = append(found, map[string]any{"text": l.Text, "target": target})
|
|
}
|
|
if len(found) == 0 {
|
|
return nil
|
|
}
|
|
f := Finding{Kind: KindGenericAnchor, URL: p.Key, Locale: locale(rec), Record: rec, Details: map[string]any{
|
|
"count": len(found), "text": found[0]["text"], "target": found[0]["target"], "anchors": found,
|
|
}}
|
|
if rec != nil {
|
|
f.Content = []string{pact.ContentToolEdit}
|
|
}
|
|
return []Finding{f}
|
|
}
|