feat: crawler checks for JSON-LD, hreflang, links, images, raw HTML and the rendered-comparison seam
- jsonld_invalid, jsonld_multiple, and jsonld_required/recommended from schema.Validate (the E9 rules); publisher nodes route to settings - hreflang_error and hreflang_reciprocal over the fetched alternates - broken_link and redirected_link with the linking pages, generic_anchor against the en and pl seed lists - image_alt, image_size and featured_lazy - content_js check cms: the record title in the H1, half its words and its primary parent's link in the raw HTML - compareRendered (pure): links, an H1 or text only after JavaScript as content_js check rendered; the Renderer seam and render bookkeeping - TestRenderSkipped: skipped without a renderer, failed with its note
This commit is contained in:
Jakub Zych
@@ -1,10 +1,13 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes/schema"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"github.com/jimsmart/grobotstxt"
|
||||
@@ -187,7 +190,14 @@ func Check(res *Result, env *Env) []Finding {
|
||||
}
|
||||
out = append(out, checkHead(res, p, rec)...)
|
||||
out = append(out, checkHeadings(p, rec)...)
|
||||
out = append(out, checkJSONLD(p, rec)...)
|
||||
out = append(out, checkHreflang(res, p, rec)...)
|
||||
out = append(out, checkGenericAnchors(res, p, rec, env.GenericAnchors)...)
|
||||
out = append(out, checkImages(p, rec)...)
|
||||
out = append(out, checkContentCMS(res, p, rec)...)
|
||||
out = append(out, checkRenderedPage(res, p, rec)...)
|
||||
}
|
||||
out = append(out, checkLinks(res, env)...)
|
||||
out = append(out, checkSitemaps(res, env)...)
|
||||
out = append(out, checkDuplicates(res, env)...)
|
||||
return out
|
||||
@@ -440,3 +450,264 @@ func checkDuplicates(res *Result, env *Env) []Finding {
|
||||
emit(descs, KindDuplicateDesc)
|
||||
return out
|
||||
}
|
||||
|
||||
// publisherTypes are the node types the SEO settings generate (publisher,
|
||||
// local business, the site): their missing properties are fixed through
|
||||
// seo_update_settings.
|
||||
var publisherTypes = map[string]bool{
|
||||
"Organization": true, "Person": true, "WebSite": true, "LocalBusiness": true,
|
||||
"ProfessionalService": true, "Store": true, "FoodEstablishment": true,
|
||||
"HealthAndBeautyBusiness": true, "MedicalBusiness": true, "LegalService": true,
|
||||
"HomeAndConstructionBusiness": true, "AutomotiveBusiness": true, "FinancialService": true,
|
||||
}
|
||||
|
||||
// checkJSONLD reads every JSON-LD script of a page: jsonld_invalid for one
|
||||
// that is not JSON, jsonld_multiple for more than one script, and the
|
||||
// graph validated with the rules of the admin preview (schema.Validate,
|
||||
// D-46): jsonld_required and jsonld_recommended. Types without Google
|
||||
// rules (FAQPage and the like) find nothing.
|
||||
func checkJSONLD(p *Page, rec *Record) []Finding {
|
||||
scripts := p.Facts.JSONLD
|
||||
if len(scripts) == 0 {
|
||||
return nil
|
||||
}
|
||||
loc := locale(rec)
|
||||
var out []Finding
|
||||
var errs []string
|
||||
var nodes []any
|
||||
for _, raw := range scripts {
|
||||
var v any
|
||||
dec := json.NewDecoder(strings.NewReader(raw))
|
||||
dec.UseNumber()
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
errs = append(errs, err.Error())
|
||||
continue
|
||||
}
|
||||
switch x := v.(type) {
|
||||
case map[string]any:
|
||||
if g, ok := x["@graph"].([]any); ok {
|
||||
nodes = append(nodes, g...)
|
||||
} else {
|
||||
nodes = append(nodes, x)
|
||||
}
|
||||
case []any:
|
||||
nodes = append(nodes, x...)
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
f := Finding{Kind: KindJSONLDInvalid, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"error": errs[0], "count": len(errs)}}
|
||||
if rec != nil && rec.CustomJSONLD {
|
||||
f.Tool = ToolSetURLSEO
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
if len(scripts) > 1 {
|
||||
out = append(out, Finding{Kind: KindJSONLDMultiple, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"count": len(scripts)}})
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
return out
|
||||
}
|
||||
type miss struct{ typ, prop string }
|
||||
var required, recommended []miss
|
||||
for _, rep := range schema.Validate(map[string]any{"@graph": nodes}) {
|
||||
for _, f := range rep.Findings {
|
||||
m := miss{rep.Type, f.Property}
|
||||
if f.Level == schema.LevelRequired {
|
||||
required = append(required, m)
|
||||
} else {
|
||||
recommended = append(recommended, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
emit := func(kind string, list []miss) {
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
all := make([]map[string]any, 0, len(list))
|
||||
for _, m := range list {
|
||||
all = append(all, map[string]any{"type": m.typ, "property": m.prop})
|
||||
}
|
||||
tool := ToolSetURLSEO
|
||||
if publisherTypes[list[0].typ] {
|
||||
tool = ToolUpdateSettings
|
||||
}
|
||||
out = append(out, Finding{Kind: kind, URL: p.Key, Locale: loc, Record: rec, Tool: tool, Details: map[string]any{
|
||||
"type": list[0].typ, "property": list[0].prop, "count": len(list), "missing": all,
|
||||
}})
|
||||
}
|
||||
emit(KindJSONLDRequired, required)
|
||||
emit(KindJSONLDRecommended, recommended)
|
||||
return out
|
||||
}
|
||||
|
||||
// checkHreflang follows a page's hreflang alternates: hreflang_error for
|
||||
// an alternate that does not answer 200, hreflang_reciprocal for one that
|
||||
// answers but does not link back.
|
||||
func checkHreflang(res *Result, p *Page, rec *Record) []Finding {
|
||||
if res.fetcher == nil {
|
||||
return nil
|
||||
}
|
||||
loc := locale(rec)
|
||||
var broken, oneWay []map[string]any
|
||||
for _, a := range p.Facts.Alternates {
|
||||
key := res.fetcher.Key(a.URL)
|
||||
if key == "" || key == p.Key || strings.EqualFold(a.Lang, "x-default") {
|
||||
continue
|
||||
}
|
||||
t := res.Page(key)
|
||||
if t == nil || t.Resp == nil || t.Resp.Err == "canceled" {
|
||||
continue
|
||||
}
|
||||
if !t.OK() {
|
||||
broken = append(broken, map[string]any{"locale": a.Lang, "url": key, "status": t.Resp.Status})
|
||||
continue
|
||||
}
|
||||
back := false
|
||||
for _, b := range t.Facts.Alternates {
|
||||
if res.fetcher.Key(b.URL) == p.Key {
|
||||
back = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !back {
|
||||
oneWay = append(oneWay, map[string]any{"locale": a.Lang, "url": key})
|
||||
}
|
||||
}
|
||||
var out []Finding
|
||||
if len(broken) > 0 {
|
||||
d := map[string]any{"count": len(broken), "alternates": broken}
|
||||
for k, v := range broken[0] {
|
||||
d[k] = v
|
||||
}
|
||||
out = append(out, Finding{Kind: KindHreflangError, URL: p.Key, Locale: loc, Record: rec, Tool: ToolCreateRedirect, Details: d})
|
||||
}
|
||||
if len(oneWay) > 0 {
|
||||
d := map[string]any{"count": len(oneWay), "alternates": oneWay}
|
||||
for k, v := range oneWay[0] {
|
||||
d[k] = v
|
||||
}
|
||||
f := Finding{Kind: KindHreflangRecip, URL: p.Key, Locale: loc, Record: rec, Details: d}
|
||||
if rec != nil {
|
||||
f.Content = []string{pact.ContentToolEdit}
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// maxListed caps the images a finding names.
|
||||
const maxListed = 10
|
||||
|
||||
// checkImages reports image_alt (an <img> without an alt attribute; an
|
||||
// empty alt is a decorative image), image_size (no width or height) and
|
||||
// featured_lazy (the og:image loads lazily). An image the record lists is
|
||||
// fixed through the record's image_alt tool.
|
||||
func checkImages(p *Page, rec *Record) []Finding {
|
||||
loc := locale(rec)
|
||||
var noAlt, noSize []string
|
||||
var lazy string
|
||||
recordImage := false
|
||||
for _, img := range p.Facts.Images {
|
||||
if !img.HasAlt {
|
||||
noAlt = append(noAlt, img.Src)
|
||||
if rec != nil {
|
||||
for _, ri := range rec.Images {
|
||||
if ri.URL != "" && (strings.HasSuffix(img.Src, ri.URL) || ri.URL == img.Src) {
|
||||
recordImage = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(img.Width) == "" || strings.TrimSpace(img.Height) == "" {
|
||||
noSize = append(noSize, img.Src)
|
||||
}
|
||||
if p.Facts.OGImage != "" && img.Src == p.Facts.OGImage && img.Loading == "lazy" {
|
||||
lazy = img.Src
|
||||
}
|
||||
}
|
||||
cut := func(list []string) []string {
|
||||
if len(list) > maxListed {
|
||||
return list[:maxListed]
|
||||
}
|
||||
return list
|
||||
}
|
||||
var out []Finding
|
||||
if len(noAlt) > 0 {
|
||||
f := Finding{Kind: KindImageAlt, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"count": len(noAlt), "images": cut(noAlt)}}
|
||||
if recordImage {
|
||||
f.Content = []string{pact.ContentToolImageAlt, pact.ContentToolEdit}
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
if len(noSize) > 0 {
|
||||
out = append(out, Finding{Kind: KindImageSize, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"count": len(noSize), "images": cut(noSize)}})
|
||||
}
|
||||
if lazy != "" {
|
||||
out = append(out, Finding{Kind: KindFeaturedLazy, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"image": lazy}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkContentCMS compares the server-rendered HTML with what the CMS
|
||||
// record says (D-70, D-71; check cms): the record title must appear in an
|
||||
// H1, the text must reach half of the record's word count, and the
|
||||
// record's CMS-known links must be present. A miss is content_js.
|
||||
func checkContentCMS(res *Result, p *Page, rec *Record) []Finding {
|
||||
if rec == nil {
|
||||
return nil
|
||||
}
|
||||
var what []string
|
||||
details := map[string]any{}
|
||||
if title := normal(rec.Title); title != "" {
|
||||
found := false
|
||||
for _, h := range p.Facts.H1() {
|
||||
if strings.Contains(normal(h), title) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
details["title"] = rec.Title
|
||||
what = append(what, "the title “"+rec.Title+"” in the H1")
|
||||
}
|
||||
}
|
||||
if rec.Words != nil && *rec.Words > 0 && p.Facts.Words*2 < *rec.Words {
|
||||
details["words_raw"] = p.Facts.Words
|
||||
details["words_cms"] = *rec.Words
|
||||
what = append(what, strconv.Itoa(p.Facts.Words)+" of "+strconv.Itoa(*rec.Words)+" words")
|
||||
}
|
||||
if len(rec.Links) > 0 && res.fetcher != nil {
|
||||
have := map[string]bool{}
|
||||
for _, l := range p.Facts.Links {
|
||||
if k := res.fetcher.Key(l.URL); k != "" {
|
||||
have[k] = true
|
||||
}
|
||||
}
|
||||
var missing []string
|
||||
for _, l := range rec.Links {
|
||||
if !have[l] {
|
||||
missing = append(missing, l)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
details["links"] = missing
|
||||
what = append(what, "links "+strings.Join(missing, ", "))
|
||||
}
|
||||
}
|
||||
if len(what) == 0 {
|
||||
return nil
|
||||
}
|
||||
return []Finding{{Kind: KindContentJS, URL: p.Key, Locale: locale(rec), Record: rec, Check: checkCMS, Details: details, What: strings.Join(what, "; ")}}
|
||||
}
|
||||
|
||||
// checkRenderedPage runs the raw versus rendered comparison of a page the
|
||||
// renderer rendered.
|
||||
func checkRenderedPage(res *Result, p *Page, rec *Record) []Finding {
|
||||
if p.Rendered == nil || res.fetcher == nil {
|
||||
return nil
|
||||
}
|
||||
out := compareRendered(p.Facts, p.Rendered, res.fetcher.Hosts())
|
||||
for i := range out {
|
||||
out[i].URL, out[i].Locale, out[i].Record = p.Key, locale(rec), rec
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
97
classes/crawl/compare.go
Normal file
97
classes/crawl/compare.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The thresholds of the rendered comparison (P33-55): text counts as
|
||||
// added by JavaScript when the rendered page has at least growthWords more
|
||||
// words and growthShare more than the raw HTML.
|
||||
const (
|
||||
growthWords = 50
|
||||
growthShare = 0.2
|
||||
maxJSOnlyURLs = 5
|
||||
)
|
||||
|
||||
// compareRendered compares a page's raw HTML with its JavaScript-rendered
|
||||
// DOM (D-70, P33-55; pure): same-host links, an H1 or a substantial amount
|
||||
// of text that appear only after JavaScript are one content_js finding
|
||||
// with details.check rendered and details.what naming them. hosts are the
|
||||
// site host and its www twin. An identical rendering finds nothing.
|
||||
func compareRendered(raw, rendered *PageFacts, hosts []string) []Finding {
|
||||
if raw == nil || rendered == nil {
|
||||
return nil
|
||||
}
|
||||
same := func(u *url.URL) (string, bool) {
|
||||
if u == nil {
|
||||
return "", false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
for _, h := range hosts {
|
||||
if host == h {
|
||||
p := u.EscapedPath()
|
||||
if p == "" {
|
||||
p = "/"
|
||||
}
|
||||
if u.RawQuery != "" {
|
||||
p += "?" + u.RawQuery
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
before := map[string]bool{}
|
||||
for _, l := range raw.Links {
|
||||
if k, ok := same(l.URL); ok {
|
||||
before[k] = true
|
||||
}
|
||||
}
|
||||
var added []string
|
||||
seen := map[string]bool{}
|
||||
for _, l := range rendered.Links {
|
||||
k, ok := same(l.URL)
|
||||
if !ok || before[k] || seen[k] {
|
||||
continue
|
||||
}
|
||||
seen[k] = true
|
||||
added = append(added, k)
|
||||
}
|
||||
details := map[string]any{}
|
||||
var what []string
|
||||
if len(added) > 0 {
|
||||
shown := added
|
||||
if len(shown) > maxJSOnlyURLs {
|
||||
shown = shown[:maxJSOnlyURLs]
|
||||
}
|
||||
details["links"] = shown
|
||||
what = append(what, "links "+strings.Join(shown, ", "))
|
||||
}
|
||||
if len(raw.H1()) == 0 {
|
||||
if h1 := rendered.H1(); len(h1) > 0 {
|
||||
details["h1"] = h1[0]
|
||||
what = append(what, "H1 “"+h1[0]+"”")
|
||||
}
|
||||
}
|
||||
if grow := rendered.Words - raw.Words; grow >= growthWords && float64(grow) >= growthShare*float64(raw.Words) {
|
||||
details["words_raw"] = raw.Words
|
||||
details["words_rendered"] = rendered.Words
|
||||
what = append(what, strconv.Itoa(grow)+" words of text")
|
||||
}
|
||||
if len(what) == 0 {
|
||||
return nil
|
||||
}
|
||||
return []Finding{{Kind: KindContentJS, Check: checkRendered, Details: details, What: strings.Join(what, "; ")}}
|
||||
}
|
||||
|
||||
// The two content_js checks: the raw HTML against the CMS record, and the
|
||||
// raw HTML against the JavaScript-rendered DOM.
|
||||
const (
|
||||
checkCMS = "cms"
|
||||
checkRendered = "rendered"
|
||||
)
|
||||
|
||||
// parseURL parses an absolute or site-relative address.
|
||||
func parseURL(raw string) (*url.URL, error) { return url.Parse(strings.TrimSpace(raw)) }
|
||||
@@ -221,7 +221,7 @@ func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.C
|
||||
if err := SavePages(tx, row.ID, res); err != nil {
|
||||
return err
|
||||
}
|
||||
findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging()})
|
||||
findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging(), GenericAnchors: DefaultGenericAnchors})
|
||||
cms, err := svc.Issues(classes.WithTx(ctx, tx))
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
102
classes/crawl/links.go
Normal file
102
classes/crawl/links.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
)
|
||||
|
||||
// DefaultGenericAnchors are the generic link texts of the en and pl seed
|
||||
// lists (UI-SPEC C27, Q-1): the Site audit settings start with them.
|
||||
var DefaultGenericAnchors = []string{
|
||||
"click here", "here", "more", "read more", "learn more", "this link", "link",
|
||||
"kliknij tutaj", "tutaj", "tu", "więcej", "czytaj więcej", "czytaj dalej", "ten link",
|
||||
}
|
||||
|
||||
// ParseAnchors splits the generic anchors setting (one per line).
|
||||
func ParseAnchors(text string) []string {
|
||||
var out []string
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
if v := normal(line); v != "" {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// maxLinkedFrom caps the linking pages a link finding keeps.
|
||||
const maxLinkedFrom = 10
|
||||
|
||||
// checkLinks reports, per linked address, broken_link (404 or 410) and
|
||||
// redirected_link (3xx) with the pages linking to it.
|
||||
func checkLinks(res *Result, env *Env) []Finding {
|
||||
var out []Finding
|
||||
links := inbound(res)
|
||||
for _, key := range res.Order {
|
||||
p := res.Pages[key]
|
||||
from := links[key]
|
||||
if len(from) == 0 || p.Resp == nil || p.Resp.Err != "" {
|
||||
continue
|
||||
}
|
||||
kept := from
|
||||
if len(kept) > maxLinkedFrom {
|
||||
kept = kept[:maxLinkedFrom]
|
||||
}
|
||||
rec := env.record(key)
|
||||
switch s := p.Resp.Status; {
|
||||
case s == 404 || s == 410:
|
||||
out = append(out, Finding{Kind: KindBrokenLink, URL: key, Locale: locale(rec), Record: rec, Tool: ToolCreateRedirect, Details: map[string]any{"status": s, "count": len(from), "from": kept}})
|
||||
case s >= 300 && s < 400:
|
||||
target := p.Resp.Location
|
||||
if u, err := parseURL(target); err == nil && res.fetcher != nil {
|
||||
if k := res.fetcher.Key(u); k != "" {
|
||||
target = k
|
||||
}
|
||||
}
|
||||
// The link sits on the linking pages: their record's edit tool
|
||||
// fixes it, else the template code.
|
||||
linking := env.record(from[0])
|
||||
f := Finding{Kind: KindRedirectedLink, URL: key, Locale: locale(rec), Record: linking, Details: map[string]any{"status": s, "target": target, "count": len(from), "from": kept}}
|
||||
if linking != nil {
|
||||
f.Content = []string{pact.ContentToolEdit}
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkGenericAnchors reports a page whose links have a generic text
|
||||
// (D-81): the whole text, any case, is one of the generic anchors.
|
||||
func checkGenericAnchors(res *Result, p *Page, rec *Record, anchors []string) []Finding {
|
||||
if len(anchors) == 0 {
|
||||
return nil
|
||||
}
|
||||
set := map[string]bool{}
|
||||
for _, a := range anchors {
|
||||
set[normal(a)] = true
|
||||
}
|
||||
var found []map[string]any
|
||||
for _, l := range p.Facts.Links {
|
||||
if !set[normal(l.Text)] {
|
||||
continue
|
||||
}
|
||||
target := l.URL.String()
|
||||
if res.fetcher != nil {
|
||||
if k := res.fetcher.Key(l.URL); k != "" {
|
||||
target = k
|
||||
}
|
||||
}
|
||||
found = append(found, map[string]any{"text": l.Text, "target": target})
|
||||
}
|
||||
if len(found) == 0 {
|
||||
return nil
|
||||
}
|
||||
f := Finding{Kind: KindGenericAnchor, URL: p.Key, Locale: locale(rec), Record: rec, Details: map[string]any{
|
||||
"count": len(found), "text": found[0]["text"], "target": found[0]["target"], "anchors": found,
|
||||
}}
|
||||
if rec != nil {
|
||||
f.Content = []string{pact.ContentToolEdit}
|
||||
}
|
||||
return []Finding{f}
|
||||
}
|
||||
102
crawler_test.go
102
crawler_test.go
@@ -56,7 +56,10 @@ var checkedKinds = map[string]bool{
|
||||
"sitemap_missing": true, "canonical_missing": true, "canonical_error": true, "missing_title": true,
|
||||
"duplicate_title": true, "missing_description": true, "duplicate_description": true,
|
||||
"h1_missing": true, "h1_multiple": true, "heading_skipped": true, "heading_empty": true,
|
||||
"h1_title_mismatch": true,
|
||||
"h1_title_mismatch": true, "jsonld_invalid": true, "jsonld_multiple": true, "jsonld_required": true,
|
||||
"jsonld_recommended": true, "hreflang_reciprocal": true, "hreflang_error": true, "generic_anchor": true,
|
||||
"broken_link": true, "redirected_link": true, "image_alt": true, "image_size": true, "featured_lazy": true,
|
||||
"content_js": true,
|
||||
}
|
||||
|
||||
// onlyChecked keeps the "kind url" entries of checkedKinds.
|
||||
@@ -140,4 +143,101 @@ func TestCrawlerChecks(t *testing.T) {
|
||||
if i := findIssue(t, list, "missing_title", "/p/no-title"); i.Tool != "seo_set_url_seo" {
|
||||
t.Fatalf("missing_title = %+v", i)
|
||||
}
|
||||
detail("jsonld_multiple", "/p/jsonld-two", "count", 2)
|
||||
detail("jsonld_required", "/p/jsonld-req", "type", "BreadcrumbList")
|
||||
detail("jsonld_required", "/p/jsonld-req", "property", "itemListElement")
|
||||
detail("jsonld_recommended", "/p/jsonld-rec", "property", "logo")
|
||||
if i := findIssue(t, list, "jsonld_recommended", "/p/jsonld-rec"); i.Tool != "seo_update_settings" || i.Severity != "notice" {
|
||||
t.Fatalf("jsonld_recommended = %+v", i)
|
||||
}
|
||||
if i := findIssue(t, list, "jsonld_invalid", "/p/jsonld-bad"); i.Details["error"] == nil || i.Fix != "code" {
|
||||
t.Fatalf("jsonld_invalid = %+v", i)
|
||||
}
|
||||
detail("hreflang_reciprocal", "/en/x", "locale", "pl")
|
||||
detail("hreflang_reciprocal", "/en/x", "url", "/pl/x")
|
||||
detail("hreflang_error", "/p/hreflang-err", "url", "/de/missing")
|
||||
detail("hreflang_error", "/p/hreflang-err", "status", 404)
|
||||
detail("generic_anchor", "/p/generic", "count", 2)
|
||||
detail("generic_anchor", "/p/generic", "text", "kliknij tutaj")
|
||||
detail("generic_anchor", "/p/generic", "target", "/clean")
|
||||
detail("broken_link", "/p/gone", "from", []string{"/"})
|
||||
if i := findIssue(t, list, "broken_link", "/p/gone"); i.Tool != "seo_create_redirect" || i.Severity != "error" {
|
||||
t.Fatalf("broken_link = %+v", i)
|
||||
}
|
||||
detail("redirected_link", "/p/moved", "target", "/clean")
|
||||
detail("image_alt", "/p/img-noalt", "count", 1)
|
||||
detail("image_size", "/p/img-nosize", "count", 1)
|
||||
detail("featured_lazy", "/p/lazy", "image", siteURL+"/hero.jpg")
|
||||
content := func(url, check, what string) {
|
||||
t.Helper()
|
||||
i := findIssue(t, list, "content_js", url)
|
||||
if i.Details["check"] != check || !strings.Contains(fmt.Sprint(i.Details["what"]), what) || i.Fix != "code" {
|
||||
t.Fatalf("content_js %s = %+v, want check %s naming %q", url, i, check, what)
|
||||
}
|
||||
}
|
||||
content("/c/js-h1", "cms", "Record heading")
|
||||
content("/c/js-words", "cms", "of 400 words")
|
||||
content("/c/js-links", "cms", "/c/parent")
|
||||
content("/p/render", "rendered", "/p/js-only")
|
||||
content("/p/render", "rendered", "words")
|
||||
content("/p/render-h1", "rendered", "H1")
|
||||
var crawlRow struct {
|
||||
RenderStatus string
|
||||
RenderPages int
|
||||
}
|
||||
if err := env.db.Raw(`SELECT render_status, render_pages FROM golem15_seo_crawls ORDER BY id DESC LIMIT 1`).Scan(&crawlRow).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if crawlRow.RenderStatus != "done" || crawlRow.RenderPages < 10 {
|
||||
t.Fatalf("render state = %+v", crawlRow)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderSkipped checks the comparison's absence and failure (D-70,
|
||||
// P33-55): without a renderer (no golem15.seo.crawl_chrome_path) the crawl
|
||||
// runs every raw check and records render_status skipped with no rendered
|
||||
// finding; a renderer whose Render fails records render_status failed with
|
||||
// its note, and the crawl still finishes with the raw checks.
|
||||
func TestRenderSkipped(t *testing.T) {
|
||||
env := bootCrawlEnv(t)
|
||||
token := env.admin(t, true)
|
||||
setCrawlRecords(t, auditRecords())
|
||||
site := newFakeSite()
|
||||
plantAudit(site)
|
||||
|
||||
renderState := func(id uint) (status string, pages int, note, crawlStatus string) {
|
||||
t.Helper()
|
||||
var row struct {
|
||||
RenderStatus string
|
||||
RenderPages int
|
||||
RenderNote string
|
||||
Status string
|
||||
}
|
||||
if err := env.db.Raw(`SELECT render_status, render_pages, render_note, status FROM golem15_seo_crawls WHERE id = ?`, id).Scan(&row).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return row.RenderStatus, row.RenderPages, row.RenderNote, row.Status
|
||||
}
|
||||
|
||||
id := env.crawlOnce(t, token, siteCtx(t, site))
|
||||
status, pages, _, crawlStatus := renderState(id)
|
||||
if status != "skipped" || pages != 0 || crawlStatus != "finished" {
|
||||
t.Fatalf("without a renderer: render %s, %d pages, crawl %s", status, pages, crawlStatus)
|
||||
}
|
||||
list := env.issues(t, token, "")
|
||||
for _, i := range list {
|
||||
if i.Kind == "content_js" && i.Details["check"] == "rendered" {
|
||||
t.Fatalf("rendered finding without a renderer: %+v", i)
|
||||
}
|
||||
}
|
||||
findIssue(t, list, "content_js", "/c/js-h1")
|
||||
findIssue(t, list, "h1_multiple", "/p/two-h1")
|
||||
|
||||
failing := &fakeRenderer{site: site, fail: fmt.Errorf("chrome exited")}
|
||||
id = env.crawlOnce(t, token, crawl.WithRenderer(siteCtx(t, site), failing))
|
||||
status, _, note, crawlStatus := renderState(id)
|
||||
if status != "failed" || !strings.Contains(note, "chrome exited") || crawlStatus != "finished" {
|
||||
t.Fatalf("failing renderer: render %s (%q), crawl %s", status, note, crawlStatus)
|
||||
}
|
||||
findIssue(t, env.issues(t, token, ""), "h1_multiple", "/p/two-h1")
|
||||
}
|
||||
|
||||
@@ -526,6 +526,32 @@ crawls:
|
||||
already_running: A crawl is already running (started :time). Wait for it, or cancel it in SEO › Crawls.
|
||||
no_site_url: The site address for crawls is not configured (golem15.seo.site_url). Ask the administrator to set it.
|
||||
issues:
|
||||
kind_jsonld_invalid: "Structured data cannot be read"
|
||||
sum_jsonld_invalid: ":error"
|
||||
kind_jsonld_multiple: "More than one JSON-LD graph"
|
||||
sum_jsonld_multiple: ":count scripts"
|
||||
kind_jsonld_required: "Structured data misses a required property"
|
||||
sum_jsonld_required: ":type: :property"
|
||||
kind_jsonld_recommended: "Structured data misses a recommended property"
|
||||
sum_jsonld_recommended: ":type: :property"
|
||||
kind_hreflang_reciprocal: "hreflang has no return link"
|
||||
sum_hreflang_reciprocal: ":locale → :url"
|
||||
kind_hreflang_error: "hreflang points to an error page"
|
||||
sum_hreflang_error: ":locale → :url (:status)"
|
||||
kind_generic_anchor: "Generic link text"
|
||||
sum_generic_anchor: "“:text” → :target"
|
||||
kind_broken_link: "Link to a missing page"
|
||||
sum_broken_link: "From :count pages"
|
||||
kind_redirected_link: "Link to a redirect"
|
||||
sum_redirected_link: "Redirects to :target"
|
||||
kind_image_alt: "Image without alt text"
|
||||
sum_image_alt: ":count images"
|
||||
kind_image_size: "Image without width and height"
|
||||
sum_image_size: ":count images"
|
||||
kind_featured_lazy: "Main image loads lazily"
|
||||
sum_featured_lazy: "—"
|
||||
kind_content_js: "Content needs JavaScript"
|
||||
sum_content_js: "Missing from the raw HTML: :what"
|
||||
kind_page_status: "Page returns an error"
|
||||
sum_page_status: "Status :status"
|
||||
kind_sitemap_error: "Sitemap lists a page that does not answer 200"
|
||||
|
||||
@@ -540,6 +540,32 @@ crawls:
|
||||
already_running: Skanowanie już trwa (rozpoczęte :time). Poczekaj na nie lub anuluj je w SEO › Skanowania.
|
||||
no_site_url: Adres witryny do skanowania nie jest skonfigurowany (golem15.seo.site_url). Poproś administratora o jego ustawienie.
|
||||
issues:
|
||||
kind_jsonld_invalid: "Nie można odczytać danych strukturalnych"
|
||||
sum_jsonld_invalid: ":error"
|
||||
kind_jsonld_multiple: "Więcej niż jeden graf JSON-LD"
|
||||
sum_jsonld_multiple: "Skrypty: :count"
|
||||
kind_jsonld_required: "Brak wymaganej właściwości w danych strukturalnych"
|
||||
sum_jsonld_required: ":type: :property"
|
||||
kind_jsonld_recommended: "Brak zalecanej właściwości w danych strukturalnych"
|
||||
sum_jsonld_recommended: ":type: :property"
|
||||
kind_hreflang_reciprocal: "hreflang bez linku zwrotnego"
|
||||
sum_hreflang_reciprocal: ":locale → :url"
|
||||
kind_hreflang_error: "hreflang wskazuje stronę z błędem"
|
||||
sum_hreflang_error: ":locale → :url (:status)"
|
||||
kind_generic_anchor: "Ogólnikowy tekst linku"
|
||||
sum_generic_anchor: "„:text” → :target"
|
||||
kind_broken_link: "Link do nieistniejącej strony"
|
||||
sum_broken_link: "Z :count stron"
|
||||
kind_redirected_link: "Link do przekierowania"
|
||||
sum_redirected_link: "Przekierowuje na :target"
|
||||
kind_image_alt: "Obraz bez tekstu alternatywnego"
|
||||
sum_image_alt: "Obrazy: :count"
|
||||
kind_image_size: "Obraz bez szerokości i wysokości"
|
||||
sum_image_size: "Obrazy: :count"
|
||||
kind_featured_lazy: "Główny obraz ładuje się z opóźnieniem (lazy)"
|
||||
sum_featured_lazy: "—"
|
||||
kind_content_js: "Treść wymaga JavaScriptu"
|
||||
sum_content_js: "Brak w surowym HTML: :what"
|
||||
kind_page_status: "Strona zwraca błąd"
|
||||
sum_page_status: "Kod :status"
|
||||
kind_sitemap_error: "Mapa witryny zawiera stronę bez kodu 200"
|
||||
|
||||
Reference in New Issue
Block a user