feat: crawler checks for status, sitemaps, canonical, titles and headings

- page_status, sitemap_error, sitemap_noindex, sitemap_blocked and
  sitemap_missing against the fetched sitemap files and robots.txt
- canonical_missing and canonical_error, missing and duplicate titles and
  descriptions over the crawl
- h1_missing, h1_multiple, heading_skipped, heading_empty and
  h1_title_mismatch; content headings route to the record's tool
- the staging lock's noindex is never counted (Pitfall 12)
- TestCrawlerChecks on the planted site with a clean control page
This commit is contained in:
Jakub Zych
2026-11-04 15:10:22 +01:00
parent c2b61bf8e9
commit f2c93aeabe
6 changed files with 648 additions and 10 deletions

View File

@@ -525,3 +525,46 @@ crawls:
started: Crawl started. Follow it in SEO › Crawls; the issues appear in Site audit when it finishes.
already_running: A crawl is already running (started :time). Wait for it, or cancel it in SEO › Crawls.
no_site_url: The site address for crawls is not configured (golem15.seo.site_url). Ask the administrator to set it.
issues:
kind_page_status: "Page returns an error"
sum_page_status: "Status :status"
kind_sitemap_error: "Sitemap lists a page that does not answer 200"
sum_sitemap_error: "Status :status in :file"
kind_sitemap_noindex: "Sitemap lists a noindex page"
sum_sitemap_noindex: "In :file"
kind_sitemap_blocked: "Sitemap lists an address blocked in robots.txt"
sum_sitemap_blocked: "Blocked by line :line"
kind_sitemap_missing: "Indexable page missing from the sitemaps"
sum_sitemap_missing: "Found through :count links"
kind_canonical_missing: "No canonical link"
sum_canonical_missing: "—"
kind_canonical_error: "Canonical points to an error page"
sum_canonical_error: ":url answers :status"
kind_canonical_noindex: "Canonical points to a noindex page"
sum_canonical_noindex: ":url"
kind_canonical_redirected: "Canonical points to a redirect"
sum_canonical_redirected: ":url → :target"
kind_missing_title: "Missing title"
sum_missing_title: "—"
kind_duplicate_title: "Duplicate title"
sum_duplicate_title: "Same as :count other pages"
kind_missing_description: "Missing meta description"
sum_missing_description: "—"
kind_duplicate_description: "Duplicate meta description"
sum_duplicate_description: "Same as :count other pages"
kind_rule_overrides_record: "URL rule overrides the record"
sum_rule_overrides_record: ":fields"
kind_redirect_chain: "Redirect chain"
sum_redirect_chain: ":count hops to :final"
kind_noindex_blocked: "noindex page blocked in robots.txt"
sum_noindex_blocked: "Line :line"
kind_h1_missing: "No H1 heading"
sum_h1_missing: "—"
kind_h1_multiple: "More than one H1"
sum_h1_multiple: ":count H1 headings"
kind_heading_skipped: "Skipped heading level"
sum_heading_skipped: "H:from → H:to"
kind_heading_empty: "Empty heading"
sum_heading_empty: "H:level"
kind_h1_title_mismatch: "H1 does not match the title"
sum_h1_title_mismatch: "“:h1”"