feat: crawler finds a page with two H1 headings and stores the issue (tracer)

- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37)
- classes/crawl: fetchguard client on the site host and its www twin,
  sitemap index and robots.txt seeds, two workers with a delay, limits
- h1_missing and h1_multiple with the P33-38 fix route
- POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues
  answers the stored issues with id, severity, source, fix, found_at,
  ignored and crawl_id
This commit is contained in:
Jakub Zych
2026-11-04 15:05:45 +01:00
parent d38049a956
commit c2b61bf8e9
23 changed files with 2967 additions and 13 deletions

View File

@@ -0,0 +1,111 @@
package updates
import (
"github.com/go-gormigrate/gormigrate/v2"
"gorm.io/gorm"
)
// createCrawls creates the site audit's tables (D-14, P33-37): the crawls,
// the pages and links each crawl fetched, the stored issues (crawl and
// CMS-data findings, with the fixing tool or fix code and the fingerprint)
// and the ignored fingerprints. Deleting a crawl deletes its pages, links
// and issues.
var createCrawls = []*gormigrate.Migration{
{
ID: "202610280005_create_golem15_seo_crawls",
Migrate: func(tx *gorm.DB) error {
for _, stmt := range []string{
`CREATE TABLE golem15_seo_crawls (
id BIGSERIAL PRIMARY KEY,
status TEXT NOT NULL DEFAULT 'queued',
trigger TEXT NOT NULL DEFAULT 'admin',
trigger_name TEXT NOT NULL DEFAULT '',
site TEXT NOT NULL DEFAULT '',
max_pages INTEGER NOT NULL DEFAULT 0,
pages INTEGER NOT NULL DEFAULT 0,
limit_hit BOOLEAN NOT NULL DEFAULT FALSE,
error TEXT NOT NULL DEFAULT '',
render_status TEXT NOT NULL DEFAULT 'skipped',
render_pages INTEGER NOT NULL DEFAULT 0,
render_note TEXT NOT NULL DEFAULT '',
cancelled_by TEXT NOT NULL DEFAULT '',
job_id BIGINT NULL,
started_at TIMESTAMPTZ NULL,
finished_at TIMESTAMPTZ NULL,
created_at TIMESTAMPTZ NULL
)`,
`CREATE INDEX golem15_seo_crawls_status ON golem15_seo_crawls (status)`,
`CREATE TABLE golem15_seo_crawl_pages (
id BIGSERIAL PRIMARY KEY,
crawl_id BIGINT NOT NULL REFERENCES golem15_seo_crawls (id) ON DELETE CASCADE,
url TEXT NOT NULL,
status INTEGER NOT NULL DEFAULT 0,
location TEXT NOT NULL DEFAULT '',
content_type TEXT NOT NULL DEFAULT '',
title TEXT NOT NULL DEFAULT '',
description TEXT NOT NULL DEFAULT '',
canonical TEXT NOT NULL DEFAULT '',
robots TEXT NOT NULL DEFAULT '',
x_robots TEXT NOT NULL DEFAULT '',
h1 JSONB NULL,
headings JSONB NULL,
words INTEGER NOT NULL DEFAULT 0,
text TEXT NOT NULL DEFAULT '',
depth INTEGER NOT NULL DEFAULT 0,
in_sitemap BOOLEAN NOT NULL DEFAULT FALSE,
sitemap_file TEXT NOT NULL DEFAULT '',
links_out INTEGER NOT NULL DEFAULT 0,
links_in INTEGER NOT NULL DEFAULT 0,
fetched_at TIMESTAMPTZ NULL
)`,
`CREATE UNIQUE INDEX golem15_seo_crawl_pages_url ON golem15_seo_crawl_pages (crawl_id, url)`,
`CREATE TABLE golem15_seo_crawl_links (
id BIGSERIAL PRIMARY KEY,
crawl_id BIGINT NOT NULL REFERENCES golem15_seo_crawls (id) ON DELETE CASCADE,
from_url TEXT NOT NULL,
to_url TEXT NOT NULL,
text TEXT NOT NULL DEFAULT '',
rel TEXT NOT NULL DEFAULT '',
internal BOOLEAN NOT NULL DEFAULT TRUE
)`,
`CREATE INDEX golem15_seo_crawl_links_to ON golem15_seo_crawl_links (crawl_id, to_url)`,
`CREATE TABLE golem15_seo_issues (
id BIGSERIAL PRIMARY KEY,
crawl_id BIGINT NULL REFERENCES golem15_seo_crawls (id) ON DELETE CASCADE,
source TEXT NOT NULL DEFAULT 'crawl',
kind TEXT NOT NULL,
severity TEXT NOT NULL DEFAULT 'warning',
rank SMALLINT NOT NULL DEFAULT 2,
url TEXT NOT NULL DEFAULT '',
locale TEXT NOT NULL DEFAULT '',
details JSONB NULL,
tool TEXT NOT NULL DEFAULT '',
fix TEXT NOT NULL DEFAULT 'tool',
fingerprint TEXT NOT NULL,
found_at TIMESTAMPTZ NULL
)`,
`CREATE INDEX golem15_seo_issues_crawl ON golem15_seo_issues (crawl_id, rank, kind)`,
`CREATE INDEX golem15_seo_issues_fingerprint ON golem15_seo_issues (fingerprint)`,
`CREATE TABLE golem15_seo_issue_ignores (
id BIGSERIAL PRIMARY KEY,
fingerprint TEXT NOT NULL,
admin_id BIGINT NULL,
ignored_at TIMESTAMPTZ NULL
)`,
`CREATE UNIQUE INDEX golem15_seo_issue_ignores_fingerprint ON golem15_seo_issue_ignores (fingerprint)`,
} {
if err := tx.Exec(stmt).Error; err != nil {
return err
}
}
return nil
},
Rollback: func(tx *gorm.DB) error {
return tx.Exec(`DROP TABLE IF EXISTS golem15_seo_issue_ignores, golem15_seo_issues, golem15_seo_crawl_links, golem15_seo_crawl_pages, golem15_seo_crawls`).Error
},
},
}
func init() {
Register(createCrawls...)
}