feat: crawler finds a page with two H1 headings and stores the issue (tracer)
- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37) - classes/crawl: fetchguard client on the site host and its www twin, sitemap index and robots.txt seeds, two workers with a delay, limits - h1_missing and h1_multiple with the P33-38 fix route - POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues answers the stored issues with id, severity, source, fix, found_at, ignored and crawl_id
This commit is contained in:
111
updates/202610280005_create_golem15_seo_crawls.go
Normal file
111
updates/202610280005_create_golem15_seo_crawls.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"github.com/go-gormigrate/gormigrate/v2"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// createCrawls creates the site audit's tables (D-14, P33-37): the crawls,
|
||||
// the pages and links each crawl fetched, the stored issues (crawl and
|
||||
// CMS-data findings, with the fixing tool or fix code and the fingerprint)
|
||||
// and the ignored fingerprints. Deleting a crawl deletes its pages, links
|
||||
// and issues.
|
||||
var createCrawls = []*gormigrate.Migration{
|
||||
{
|
||||
ID: "202610280005_create_golem15_seo_crawls",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
for _, stmt := range []string{
|
||||
`CREATE TABLE golem15_seo_crawls (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
trigger TEXT NOT NULL DEFAULT 'admin',
|
||||
trigger_name TEXT NOT NULL DEFAULT '',
|
||||
site TEXT NOT NULL DEFAULT '',
|
||||
max_pages INTEGER NOT NULL DEFAULT 0,
|
||||
pages INTEGER NOT NULL DEFAULT 0,
|
||||
limit_hit BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
render_status TEXT NOT NULL DEFAULT 'skipped',
|
||||
render_pages INTEGER NOT NULL DEFAULT 0,
|
||||
render_note TEXT NOT NULL DEFAULT '',
|
||||
cancelled_by TEXT NOT NULL DEFAULT '',
|
||||
job_id BIGINT NULL,
|
||||
started_at TIMESTAMPTZ NULL,
|
||||
finished_at TIMESTAMPTZ NULL,
|
||||
created_at TIMESTAMPTZ NULL
|
||||
)`,
|
||||
`CREATE INDEX golem15_seo_crawls_status ON golem15_seo_crawls (status)`,
|
||||
`CREATE TABLE golem15_seo_crawl_pages (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
crawl_id BIGINT NOT NULL REFERENCES golem15_seo_crawls (id) ON DELETE CASCADE,
|
||||
url TEXT NOT NULL,
|
||||
status INTEGER NOT NULL DEFAULT 0,
|
||||
location TEXT NOT NULL DEFAULT '',
|
||||
content_type TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
canonical TEXT NOT NULL DEFAULT '',
|
||||
robots TEXT NOT NULL DEFAULT '',
|
||||
x_robots TEXT NOT NULL DEFAULT '',
|
||||
h1 JSONB NULL,
|
||||
headings JSONB NULL,
|
||||
words INTEGER NOT NULL DEFAULT 0,
|
||||
text TEXT NOT NULL DEFAULT '',
|
||||
depth INTEGER NOT NULL DEFAULT 0,
|
||||
in_sitemap BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
sitemap_file TEXT NOT NULL DEFAULT '',
|
||||
links_out INTEGER NOT NULL DEFAULT 0,
|
||||
links_in INTEGER NOT NULL DEFAULT 0,
|
||||
fetched_at TIMESTAMPTZ NULL
|
||||
)`,
|
||||
`CREATE UNIQUE INDEX golem15_seo_crawl_pages_url ON golem15_seo_crawl_pages (crawl_id, url)`,
|
||||
`CREATE TABLE golem15_seo_crawl_links (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
crawl_id BIGINT NOT NULL REFERENCES golem15_seo_crawls (id) ON DELETE CASCADE,
|
||||
from_url TEXT NOT NULL,
|
||||
to_url TEXT NOT NULL,
|
||||
text TEXT NOT NULL DEFAULT '',
|
||||
rel TEXT NOT NULL DEFAULT '',
|
||||
internal BOOLEAN NOT NULL DEFAULT TRUE
|
||||
)`,
|
||||
`CREATE INDEX golem15_seo_crawl_links_to ON golem15_seo_crawl_links (crawl_id, to_url)`,
|
||||
`CREATE TABLE golem15_seo_issues (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
crawl_id BIGINT NULL REFERENCES golem15_seo_crawls (id) ON DELETE CASCADE,
|
||||
source TEXT NOT NULL DEFAULT 'crawl',
|
||||
kind TEXT NOT NULL,
|
||||
severity TEXT NOT NULL DEFAULT 'warning',
|
||||
rank SMALLINT NOT NULL DEFAULT 2,
|
||||
url TEXT NOT NULL DEFAULT '',
|
||||
locale TEXT NOT NULL DEFAULT '',
|
||||
details JSONB NULL,
|
||||
tool TEXT NOT NULL DEFAULT '',
|
||||
fix TEXT NOT NULL DEFAULT 'tool',
|
||||
fingerprint TEXT NOT NULL,
|
||||
found_at TIMESTAMPTZ NULL
|
||||
)`,
|
||||
`CREATE INDEX golem15_seo_issues_crawl ON golem15_seo_issues (crawl_id, rank, kind)`,
|
||||
`CREATE INDEX golem15_seo_issues_fingerprint ON golem15_seo_issues (fingerprint)`,
|
||||
`CREATE TABLE golem15_seo_issue_ignores (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
fingerprint TEXT NOT NULL,
|
||||
admin_id BIGINT NULL,
|
||||
ignored_at TIMESTAMPTZ NULL
|
||||
)`,
|
||||
`CREATE UNIQUE INDEX golem15_seo_issue_ignores_fingerprint ON golem15_seo_issue_ignores (fingerprint)`,
|
||||
} {
|
||||
if err := tx.Exec(stmt).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Exec(`DROP TABLE IF EXISTS golem15_seo_issue_ignores, golem15_seo_issues, golem15_seo_crawl_links, golem15_seo_crawl_pages, golem15_seo_crawls`).Error
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
Register(createCrawls...)
|
||||
}
|
||||
Reference in New Issue
Block a user