feat: crawler finds a page with two H1 headings and stores the issue (tracer)
- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37) - classes/crawl: fetchguard client on the site host and its www twin, sitemap index and robots.txt seeds, two workers with a delay, limits - h1_missing and h1_multiple with the P33-38 fix route - POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues answers the stored issues with id, severity, source, fix, found_at, ignored and crawl_id
This commit is contained in:
Jakub Zych
103
models/crawl.go
Normal file
103
models/crawl.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// The states of a crawl (UI-SPEC C24).
|
||||
const (
|
||||
CrawlQueued = "queued"
|
||||
CrawlRunning = "running"
|
||||
CrawlFinished = "finished"
|
||||
CrawlFailed = "failed"
|
||||
CrawlCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// CrawlStatuses lists the states in filter order.
|
||||
var CrawlStatuses = []string{CrawlQueued, CrawlRunning, CrawlFinished, CrawlFailed, CrawlCancelled}
|
||||
|
||||
// What started a crawl.
|
||||
const (
|
||||
TriggerAdmin = "admin"
|
||||
TriggerSchedule = "schedule"
|
||||
TriggerMCP = "mcp"
|
||||
)
|
||||
|
||||
// The states of the rendered comparison of a crawl (P33-55).
|
||||
const (
|
||||
RenderSkipped = "skipped"
|
||||
RenderDone = "done"
|
||||
RenderFailed = "failed"
|
||||
)
|
||||
|
||||
// Crawl is one run of the site audit's crawler on golem15_seo_crawls
|
||||
// (D-14, P33-37): its state, what started it and who, the site address,
|
||||
// the page limit, the pages fetched and whether the limit stopped it, the
|
||||
// error of a failed run, the state of the rendered comparison, the conga
|
||||
// job and its times.
|
||||
type Crawl struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
Status string `gorm:"column:status"`
|
||||
Trigger string `gorm:"column:trigger"`
|
||||
TriggerName string `gorm:"column:trigger_name"`
|
||||
Site string `gorm:"column:site"`
|
||||
MaxPages int `gorm:"column:max_pages"`
|
||||
Pages int `gorm:"column:pages"`
|
||||
LimitHit bool `gorm:"column:limit_hit"`
|
||||
Error string `gorm:"column:error"`
|
||||
RenderStatus string `gorm:"column:render_status"`
|
||||
RenderPages int `gorm:"column:render_pages"`
|
||||
RenderNote string `gorm:"column:render_note"`
|
||||
CancelledBy string `gorm:"column:cancelled_by"`
|
||||
JobID *uint `gorm:"column:job_id"`
|
||||
StartedAt *time.Time `gorm:"column:started_at"`
|
||||
FinishedAt *time.Time `gorm:"column:finished_at"`
|
||||
CreatedAt *time.Time `gorm:"column:created_at"`
|
||||
// The list's values, filled by the Crawls controller's
|
||||
// ListExtendRecords and never saved.
|
||||
IssuesCell string `gorm:"-:all;column:issues"`
|
||||
PagesCell string `gorm:"-:all;column:pages_cell"`
|
||||
DurationCell string `gorm:"-:all;column:duration"`
|
||||
TriggerCell string `gorm:"-:all;column:trigger_cell"`
|
||||
}
|
||||
|
||||
func (Crawl) TableName() string { return "golem15_seo_crawls" }
|
||||
|
||||
// MorphName names a crawl in the admin.
|
||||
func (Crawl) MorphName() string { return `Golem15\SEO\Models\Crawl` }
|
||||
|
||||
// Fillable is empty: crawls are written by the crawler only.
|
||||
func (Crawl) Fillable() []string { return []string{} }
|
||||
|
||||
// Rules is empty: nothing is edited.
|
||||
func (Crawl) Rules() map[string]string { return map[string]string{} }
|
||||
|
||||
// Active reports whether the crawl is queued or running.
|
||||
func (c Crawl) Active() bool { return c.Status == CrawlQueued || c.Status == CrawlRunning }
|
||||
|
||||
// The Crawls list's filter scope.
|
||||
const FilterCrawlStatus = "FilterCrawlStatus"
|
||||
|
||||
// FilterScopes lists the Crawls list's filter scopes.
|
||||
func (Crawl) FilterScopes() []string { return []string{FilterCrawlStatus} }
|
||||
|
||||
// FilterScope narrows the Crawls list to one state; an unknown value
|
||||
// matches nothing.
|
||||
func (Crawl) FilterScope(name string, db *gorm.DB, value any) *gorm.DB {
|
||||
if db == nil || name != FilterCrawlStatus {
|
||||
return db
|
||||
}
|
||||
s, _ := value.(string)
|
||||
for _, st := range CrawlStatuses {
|
||||
if s == st {
|
||||
return db.Where("golem15_seo_crawls.status = ?", s)
|
||||
}
|
||||
}
|
||||
return db.Where("1 = 0")
|
||||
}
|
||||
|
||||
func init() {
|
||||
Register(Crawl{})
|
||||
}
|
||||
66
models/crawl_page.go
Normal file
66
models/crawl_page.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
)
|
||||
|
||||
// CrawlHeading is one heading of a crawled page in document order.
|
||||
type CrawlHeading struct {
|
||||
Level int `json:"level"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// MaxPageText caps the stored main text of a crawled page, in characters.
|
||||
const MaxPageText = 100_000
|
||||
|
||||
// CrawlPage is one address a crawl fetched, on golem15_seo_crawl_pages
|
||||
// (P33-37): the answer (status, Location, content type) and what the raw
|
||||
// HTML says (title, description, canonical, robots, headings, word count,
|
||||
// main text), its depth from the home page, whether a sitemap lists it and
|
||||
// which file, and its outgoing and incoming internal links.
|
||||
type CrawlPage struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
CrawlID uint `gorm:"column:crawl_id"`
|
||||
URL string `gorm:"column:url"`
|
||||
Status int `gorm:"column:status"`
|
||||
Location string `gorm:"column:location"`
|
||||
ContentType string `gorm:"column:content_type"`
|
||||
Title string `gorm:"column:title"`
|
||||
Description string `gorm:"column:description"`
|
||||
Canonical string `gorm:"column:canonical"`
|
||||
Robots string `gorm:"column:robots"`
|
||||
XRobots string `gorm:"column:x_robots"`
|
||||
H1 lagoon.Jsonable[[]string] `gorm:"column:h1"`
|
||||
Headings lagoon.Jsonable[[]CrawlHeading] `gorm:"column:headings"`
|
||||
Words int `gorm:"column:words"`
|
||||
Text string `gorm:"column:text"`
|
||||
Depth int `gorm:"column:depth"`
|
||||
InSitemap bool `gorm:"column:in_sitemap"`
|
||||
SitemapFile string `gorm:"column:sitemap_file"`
|
||||
LinksOut int `gorm:"column:links_out"`
|
||||
LinksIn int `gorm:"column:links_in"`
|
||||
FetchedAt *time.Time `gorm:"column:fetched_at"`
|
||||
}
|
||||
|
||||
func (CrawlPage) TableName() string { return "golem15_seo_crawl_pages" }
|
||||
|
||||
// CrawlLink is one link between two addresses of a crawl, on
|
||||
// golem15_seo_crawl_links: the linking page, the target, its text and rel,
|
||||
// and whether the target is on the site.
|
||||
type CrawlLink struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
CrawlID uint `gorm:"column:crawl_id"`
|
||||
FromURL string `gorm:"column:from_url"`
|
||||
ToURL string `gorm:"column:to_url"`
|
||||
Text string `gorm:"column:text"`
|
||||
Rel string `gorm:"column:rel"`
|
||||
Internal bool `gorm:"column:internal"`
|
||||
}
|
||||
|
||||
func (CrawlLink) TableName() string { return "golem15_seo_crawl_links" }
|
||||
|
||||
func init() {
|
||||
Register(CrawlPage{}, CrawlLink{})
|
||||
}
|
||||
156
models/issue.go
Normal file
156
models/issue.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Where a stored issue was found (UI-SPEC C23).
|
||||
const (
|
||||
SourceCrawl = "crawl"
|
||||
SourceCMS = "cms"
|
||||
)
|
||||
|
||||
// How an issue is fixed (P33-38): through an MCP tool that changes CMS
|
||||
// data, or in the site's template code.
|
||||
const (
|
||||
FixTool = "tool"
|
||||
FixCode = "code"
|
||||
)
|
||||
|
||||
// The severities of an issue and their sort rank (errors first).
|
||||
const (
|
||||
SeverityError = "error"
|
||||
SeverityWarning = "warning"
|
||||
SeverityNotice = "notice"
|
||||
)
|
||||
|
||||
// SeverityRank orders the severities: error 1, warning 2, notice 3.
|
||||
func SeverityRank(severity string) int {
|
||||
switch severity {
|
||||
case SeverityError:
|
||||
return 1
|
||||
case SeverityWarning:
|
||||
return 2
|
||||
}
|
||||
return 3
|
||||
}
|
||||
|
||||
// Issue is one stored finding of the site audit on golem15_seo_issues
|
||||
// (P33-37): the crawl it belongs to (nil for CMS-data issues stored before
|
||||
// any crawl), where it was found, its kind and severity, the address and
|
||||
// locale, the kind's details, the fixing MCP tool or fix code, and the
|
||||
// fingerprint an ignore is keyed by.
|
||||
type Issue struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
CrawlID *uint `gorm:"column:crawl_id"`
|
||||
Source string `gorm:"column:source"`
|
||||
Kind string `gorm:"column:kind"`
|
||||
Severity string `gorm:"column:severity"`
|
||||
Rank int `gorm:"column:rank"`
|
||||
URL string `gorm:"column:url"`
|
||||
Locale string `gorm:"column:locale"`
|
||||
Details lagoon.Jsonable[map[string]any] `gorm:"column:details"`
|
||||
Tool string `gorm:"column:tool"`
|
||||
Fix string `gorm:"column:fix"`
|
||||
Fingerprint string `gorm:"column:fingerprint"`
|
||||
FoundAt *time.Time `gorm:"column:found_at"`
|
||||
// The list's values, filled by the Site audit controller's
|
||||
// ListExtendRecords and never saved.
|
||||
Summary string `gorm:"-:all;column:summary"`
|
||||
FixCell string `gorm:"-:all;column:fix_cell"`
|
||||
KindCell string `gorm:"-:all;column:kind_cell"`
|
||||
}
|
||||
|
||||
func (Issue) TableName() string { return "golem15_seo_issues" }
|
||||
|
||||
// MorphName names an issue in the admin.
|
||||
func (Issue) MorphName() string { return `Golem15\SEO\Models\Issue` }
|
||||
|
||||
// Fillable is empty: issues are written by the audit only.
|
||||
func (Issue) Fillable() []string { return []string{} }
|
||||
|
||||
// Rules is empty: nothing is edited.
|
||||
func (Issue) Rules() map[string]string { return map[string]string{} }
|
||||
|
||||
// DetailsMap returns the issue's details, never nil.
|
||||
func (i Issue) DetailsMap() map[string]any {
|
||||
if d := i.Details.Get(); d != nil {
|
||||
return d
|
||||
}
|
||||
return map[string]any{}
|
||||
}
|
||||
|
||||
// IssueIgnore is one ignored fingerprint on golem15_seo_issue_ignores: an
|
||||
// issue with this fingerprint stays hidden in later crawls until it is no
|
||||
// longer ignored (UI-SPEC C23).
|
||||
type IssueIgnore struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
Fingerprint string `gorm:"column:fingerprint"`
|
||||
AdminID *uint `gorm:"column:admin_id"`
|
||||
IgnoredAt *time.Time `gorm:"column:ignored_at"`
|
||||
}
|
||||
|
||||
func (IssueIgnore) TableName() string { return "golem15_seo_issue_ignores" }
|
||||
|
||||
// The Site audit list's filter scopes and the status and fix values.
|
||||
const (
|
||||
FilterIssueSeverity = "FilterIssueSeverity"
|
||||
FilterIssueKind = "FilterIssueKind"
|
||||
FilterIssueSource = "FilterIssueSource"
|
||||
FilterIssueFix = "FilterIssueFix"
|
||||
FilterIssueStatus = "FilterIssueStatus"
|
||||
IssueOpen = "open"
|
||||
IssueIgnored = "ignored"
|
||||
)
|
||||
|
||||
// IgnoredSQL is true for an issue whose fingerprint is ignored.
|
||||
const IgnoredSQL = `EXISTS (SELECT 1 FROM golem15_seo_issue_ignores AS ig WHERE ig.fingerprint = golem15_seo_issues.fingerprint)`
|
||||
|
||||
// FilterScopes lists the Site audit list's filter scopes.
|
||||
func (Issue) FilterScopes() []string {
|
||||
return []string{FilterIssueSeverity, FilterIssueKind, FilterIssueSource, FilterIssueFix, FilterIssueStatus}
|
||||
}
|
||||
|
||||
// FilterScope narrows the Site audit list; an unknown value matches
|
||||
// nothing.
|
||||
func (Issue) FilterScope(name string, db *gorm.DB, value any) *gorm.DB {
|
||||
if db == nil {
|
||||
return db
|
||||
}
|
||||
s, _ := value.(string)
|
||||
switch name {
|
||||
case FilterIssueSeverity:
|
||||
if s == SeverityError || s == SeverityWarning || s == SeverityNotice {
|
||||
return db.Where("golem15_seo_issues.severity = ?", s)
|
||||
}
|
||||
case FilterIssueKind:
|
||||
if s != "" {
|
||||
return db.Where("golem15_seo_issues.kind = ?", s)
|
||||
}
|
||||
case FilterIssueSource:
|
||||
if s == SourceCrawl || s == SourceCMS {
|
||||
return db.Where("golem15_seo_issues.source = ?", s)
|
||||
}
|
||||
case FilterIssueFix:
|
||||
if s == FixTool || s == FixCode {
|
||||
return db.Where("golem15_seo_issues.fix = ?", s)
|
||||
}
|
||||
case FilterIssueStatus:
|
||||
switch s {
|
||||
case IssueOpen:
|
||||
return db.Where("NOT " + IgnoredSQL)
|
||||
case IssueIgnored:
|
||||
return db.Where(IgnoredSQL)
|
||||
}
|
||||
default:
|
||||
return db
|
||||
}
|
||||
return db.Where("1 = 0")
|
||||
}
|
||||
|
||||
func init() {
|
||||
Register(Issue{}, IssueIgnore{})
|
||||
}
|
||||
Reference in New Issue
Block a user