c2b61bf8e9
- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37) - classes/crawl: fetchguard client on the site host and its www twin, sitemap index and robots.txt seeds, two workers with a delay, limits - h1_missing and h1_multiple with the P33-38 fix route - POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues answers the stored issues with id, severity, source, fix, found_at, ignored and crawl_id
104 lines
3.3 KiB
Go
104 lines
3.3 KiB
Go
package models
|
|
|
|
import (
|
|
"time"
|
|
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// The states of a crawl (UI-SPEC C24).
|
|
const (
|
|
CrawlQueued = "queued"
|
|
CrawlRunning = "running"
|
|
CrawlFinished = "finished"
|
|
CrawlFailed = "failed"
|
|
CrawlCancelled = "cancelled"
|
|
)
|
|
|
|
// CrawlStatuses lists the states in filter order.
|
|
var CrawlStatuses = []string{CrawlQueued, CrawlRunning, CrawlFinished, CrawlFailed, CrawlCancelled}
|
|
|
|
// What started a crawl.
|
|
const (
|
|
TriggerAdmin = "admin"
|
|
TriggerSchedule = "schedule"
|
|
TriggerMCP = "mcp"
|
|
)
|
|
|
|
// The states of the rendered comparison of a crawl (P33-55).
|
|
const (
|
|
RenderSkipped = "skipped"
|
|
RenderDone = "done"
|
|
RenderFailed = "failed"
|
|
)
|
|
|
|
// Crawl is one run of the site audit's crawler on golem15_seo_crawls
|
|
// (D-14, P33-37): its state, what started it and who, the site address,
|
|
// the page limit, the pages fetched and whether the limit stopped it, the
|
|
// error of a failed run, the state of the rendered comparison, the conga
|
|
// job and its times.
|
|
type Crawl struct {
|
|
ID uint `gorm:"column:id;primaryKey"`
|
|
Status string `gorm:"column:status"`
|
|
Trigger string `gorm:"column:trigger"`
|
|
TriggerName string `gorm:"column:trigger_name"`
|
|
Site string `gorm:"column:site"`
|
|
MaxPages int `gorm:"column:max_pages"`
|
|
Pages int `gorm:"column:pages"`
|
|
LimitHit bool `gorm:"column:limit_hit"`
|
|
Error string `gorm:"column:error"`
|
|
RenderStatus string `gorm:"column:render_status"`
|
|
RenderPages int `gorm:"column:render_pages"`
|
|
RenderNote string `gorm:"column:render_note"`
|
|
CancelledBy string `gorm:"column:cancelled_by"`
|
|
JobID *uint `gorm:"column:job_id"`
|
|
StartedAt *time.Time `gorm:"column:started_at"`
|
|
FinishedAt *time.Time `gorm:"column:finished_at"`
|
|
CreatedAt *time.Time `gorm:"column:created_at"`
|
|
// The list's values, filled by the Crawls controller's
|
|
// ListExtendRecords and never saved.
|
|
IssuesCell string `gorm:"-:all;column:issues"`
|
|
PagesCell string `gorm:"-:all;column:pages_cell"`
|
|
DurationCell string `gorm:"-:all;column:duration"`
|
|
TriggerCell string `gorm:"-:all;column:trigger_cell"`
|
|
}
|
|
|
|
func (Crawl) TableName() string { return "golem15_seo_crawls" }
|
|
|
|
// MorphName names a crawl in the admin.
|
|
func (Crawl) MorphName() string { return `Golem15\SEO\Models\Crawl` }
|
|
|
|
// Fillable is empty: crawls are written by the crawler only.
|
|
func (Crawl) Fillable() []string { return []string{} }
|
|
|
|
// Rules is empty: nothing is edited.
|
|
func (Crawl) Rules() map[string]string { return map[string]string{} }
|
|
|
|
// Active reports whether the crawl is queued or running.
|
|
func (c Crawl) Active() bool { return c.Status == CrawlQueued || c.Status == CrawlRunning }
|
|
|
|
// The Crawls list's filter scope.
|
|
const FilterCrawlStatus = "FilterCrawlStatus"
|
|
|
|
// FilterScopes lists the Crawls list's filter scopes.
|
|
func (Crawl) FilterScopes() []string { return []string{FilterCrawlStatus} }
|
|
|
|
// FilterScope narrows the Crawls list to one state; an unknown value
|
|
// matches nothing.
|
|
func (Crawl) FilterScope(name string, db *gorm.DB, value any) *gorm.DB {
|
|
if db == nil || name != FilterCrawlStatus {
|
|
return db
|
|
}
|
|
s, _ := value.(string)
|
|
for _, st := range CrawlStatuses {
|
|
if s == st {
|
|
return db.Where("golem15_seo_crawls.status = ?", s)
|
|
}
|
|
}
|
|
return db.Where("1 = 0")
|
|
}
|
|
|
|
func init() {
|
|
Register(Crawl{})
|
|
}
|