c2b61bf8e9
- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37) - classes/crawl: fetchguard client on the site host and its www twin, sitemap index and robots.txt seeds, two workers with a delay, limits - h1_missing and h1_multiple with the P33-38 fix route - POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues answers the stored issues with id, severity, source, fix, found_at, ignored and crawl_id
67 lines
2.8 KiB
Go
67 lines
2.8 KiB
Go
package models
|
|
|
|
import (
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/lagoon"
|
|
)
|
|
|
|
// CrawlHeading is one heading of a crawled page in document order.
|
|
type CrawlHeading struct {
|
|
Level int `json:"level"`
|
|
Text string `json:"text"`
|
|
}
|
|
|
|
// MaxPageText caps the stored main text of a crawled page, in characters.
|
|
const MaxPageText = 100_000
|
|
|
|
// CrawlPage is one address a crawl fetched, on golem15_seo_crawl_pages
|
|
// (P33-37): the answer (status, Location, content type) and what the raw
|
|
// HTML says (title, description, canonical, robots, headings, word count,
|
|
// main text), its depth from the home page, whether a sitemap lists it and
|
|
// which file, and its outgoing and incoming internal links.
|
|
type CrawlPage struct {
|
|
ID uint `gorm:"column:id;primaryKey"`
|
|
CrawlID uint `gorm:"column:crawl_id"`
|
|
URL string `gorm:"column:url"`
|
|
Status int `gorm:"column:status"`
|
|
Location string `gorm:"column:location"`
|
|
ContentType string `gorm:"column:content_type"`
|
|
Title string `gorm:"column:title"`
|
|
Description string `gorm:"column:description"`
|
|
Canonical string `gorm:"column:canonical"`
|
|
Robots string `gorm:"column:robots"`
|
|
XRobots string `gorm:"column:x_robots"`
|
|
H1 lagoon.Jsonable[[]string] `gorm:"column:h1"`
|
|
Headings lagoon.Jsonable[[]CrawlHeading] `gorm:"column:headings"`
|
|
Words int `gorm:"column:words"`
|
|
Text string `gorm:"column:text"`
|
|
Depth int `gorm:"column:depth"`
|
|
InSitemap bool `gorm:"column:in_sitemap"`
|
|
SitemapFile string `gorm:"column:sitemap_file"`
|
|
LinksOut int `gorm:"column:links_out"`
|
|
LinksIn int `gorm:"column:links_in"`
|
|
FetchedAt *time.Time `gorm:"column:fetched_at"`
|
|
}
|
|
|
|
func (CrawlPage) TableName() string { return "golem15_seo_crawl_pages" }
|
|
|
|
// CrawlLink is one link between two addresses of a crawl, on
|
|
// golem15_seo_crawl_links: the linking page, the target, its text and rel,
|
|
// and whether the target is on the site.
|
|
type CrawlLink struct {
|
|
ID uint `gorm:"column:id;primaryKey"`
|
|
CrawlID uint `gorm:"column:crawl_id"`
|
|
FromURL string `gorm:"column:from_url"`
|
|
ToURL string `gorm:"column:to_url"`
|
|
Text string `gorm:"column:text"`
|
|
Rel string `gorm:"column:rel"`
|
|
Internal bool `gorm:"column:internal"`
|
|
}
|
|
|
|
func (CrawlLink) TableName() string { return "golem15_seo_crawl_links" }
|
|
|
|
func init() {
|
|
Register(CrawlPage{}, CrawlLink{})
|
|
}
|