package models import ( "time" "git.golem15.com/golem15/summercms/modules/lagoon" ) // CrawlHeading is one heading of a crawled page in document order. type CrawlHeading struct { Level int `json:"level"` Text string `json:"text"` } // MaxPageText caps the stored main text of a crawled page, in characters. const MaxPageText = 100_000 // CrawlPage is one address a crawl fetched, on golem15_seo_crawl_pages // (P33-37): the answer (status, Location, content type) and what the raw // HTML says (title, description, canonical, robots, headings, word count, // main text), its depth from the home page, whether a sitemap lists it and // which file, and its outgoing and incoming internal links. type CrawlPage struct { ID uint `gorm:"column:id;primaryKey"` CrawlID uint `gorm:"column:crawl_id"` URL string `gorm:"column:url"` Status int `gorm:"column:status"` Location string `gorm:"column:location"` ContentType string `gorm:"column:content_type"` Title string `gorm:"column:title"` Description string `gorm:"column:description"` Canonical string `gorm:"column:canonical"` Robots string `gorm:"column:robots"` XRobots string `gorm:"column:x_robots"` H1 lagoon.Jsonable[[]string] `gorm:"column:h1"` Headings lagoon.Jsonable[[]CrawlHeading] `gorm:"column:headings"` Words int `gorm:"column:words"` Text string `gorm:"column:text"` Depth int `gorm:"column:depth"` InSitemap bool `gorm:"column:in_sitemap"` SitemapFile string `gorm:"column:sitemap_file"` LinksOut int `gorm:"column:links_out"` LinksIn int `gorm:"column:links_in"` FetchedAt *time.Time `gorm:"column:fetched_at"` } func (CrawlPage) TableName() string { return "golem15_seo_crawl_pages" } // CrawlLink is one link between two addresses of a crawl, on // golem15_seo_crawl_links: the linking page, the target, its text and rel, // and whether the target is on the site. type CrawlLink struct { ID uint `gorm:"column:id;primaryKey"` CrawlID uint `gorm:"column:crawl_id"` FromURL string `gorm:"column:from_url"` ToURL string `gorm:"column:to_url"` Text string `gorm:"column:text"` Rel string `gorm:"column:rel"` Internal bool `gorm:"column:internal"` } func (CrawlLink) TableName() string { return "golem15_seo_crawl_links" } func init() { Register(CrawlPage{}, CrawlLink{}) }