feat: crawler finds a page with two H1 headings and stores the issue (tracer)
- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37) - classes/crawl: fetchguard client on the site host and its www twin, sitemap index and robots.txt seeds, two workers with a delay, limits - h1_missing and h1_multiple with the P33-38 fix route - POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues answers the stored issues with id, severity, source, fix, found_at, ignored and crawl_id
This commit is contained in:
Jakub Zych
151
classes/crawl/checks.go
Normal file
151
classes/crawl/checks.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
)
|
||||
|
||||
// The crawl kinds of UI-SPEC C23.
|
||||
const (
|
||||
KindPageStatus = "page_status"
|
||||
KindSitemapError = "sitemap_error"
|
||||
KindSitemapNoindex = "sitemap_noindex"
|
||||
KindSitemapBlocked = "sitemap_blocked"
|
||||
KindSitemapMissing = "sitemap_missing"
|
||||
KindBrokenLink = "broken_link"
|
||||
KindRedirectedLink = "redirected_link"
|
||||
KindCanonicalMissing = "canonical_missing"
|
||||
KindCanonicalError = "canonical_error"
|
||||
KindMissingTitle = "missing_title"
|
||||
KindDuplicateTitle = "duplicate_title"
|
||||
KindMissingDesc = "missing_description"
|
||||
KindDuplicateDesc = "duplicate_description"
|
||||
KindH1Missing = "h1_missing"
|
||||
KindH1Multiple = "h1_multiple"
|
||||
KindHeadingSkipped = "heading_skipped"
|
||||
KindHeadingEmpty = "heading_empty"
|
||||
KindH1TitleMismatch = "h1_title_mismatch"
|
||||
KindJSONLDInvalid = "jsonld_invalid"
|
||||
KindJSONLDMultiple = "jsonld_multiple"
|
||||
KindJSONLDRequired = "jsonld_required"
|
||||
KindJSONLDRecommended = "jsonld_recommended"
|
||||
KindHreflangRecip = "hreflang_reciprocal"
|
||||
KindHreflangError = "hreflang_error"
|
||||
KindGenericAnchor = "generic_anchor"
|
||||
KindImageAlt = "image_alt"
|
||||
KindImageSize = "image_size"
|
||||
KindFeaturedLazy = "featured_lazy"
|
||||
KindContentJS = "content_js"
|
||||
KindOrphanPage = "orphan_page"
|
||||
// KindH1InContent is the CMS-data check of a record whose body holds
|
||||
// an H1 although its page renders the title as the H1.
|
||||
KindH1InContent = "h1_in_content"
|
||||
)
|
||||
|
||||
// severities are the default severities of the kinds table (UI-SPEC C23),
|
||||
// the Phase 32 CMS kinds included.
|
||||
var severities = map[string]string{
|
||||
KindPageStatus: models.SeverityError,
|
||||
KindSitemapError: models.SeverityError,
|
||||
KindSitemapNoindex: models.SeverityError,
|
||||
KindSitemapBlocked: models.SeverityError,
|
||||
KindSitemapMissing: models.SeverityNotice,
|
||||
KindBrokenLink: models.SeverityError,
|
||||
KindRedirectedLink: models.SeverityNotice,
|
||||
KindCanonicalMissing: models.SeverityWarning,
|
||||
KindCanonicalError: models.SeverityError,
|
||||
"canonical_noindex": models.SeverityError,
|
||||
"canonical_redirected": models.SeverityWarning,
|
||||
KindMissingTitle: models.SeverityWarning,
|
||||
KindDuplicateTitle: models.SeverityWarning,
|
||||
KindMissingDesc: models.SeverityNotice,
|
||||
KindDuplicateDesc: models.SeverityNotice,
|
||||
"rule_overrides_record": models.SeverityNotice,
|
||||
"redirect_chain": models.SeverityWarning,
|
||||
"noindex_blocked": models.SeverityError,
|
||||
KindH1Missing: models.SeverityError,
|
||||
KindH1Multiple: models.SeverityError,
|
||||
KindH1InContent: models.SeverityWarning,
|
||||
KindHeadingSkipped: models.SeverityWarning,
|
||||
KindHeadingEmpty: models.SeverityWarning,
|
||||
KindH1TitleMismatch: models.SeverityNotice,
|
||||
KindJSONLDInvalid: models.SeverityError,
|
||||
KindJSONLDMultiple: models.SeverityWarning,
|
||||
KindJSONLDRequired: models.SeverityError,
|
||||
KindJSONLDRecommended: models.SeverityNotice,
|
||||
KindHreflangRecip: models.SeverityWarning,
|
||||
KindHreflangError: models.SeverityError,
|
||||
KindGenericAnchor: models.SeverityWarning,
|
||||
KindImageAlt: models.SeverityWarning,
|
||||
KindImageSize: models.SeverityNotice,
|
||||
KindFeaturedLazy: models.SeverityNotice,
|
||||
KindContentJS: models.SeverityWarning,
|
||||
KindOrphanPage: models.SeverityWarning,
|
||||
}
|
||||
|
||||
// Severity returns a kind's severity (warning for an unknown kind).
|
||||
func Severity(kind string) string {
|
||||
if s, ok := severities[kind]; ok {
|
||||
return s
|
||||
}
|
||||
return models.SeverityWarning
|
||||
}
|
||||
|
||||
// Kinds lists every kind the audit stores, in the kinds table order.
|
||||
func Kinds() []string {
|
||||
return []string{
|
||||
KindPageStatus, KindSitemapError, KindSitemapNoindex, KindSitemapBlocked,
|
||||
KindSitemapMissing, KindBrokenLink, KindRedirectedLink, KindCanonicalMissing,
|
||||
KindCanonicalError, "canonical_noindex", "canonical_redirected", KindMissingTitle,
|
||||
KindDuplicateTitle, KindMissingDesc, KindDuplicateDesc, "rule_overrides_record",
|
||||
"redirect_chain", "noindex_blocked", KindH1Missing, KindH1Multiple, KindH1InContent,
|
||||
KindHeadingSkipped, KindHeadingEmpty, KindH1TitleMismatch, KindJSONLDInvalid,
|
||||
KindJSONLDMultiple, KindJSONLDRequired, KindJSONLDRecommended, KindHreflangRecip,
|
||||
KindHreflangError, KindGenericAnchor, KindImageAlt, KindImageSize, KindFeaturedLazy,
|
||||
KindContentJS, KindOrphanPage,
|
||||
}
|
||||
}
|
||||
|
||||
// Env is what the checks know besides the crawl: the record behind an
|
||||
// address.
|
||||
type Env struct {
|
||||
// Lookup returns the CMS record of an address, nil when none.
|
||||
Lookup func(key string) *Record
|
||||
}
|
||||
|
||||
func (e *Env) record(key string) *Record {
|
||||
if e == nil || e.Lookup == nil {
|
||||
return nil
|
||||
}
|
||||
return e.Lookup(key)
|
||||
}
|
||||
|
||||
// Check runs the per-page checks over a crawl's pages.
|
||||
func Check(res *Result, env *Env) []Finding {
|
||||
var out []Finding
|
||||
for _, key := range res.Order {
|
||||
p := res.Pages[key]
|
||||
if !p.OK() {
|
||||
continue
|
||||
}
|
||||
out = append(out, checkHeadings(p, env.record(key))...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkHeadings reports h1_missing and h1_multiple. An H1 inside the
|
||||
// record's own content is fixed through the record's headings tool.
|
||||
func checkHeadings(p *Page, rec *Record) []Finding {
|
||||
h1 := p.Facts.H1()
|
||||
switch {
|
||||
case len(h1) == 0:
|
||||
return []Finding{{Kind: KindH1Missing, URL: p.Key, Record: rec}}
|
||||
case len(h1) > 1:
|
||||
f := Finding{Kind: KindH1Multiple, URL: p.Key, Record: rec, Details: map[string]any{"count": len(h1), "h1": h1}}
|
||||
if rec.hasHeading(1, "") {
|
||||
f.Content = []string{pact.ContentToolHeadings, pact.ContentToolEdit}
|
||||
}
|
||||
return []Finding{f}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
425
classes/crawl/crawler.go
Normal file
425
classes/crawl/crawler.go
Normal file
@@ -0,0 +1,425 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrNoSiteURL is a crawl started without a site address
|
||||
// (golem15.seo.site_url, else app.url).
|
||||
var ErrNoSiteURL = errors.New("golem15.seo: the site address for crawls is not configured")
|
||||
|
||||
// The paths the crawler seeds from (RESEARCH Pattern 7).
|
||||
const (
|
||||
HomePath = "/"
|
||||
SitemapPath = "/sitemap.xml"
|
||||
RobotsPath = "/robots.txt"
|
||||
)
|
||||
|
||||
// maxSitemapFiles caps the sitemap files read from an index.
|
||||
const maxSitemapFiles = 500
|
||||
|
||||
// How a page entered the frontier.
|
||||
const (
|
||||
viaHome = "home"
|
||||
viaSitemap = "sitemap"
|
||||
viaLink = "link"
|
||||
viaRedirect = "redirect"
|
||||
viaCanonical = "canonical"
|
||||
viaAlternate = "alternate"
|
||||
)
|
||||
|
||||
// Page is one fetched address of a crawl: the answer, the facts of its raw
|
||||
// HTML, the facts of its JavaScript-rendered DOM when the comparison ran,
|
||||
// its depth from the home page and the sitemap file that lists it.
|
||||
type Page struct {
|
||||
Key string
|
||||
Depth int
|
||||
Via string
|
||||
Resp *Response
|
||||
Facts *PageFacts
|
||||
Rendered *PageFacts
|
||||
InSitemap bool
|
||||
SitemapFile string
|
||||
FetchedAt time.Time
|
||||
}
|
||||
|
||||
// OK reports whether the page answered 200 with HTML.
|
||||
func (p *Page) OK() bool {
|
||||
return p != nil && p.Resp != nil && p.Resp.Err == "" && p.Resp.Status == 200 && p.Facts != nil
|
||||
}
|
||||
|
||||
// SitemapFile is one sitemap the crawl read: its address, status and the
|
||||
// page addresses it lists.
|
||||
type SitemapFile struct {
|
||||
Key string
|
||||
Status int
|
||||
URLs []string
|
||||
}
|
||||
|
||||
// RenderState is the outcome of the rendered comparison (P33-55).
|
||||
type RenderState struct {
|
||||
Status string
|
||||
Pages int
|
||||
Note string
|
||||
}
|
||||
|
||||
// Result is what one crawl fetched, in frontier order.
|
||||
type Result struct {
|
||||
Pages map[string]*Page
|
||||
Order []string
|
||||
Sitemaps []SitemapFile
|
||||
// Listed maps a page address to the sitemap file listing it.
|
||||
Listed map[string]string
|
||||
Robots string
|
||||
RobotsStatus int
|
||||
LimitHit bool
|
||||
Canceled bool
|
||||
Render RenderState
|
||||
fetcher *Fetcher
|
||||
}
|
||||
|
||||
// Page returns the fetched page at key, nil when none.
|
||||
func (r *Result) Page(key string) *Page {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
return r.Pages[key]
|
||||
}
|
||||
|
||||
// Options are the limits and seams of one crawl.
|
||||
type Options struct {
|
||||
// MaxPages caps the fetched addresses (D-14, default 5000).
|
||||
MaxPages int
|
||||
// Delay is the pause between two requests (default 250 ms).
|
||||
Delay time.Duration
|
||||
// Renderer renders pages for the comparison; nil skips it.
|
||||
Renderer Renderer
|
||||
// RenderMax caps the rendered pages (default 200, crawl order).
|
||||
RenderMax int
|
||||
// Canceled is asked between requests; true stops the crawl and keeps
|
||||
// what was fetched.
|
||||
Canceled func(ctx context.Context) bool
|
||||
// Progress is told the number of fetched pages.
|
||||
Progress func(ctx context.Context, fetched int)
|
||||
}
|
||||
|
||||
// item is one frontier entry.
|
||||
type item struct {
|
||||
key string
|
||||
depth int
|
||||
via string
|
||||
}
|
||||
|
||||
// pacer spaces requests at least delay apart across the workers.
|
||||
type pacer struct {
|
||||
mu sync.Mutex
|
||||
next time.Time
|
||||
delay time.Duration
|
||||
}
|
||||
|
||||
func (p *pacer) wait(ctx context.Context) error {
|
||||
p.mu.Lock()
|
||||
now := time.Now()
|
||||
at := p.next
|
||||
if at.Before(now) {
|
||||
at = now
|
||||
}
|
||||
p.next = at.Add(p.delay)
|
||||
p.mu.Unlock()
|
||||
if d := time.Until(at); d > 0 {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Crawl fetches the site through f, a fetchguard client limited to the
|
||||
// site host and its www twin (fetchguard.AllowHostsMode, HTTPS, no
|
||||
// redirects followed; TrustedMode only from Go config): robots.txt, the
|
||||
// sitemap index and its files, then breadth-first from the home page and
|
||||
// the sitemap addresses over same-host links, redirect targets, canonicals
|
||||
// and hreflang alternates, two requests at a time with the delay between
|
||||
// requests, up to MaxPages. robots.txt rules are read for the report, not
|
||||
// obeyed (P33-41). Cancelling keeps what was fetched. After each HTML page
|
||||
// answering 200, a configured Renderer renders it (one tab at a time, up
|
||||
// to RenderMax pages) for the raw versus rendered comparison.
|
||||
func Crawl(ctx context.Context, f *Fetcher, opt Options) (*Result, error) {
|
||||
if opt.MaxPages <= 0 {
|
||||
opt.MaxPages = 5000
|
||||
}
|
||||
if opt.RenderMax <= 0 {
|
||||
opt.RenderMax = DefaultRenderMax
|
||||
}
|
||||
res := &Result{Pages: map[string]*Page{}, Listed: map[string]string{}, Render: RenderState{Status: renderStatusSkipped}, fetcher: f}
|
||||
if opt.Renderer != nil {
|
||||
res.Render.Status = renderStatusDone
|
||||
}
|
||||
pace := &pacer{delay: opt.Delay}
|
||||
canceled := func() bool {
|
||||
if ctx.Err() != nil {
|
||||
return true
|
||||
}
|
||||
return opt.Canceled != nil && opt.Canceled(ctx)
|
||||
}
|
||||
|
||||
// robots.txt, for the sitemap_blocked check.
|
||||
if err := pace.wait(ctx); err != nil {
|
||||
return res, err
|
||||
}
|
||||
if r := f.Get(ctx, RobotsPath); r.Err == "" {
|
||||
res.RobotsStatus = r.Status
|
||||
if r.Status == 200 {
|
||||
res.Robots = string(r.Body)
|
||||
}
|
||||
}
|
||||
|
||||
// The sitemap index and its files.
|
||||
if err := readSitemaps(ctx, f, pace, res); err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
var queue []item
|
||||
push := func(key string, depth int, via string) {
|
||||
if key == "" || seen[key] {
|
||||
return
|
||||
}
|
||||
seen[key] = true
|
||||
queue = append(queue, item{key: key, depth: depth, via: via})
|
||||
}
|
||||
push(HomePath, 0, viaHome)
|
||||
for _, sm := range res.Sitemaps {
|
||||
for _, key := range sm.URLs {
|
||||
push(key, 1, viaSitemap)
|
||||
}
|
||||
}
|
||||
|
||||
var renderMu sync.Mutex
|
||||
rendered := 0
|
||||
renderOff := opt.Renderer == nil
|
||||
for len(queue) > 0 {
|
||||
if canceled() {
|
||||
res.Canceled = true
|
||||
break
|
||||
}
|
||||
if len(res.Order) >= opt.MaxPages {
|
||||
res.LimitHit = true
|
||||
break
|
||||
}
|
||||
n := Workers
|
||||
if room := opt.MaxPages - len(res.Order); n > room {
|
||||
n = room
|
||||
}
|
||||
if n > len(queue) {
|
||||
n = len(queue)
|
||||
}
|
||||
batch := queue[:n]
|
||||
queue = queue[n:]
|
||||
pages := make([]*Page, len(batch))
|
||||
var wg sync.WaitGroup
|
||||
for i, it := range batch {
|
||||
wg.Add(1)
|
||||
go func(i int, it item) {
|
||||
defer wg.Done()
|
||||
p := &Page{Key: it.key, Depth: it.depth, Via: it.via}
|
||||
if err := pace.wait(ctx); err != nil {
|
||||
p.Resp = &Response{Err: "canceled"}
|
||||
pages[i] = p
|
||||
return
|
||||
}
|
||||
p.Resp = f.Get(ctx, it.key)
|
||||
p.FetchedAt = time.Now().UTC()
|
||||
if p.Resp.Err == "" && p.Resp.Status == 200 && p.Resp.HTML() {
|
||||
base, _ := url.Parse(f.Absolute(it.key))
|
||||
p.Facts = Parse(base, p.Resp.Body)
|
||||
renderMu.Lock()
|
||||
if !renderOff && rendered < opt.RenderMax && ctx.Err() == nil {
|
||||
rendered++
|
||||
dom, err := opt.Renderer.Render(ctx, f.Absolute(it.key))
|
||||
if err != nil {
|
||||
renderOff = true
|
||||
rendered--
|
||||
res.Render = RenderState{Status: renderStatusFailed, Note: err.Error()}
|
||||
} else {
|
||||
p.Rendered = Parse(base, []byte(dom))
|
||||
}
|
||||
}
|
||||
renderMu.Unlock()
|
||||
}
|
||||
pages[i] = p
|
||||
}(i, it)
|
||||
}
|
||||
wg.Wait()
|
||||
for _, p := range pages {
|
||||
if file, ok := res.Listed[p.Key]; ok {
|
||||
p.InSitemap, p.SitemapFile = true, file
|
||||
}
|
||||
res.Pages[p.Key] = p
|
||||
res.Order = append(res.Order, p.Key)
|
||||
p.Resp.Body = nil
|
||||
for _, next := range discovered(f, p) {
|
||||
push(next.key, p.Depth+1, next.via)
|
||||
}
|
||||
}
|
||||
if opt.Progress != nil {
|
||||
opt.Progress(ctx, len(res.Order))
|
||||
}
|
||||
}
|
||||
if len(queue) > 0 && !res.Canceled && len(res.Order) >= opt.MaxPages {
|
||||
res.LimitHit = true
|
||||
}
|
||||
res.Render.Pages = rendered
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// discovered lists the same-host addresses a fetched page leads to: its
|
||||
// redirect target, its links, canonical and hreflang alternates.
|
||||
func discovered(f *Fetcher, p *Page) []item {
|
||||
var out []item
|
||||
add := func(raw string, via string) {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
u.Fragment = ""
|
||||
if key := f.Key(u); key != "" {
|
||||
out = append(out, item{key: key, via: via})
|
||||
}
|
||||
}
|
||||
if p.Resp != nil && p.Resp.Location != "" {
|
||||
add(p.Resp.Location, viaRedirect)
|
||||
}
|
||||
if p.Facts == nil {
|
||||
return out
|
||||
}
|
||||
for _, l := range p.Facts.Links {
|
||||
if l.URL != nil {
|
||||
add(l.URL.String(), viaLink)
|
||||
}
|
||||
}
|
||||
if p.Facts.Canonical != "" {
|
||||
add(p.Facts.Canonical, viaCanonical)
|
||||
}
|
||||
for _, a := range p.Facts.Alternates {
|
||||
if a.URL != nil {
|
||||
add(a.URL.String(), viaAlternate)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// readSitemaps reads /sitemap.xml: an index's files (same host only) or a
|
||||
// single urlset.
|
||||
func readSitemaps(ctx context.Context, f *Fetcher, pace *pacer, res *Result) error {
|
||||
if err := pace.wait(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
r := f.Get(ctx, SitemapPath)
|
||||
if r.Err != "" || r.Status != 200 {
|
||||
res.Sitemaps = append(res.Sitemaps, SitemapFile{Key: SitemapPath, Status: r.Status})
|
||||
return nil
|
||||
}
|
||||
index, locs := parseSitemap(r.Body)
|
||||
if !index {
|
||||
res.addSitemap(f, SitemapFile{Key: SitemapPath, Status: 200}, locs)
|
||||
return nil
|
||||
}
|
||||
for i, loc := range locs {
|
||||
if i >= maxSitemapFiles {
|
||||
break
|
||||
}
|
||||
u, err := url.Parse(loc)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
key := f.Key(u)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if err := pace.wait(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
sr := f.Get(ctx, key)
|
||||
file := SitemapFile{Key: key, Status: sr.Status}
|
||||
if sr.Err != "" || sr.Status != 200 {
|
||||
res.Sitemaps = append(res.Sitemaps, file)
|
||||
continue
|
||||
}
|
||||
_, urls := parseSitemap(sr.Body)
|
||||
res.addSitemap(f, file, urls)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (res *Result) addSitemap(f *Fetcher, file SitemapFile, locs []string) {
|
||||
for _, loc := range locs {
|
||||
u, err := url.Parse(loc)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
key := f.Key(u)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := res.Listed[key]; !dup {
|
||||
res.Listed[key] = file.Key
|
||||
}
|
||||
file.URLs = append(file.URLs, key)
|
||||
}
|
||||
res.Sitemaps = append(res.Sitemaps, file)
|
||||
}
|
||||
|
||||
// parseSitemap reads the <loc> values of a sitemap document; index is true
|
||||
// for a <sitemapindex>.
|
||||
func parseSitemap(body []byte) (index bool, locs []string) {
|
||||
dec := xml.NewDecoder(io.LimitReader(bytes.NewReader(body), MaxBody))
|
||||
depth := 0
|
||||
inLoc := false
|
||||
var parent string
|
||||
var stack []string
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
switch t := tok.(type) {
|
||||
case xml.StartElement:
|
||||
depth++
|
||||
name := strings.ToLower(t.Name.Local)
|
||||
if depth == 1 && name == "sitemapindex" {
|
||||
index = true
|
||||
}
|
||||
stack = append(stack, name)
|
||||
if name == "loc" && len(stack) >= 2 {
|
||||
parent = stack[len(stack)-2]
|
||||
inLoc = parent == "url" || parent == "sitemap"
|
||||
}
|
||||
case xml.EndElement:
|
||||
depth--
|
||||
if len(stack) > 0 {
|
||||
stack = stack[:len(stack)-1]
|
||||
}
|
||||
inLoc = false
|
||||
case xml.CharData:
|
||||
if inLoc {
|
||||
if s := strings.TrimSpace(string(t)); s != "" {
|
||||
locs = append(locs, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return index, locs
|
||||
}
|
||||
191
classes/crawl/fetch.go
Normal file
191
classes/crawl/fetch.go
Normal file
@@ -0,0 +1,191 @@
|
||||
// Package crawl is the site audit's crawler (D-14): it fetches the
|
||||
// rendered site like a search engine, checks every page (UI-SPEC C23),
|
||||
// stores the findings with the CMS-data issues and names how each one is
|
||||
// fixed (P33-38).
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/compass"
|
||||
"git.golem15.com/golem15/summercms/modules/fetchguard"
|
||||
)
|
||||
|
||||
// The config keys of the crawler.
|
||||
const (
|
||||
// TrustedConfigKey lets the crawler fetch plain http and private
|
||||
// addresses (staging, local sites). Only Go config can set it.
|
||||
TrustedConfigKey = "golem15.seo.crawl_trusted"
|
||||
)
|
||||
|
||||
// The crawler's limits (D-14, RESEARCH Pattern 7).
|
||||
const (
|
||||
// MaxBody caps one answer's body.
|
||||
MaxBody = 5 << 20
|
||||
// FetchTimeout is one request's timeout.
|
||||
FetchTimeout = 10 * time.Second
|
||||
// Workers is the number of requests at a time.
|
||||
Workers = 2
|
||||
// MaxLinksPerPage caps the links stored and followed per page.
|
||||
MaxLinksPerPage = 1000
|
||||
)
|
||||
|
||||
// UserAgent is the crawler's identity (P33-41).
|
||||
func UserAgent(site string) string {
|
||||
return "SummerCMS-SEO-Crawler/1 (+" + site + ")"
|
||||
}
|
||||
|
||||
// Response is one fetched address: its status, Location, content type,
|
||||
// X-Robots-Tag and body, or the fetch error.
|
||||
type Response struct {
|
||||
Status int
|
||||
Location string
|
||||
ContentType string
|
||||
XRobots string
|
||||
Body []byte
|
||||
Err string
|
||||
}
|
||||
|
||||
// HTML reports whether the answer is an HTML page.
|
||||
func (r *Response) HTML() bool {
|
||||
ct := strings.ToLower(r.ContentType)
|
||||
return ct == "" || strings.Contains(ct, "text/html") || strings.Contains(ct, "application/xhtml")
|
||||
}
|
||||
|
||||
// Fetcher fetches the site's own addresses through a fetchguard client
|
||||
// limited to the site host and its www twin: HTTPS only, private
|
||||
// addresses refused at dial time and redirects never followed. Trusted
|
||||
// mode (golem15.seo.crawl_trusted, Go config only) allows plain http and
|
||||
// private addresses, still only on those hosts.
|
||||
type Fetcher struct {
|
||||
client *fetchguard.Client
|
||||
site *url.URL
|
||||
hosts []string
|
||||
trusted bool
|
||||
ua string
|
||||
}
|
||||
|
||||
// SiteHosts returns the host of site and its www twin.
|
||||
func SiteHosts(host string) []string {
|
||||
host = strings.ToLower(strings.TrimSpace(host))
|
||||
if host == "" {
|
||||
return nil
|
||||
}
|
||||
if bare, ok := strings.CutPrefix(host, "www."); ok {
|
||||
return []string{host, bare}
|
||||
}
|
||||
return []string{host, "www." + host}
|
||||
}
|
||||
|
||||
// Trusted reports whether config turns trusted mode on.
|
||||
func Trusted(cfg *compass.Config) bool {
|
||||
return cfg != nil && cfg.Bool(TrustedConfigKey)
|
||||
}
|
||||
|
||||
// NewFetcher builds the fetcher of site (an absolute origin).
|
||||
func NewFetcher(site string, cfg *compass.Config) (*Fetcher, error) {
|
||||
u, err := url.Parse(strings.TrimRight(site, "/"))
|
||||
if err != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") {
|
||||
return nil, ErrNoSiteURL
|
||||
}
|
||||
f := &Fetcher{site: u, hosts: SiteHosts(u.Hostname()), trusted: Trusted(cfg), ua: UserAgent(u.Scheme + "://" + u.Host)}
|
||||
policy := fetchguard.Policy{Mode: fetchguard.AllowHostsMode, AllowHosts: f.hosts, MaxBytes: MaxBody, Timeout: FetchTimeout}
|
||||
if f.trusted {
|
||||
policy.Mode = fetchguard.TrustedMode
|
||||
}
|
||||
f.client, err = fetchguard.NewClient(policy, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Hosts returns the site host and its www twin.
|
||||
func (f *Fetcher) Hosts() []string { return f.hosts }
|
||||
|
||||
// IsTrusted reports whether the fetcher runs in trusted mode.
|
||||
func (f *Fetcher) IsTrusted() bool { return f.trusted }
|
||||
|
||||
// Site returns the site origin.
|
||||
func (f *Fetcher) Site() *url.URL { return f.site }
|
||||
|
||||
// UA returns the crawler's User-Agent.
|
||||
func (f *Fetcher) UA() string { return f.ua }
|
||||
|
||||
// Allowed reports whether u may be fetched: https (http only in trusted
|
||||
// mode) on the site host or its www twin, without credentials. In trusted
|
||||
// mode the host list is checked here, since fetchguard checks none then.
|
||||
func (f *Fetcher) Allowed(u *url.URL) bool {
|
||||
if u == nil || u.User != nil {
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "https":
|
||||
case "http":
|
||||
if !f.trusted {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
for _, h := range f.hosts {
|
||||
if host == h {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Key is the address of u on the site: its path with the query, or ""
|
||||
// when u is not on the site.
|
||||
func (f *Fetcher) Key(u *url.URL) string {
|
||||
if !f.Allowed(u) {
|
||||
return ""
|
||||
}
|
||||
p := u.EscapedPath()
|
||||
if p == "" {
|
||||
p = "/"
|
||||
}
|
||||
if u.RawQuery != "" {
|
||||
p += "?" + u.RawQuery
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Absolute returns the site URL of an address key.
|
||||
func (f *Fetcher) Absolute(key string) string {
|
||||
return f.site.Scheme + "://" + f.site.Host + key
|
||||
}
|
||||
|
||||
// Get fetches one address key of the site.
|
||||
func (f *Fetcher) Get(ctx context.Context, key string) *Response {
|
||||
header := http.Header{}
|
||||
header.Set("User-Agent", f.ua)
|
||||
header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.5")
|
||||
res, err := f.client.Get(ctx, f.Absolute(key), header)
|
||||
if err != nil {
|
||||
reason := "network_error"
|
||||
if fe, ok := err.(*fetchguard.Error); ok {
|
||||
reason = string(fe.Reason)
|
||||
}
|
||||
return &Response{Err: reason}
|
||||
}
|
||||
out := &Response{
|
||||
Status: res.StatusCode,
|
||||
ContentType: res.Header.Get("Content-Type"),
|
||||
XRobots: strings.Join(res.Header.Values("X-Robots-Tag"), ", "),
|
||||
Body: res.Body,
|
||||
}
|
||||
if loc := res.Header.Get("Location"); loc != "" {
|
||||
base, _ := url.Parse(f.Absolute(key))
|
||||
if ref, err := url.Parse(loc); err == nil && base != nil {
|
||||
out.Location = base.ResolveReference(ref).String()
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
145
classes/crawl/fix.go
Normal file
145
classes/crawl/fix.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
)
|
||||
|
||||
// The SEO tools an issue may name (pier prefixes them with seo_).
|
||||
const (
|
||||
ToolSetURLSEO = "seo_set_url_seo"
|
||||
ToolUpdateRobots = "seo_update_robots"
|
||||
ToolCreateRedirect = "seo_create_redirect"
|
||||
ToolUpdatePage = "seo_update_page"
|
||||
ToolCreatePage = "seo_create_page"
|
||||
ToolUpdateSettings = "seo_update_settings"
|
||||
)
|
||||
|
||||
// Record is the CMS record behind a crawled page: its content type, id
|
||||
// and locale, the descriptor facts the checks compare the page with
|
||||
// (title, word count, headings, images, the highest heading level) and the
|
||||
// fixing tools its type names (pact.ContentDescriptor.Tools).
|
||||
type Record struct {
|
||||
Type string
|
||||
ID string
|
||||
Locale string
|
||||
Title string
|
||||
Words *int
|
||||
Headings []pact.ContentHeading
|
||||
Images []pact.ContentImage
|
||||
TopHeading int
|
||||
// Links are the record's CMS-known links: the address of its primary
|
||||
// parent, which the page's breadcrumb trail links to.
|
||||
Links []string
|
||||
Tools map[string]string
|
||||
// CustomJSONLD is true when the record's SEO tab stores custom JSON-LD.
|
||||
CustomJSONLD bool
|
||||
}
|
||||
|
||||
// tool returns the record's tool for the first operation of ops it
|
||||
// names, "" when it names none.
|
||||
func (r *Record) tool(ops ...string) string {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
for _, op := range ops {
|
||||
if t := strings.TrimSpace(r.Tools[op]); t != "" {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hasHeading reports whether the record's own headings hold one of level
|
||||
// (any level when level is 0) whose text is text (any text when empty).
|
||||
func (r *Record) hasHeading(level int, text string) bool {
|
||||
if r == nil {
|
||||
return false
|
||||
}
|
||||
for _, h := range r.Headings {
|
||||
if level != 0 && h.Level != level {
|
||||
continue
|
||||
}
|
||||
if text == "" || normal(h.Text) == normal(text) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Finding is one problem a check found on a page, before it is stored:
|
||||
// the kind, the address and locale, the details of the kind, and how it
|
||||
// is fixed (P33-38). Tool is a fixed CMS tool (seo_*); Content lists the
|
||||
// record operations (pact.ContentTool*) that fix a finding inside the
|
||||
// record's own content, tried in order; What names the template element
|
||||
// when it is fixed in code.
|
||||
type Finding struct {
|
||||
Kind string
|
||||
URL string
|
||||
Locale string
|
||||
Details map[string]any
|
||||
Tool string
|
||||
Content []string
|
||||
Record *Record
|
||||
What string
|
||||
// Check tells the two content_js findings apart (cms, rendered).
|
||||
Check string
|
||||
}
|
||||
|
||||
// Route applies P33-38: a finding inside the record's own content names
|
||||
// the record type's fitting tool (H1 findings prefer Tools[headings],
|
||||
// other content findings Tools[edit]); else a fixed CMS tool; else fix
|
||||
// code with the template element in details.what. Every finding gets one
|
||||
// of the two.
|
||||
func Route(f Finding) (fix, tool, what string) {
|
||||
if len(f.Content) > 0 {
|
||||
if t := f.Record.tool(f.Content...); t != "" {
|
||||
return models.FixTool, t, ""
|
||||
}
|
||||
}
|
||||
if f.Tool != "" {
|
||||
return models.FixTool, f.Tool, ""
|
||||
}
|
||||
what = f.What
|
||||
if what == "" {
|
||||
what = defaultWhat[f.Kind]
|
||||
}
|
||||
if what == "" {
|
||||
what = "the page template"
|
||||
}
|
||||
return models.FixCode, "", what
|
||||
}
|
||||
|
||||
// defaultWhat names the template element of a kind fixed in code.
|
||||
var defaultWhat = map[string]string{
|
||||
KindCanonicalMissing: `<link rel="canonical"> in the page head`,
|
||||
KindH1Missing: "the <h1> of the page template",
|
||||
KindH1Multiple: "the <h1> headings of the page template",
|
||||
KindHeadingSkipped: "the heading levels of the page template",
|
||||
KindHeadingEmpty: "the empty heading of the page template",
|
||||
KindJSONLDInvalid: `the <script type="application/ld+json"> of the page template`,
|
||||
KindJSONLDMultiple: `the <script type="application/ld+json"> blocks of the page template`,
|
||||
KindHreflangRecip: `the <link rel="alternate" hreflang> links of the page template`,
|
||||
KindImageAlt: "the alt attribute of the template's <img>",
|
||||
KindImageSize: "the width and height attributes of the template's <img>",
|
||||
KindFeaturedLazy: `loading="lazy" on the main image`,
|
||||
KindContentJS: "server-side rendering of the page content",
|
||||
KindGenericAnchor: "the link text in the page template",
|
||||
KindRedirectedLink: "the link in the page template",
|
||||
KindOrphanPage: "a link to this page from the site navigation or another page",
|
||||
}
|
||||
|
||||
// Fingerprint identifies a finding across crawls: the kind, the address,
|
||||
// the locale and the check (content_js), so an ignore survives later
|
||||
// crawls (P33-37).
|
||||
func Fingerprint(kind, url, locale, check string) string {
|
||||
sum := sha256.Sum256([]byte(kind + "\x00" + url + "\x00" + locale + "\x00" + check))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
// normal is text compared without case and extra whitespace.
|
||||
func normal(s string) string { return strings.ToLower(strings.Join(strings.Fields(s), " ")) }
|
||||
361
classes/crawl/job.go
Normal file
361
classes/crawl/job.go
Normal file
@@ -0,0 +1,361 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes/schema"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/conga"
|
||||
"git.golem15.com/golem15/summercms/modules/diary"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// CrawlKind is the River kind of the crawl job.
|
||||
const CrawlKind = "golem15.seo.crawl"
|
||||
|
||||
// CrawlArgs are the arguments of the crawl job: the crawl row it runs.
|
||||
type CrawlArgs struct {
|
||||
CrawlID uint `json:"crawl_id"`
|
||||
}
|
||||
|
||||
// Kind is CrawlKind.
|
||||
func (CrawlArgs) Kind() string { return CrawlKind }
|
||||
|
||||
// The crawl limits used until the Site audit settings say otherwise.
|
||||
const (
|
||||
DefaultMaxPages = 5000
|
||||
DefaultDelay = 250 * time.Millisecond
|
||||
)
|
||||
|
||||
// AlreadyRunningError refuses a start while a crawl is queued or running
|
||||
// (P33-39): one crawl runs at a time.
|
||||
type AlreadyRunningError struct {
|
||||
Crawl models.Crawl
|
||||
}
|
||||
|
||||
func (e *AlreadyRunningError) Error() string { return "golem15.seo: a crawl is already running" }
|
||||
|
||||
// Since is the time the running crawl started, else its creation time.
|
||||
func (e *AlreadyRunningError) Since() time.Time {
|
||||
if e.Crawl.StartedAt != nil {
|
||||
return *e.Crawl.StartedAt
|
||||
}
|
||||
if e.Crawl.CreatedAt != nil {
|
||||
return *e.Crawl.CreatedAt
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// Start queues a crawl (D-14, P33-39) inside tx: the settings row is
|
||||
// locked so two starts cannot both pass the check that no crawl is queued
|
||||
// or running, then the crawl row is written and the conga job dispatched
|
||||
// in the same transaction. A missing site address refuses with
|
||||
// ErrNoSiteURL, a running crawl with *AlreadyRunningError.
|
||||
func Start(ctx context.Context, svc *classes.Service, tx *gorm.DB, trigger, name string) (models.Crawl, error) {
|
||||
site := svc.SiteURL()
|
||||
if site == "" {
|
||||
return models.Crawl{}, ErrNoSiteURL
|
||||
}
|
||||
if _, err := classes.SettingsRow(tx); err != nil {
|
||||
return models.Crawl{}, err
|
||||
}
|
||||
if err := tx.Exec(`SELECT id FROM golem15_seo_settings WHERE id = 1 FOR UPDATE`).Error; err != nil {
|
||||
return models.Crawl{}, err
|
||||
}
|
||||
active, err := ActiveCrawl(tx)
|
||||
if err != nil {
|
||||
return models.Crawl{}, err
|
||||
}
|
||||
if active != nil {
|
||||
return models.Crawl{}, &AlreadyRunningError{Crawl: *active}
|
||||
}
|
||||
limits, err := LoadLimits(tx)
|
||||
if err != nil {
|
||||
return models.Crawl{}, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
row := models.Crawl{
|
||||
Status: models.CrawlQueued, Trigger: trigger, TriggerName: name, Site: site,
|
||||
MaxPages: limits.MaxPages, RenderStatus: models.RenderSkipped, CreatedAt: &now,
|
||||
}
|
||||
if err := tx.Create(&row).Error; err != nil {
|
||||
return models.Crawl{}, err
|
||||
}
|
||||
m, err := conga.From(svc.App())
|
||||
if err != nil {
|
||||
return models.Crawl{}, err
|
||||
}
|
||||
id, err := m.Dispatch(ctx, tx, CrawlArgs{CrawlID: row.ID}, conga.DispatchOpts{Label: svc.Phrase("golem15.seo::lang.jobs.crawl", nil)})
|
||||
if err != nil {
|
||||
return models.Crawl{}, err
|
||||
}
|
||||
row.JobID = &id
|
||||
if err := tx.Model(&models.Crawl{}).Where("id = ?", row.ID).Update("job_id", id).Error; err != nil {
|
||||
return models.Crawl{}, err
|
||||
}
|
||||
return row, nil
|
||||
}
|
||||
|
||||
// Limits are a crawl's politeness limits.
|
||||
type Limits struct {
|
||||
MaxPages int
|
||||
Delay time.Duration
|
||||
}
|
||||
|
||||
// LoadLimits reads the crawl limits.
|
||||
func LoadLimits(db *gorm.DB) (Limits, error) {
|
||||
return Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay}, nil
|
||||
}
|
||||
|
||||
// LoadCrawl reads one crawl.
|
||||
func LoadCrawl(db *gorm.DB, id uint) (models.Crawl, error) {
|
||||
var rows []models.Crawl
|
||||
if err := db.Where("id = ?", id).Limit(1).Find(&rows).Error; err != nil {
|
||||
return models.Crawl{}, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return models.Crawl{}, gorm.ErrRecordNotFound
|
||||
}
|
||||
return rows[0], nil
|
||||
}
|
||||
|
||||
// Run runs one queued crawl: it fetches the site, checks every page,
|
||||
// stores the pages, links and findings with the CMS-data issues, and marks
|
||||
// the crawl finished, cancelled (pages and issues fetched so far are kept)
|
||||
// or failed (with the error; what was fetched before the failure is kept).
|
||||
// A crawl that is no longer queued is left alone.
|
||||
func Run(ctx context.Context, svc *classes.Service, id uint) error {
|
||||
gdb, err := svc.GormDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
db := gdb.WithContext(ctx)
|
||||
row, err := LoadCrawl(db, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if row.Status != models.CrawlQueued {
|
||||
return nil
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
moved := db.Model(&models.Crawl{}).Where("id = ? AND status = ?", id, models.CrawlQueued).
|
||||
Updates(map[string]any{"status": models.CrawlRunning, "started_at": now})
|
||||
if moved.Error != nil {
|
||||
return moved.Error
|
||||
}
|
||||
if moved.RowsAffected == 0 {
|
||||
return nil
|
||||
}
|
||||
res, runErr := run(ctx, svc, db, row)
|
||||
return finish(ctx, svc, db, row, res, runErr)
|
||||
}
|
||||
|
||||
// run fetches and checks the site of one crawl.
|
||||
func run(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.Crawl) (*Result, error) {
|
||||
f, err := NewFetcher(row.Site, svc.Config())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
limits, err := LoadLimits(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if row.MaxPages > 0 {
|
||||
limits.MaxPages = row.MaxPages
|
||||
}
|
||||
var m *conga.Manager
|
||||
if app := svc.App(); app != nil {
|
||||
m, _ = conga.From(app)
|
||||
}
|
||||
opt := Options{
|
||||
MaxPages: limits.MaxPages,
|
||||
Delay: limits.Delay,
|
||||
Renderer: rendererFrom(ctx),
|
||||
Canceled: func(ctx context.Context) bool {
|
||||
var status string
|
||||
if err := db.Raw(`SELECT status FROM golem15_seo_crawls WHERE id = ?`, row.ID).Scan(&status).Error; err == nil && status == models.CrawlCancelled {
|
||||
return true
|
||||
}
|
||||
if m != nil && row.JobID != nil {
|
||||
if c, err := m.CheckIfCanceled(ctx, *row.JobID); err == nil && c {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
},
|
||||
Progress: func(ctx context.Context, n int) {
|
||||
_ = db.Model(&models.Crawl{}).Where("id = ?", row.ID).Update("pages", n).Error
|
||||
if m != nil && row.JobID != nil {
|
||||
_ = m.UpdateJobState(ctx, *row.JobID, n, nil)
|
||||
}
|
||||
},
|
||||
}
|
||||
if m != nil && row.JobID != nil {
|
||||
_ = m.StartJob(ctx, *row.JobID, limits.MaxPages)
|
||||
}
|
||||
return Crawl(ctx, f, opt)
|
||||
}
|
||||
|
||||
// finish stores what a crawl found and writes its final state.
|
||||
func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.Crawl, res *Result, runErr error) error {
|
||||
status := models.CrawlFinished
|
||||
errText := ""
|
||||
if runErr != nil {
|
||||
status, errText = models.CrawlFailed, runErr.Error()
|
||||
}
|
||||
var current string
|
||||
_ = db.Raw(`SELECT status FROM golem15_seo_crawls WHERE id = ?`, row.ID).Scan(¤t).Error
|
||||
if current == models.CrawlCancelled || (res != nil && res.Canceled) {
|
||||
status = models.CrawlCancelled
|
||||
}
|
||||
store := func(ctx context.Context, tx *gorm.DB) error {
|
||||
if res == nil {
|
||||
return nil
|
||||
}
|
||||
if err := SavePages(tx, row.ID, res); err != nil {
|
||||
return err
|
||||
}
|
||||
findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx)})
|
||||
cms, err := svc.Issues(classes.WithTx(ctx, tx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
id := row.ID
|
||||
return SaveIssues(tx, &id, findings, cms)
|
||||
}
|
||||
if err := lagoon.Transaction(ctx, db, store); err != nil {
|
||||
status, errText = models.CrawlFailed, err.Error()
|
||||
}
|
||||
updates := map[string]any{"status": status, "error": errText, "finished_at": time.Now().UTC()}
|
||||
if res != nil {
|
||||
updates["pages"] = len(res.Order)
|
||||
updates["limit_hit"] = res.LimitHit
|
||||
updates["render_status"] = res.Render.Status
|
||||
updates["render_pages"] = res.Render.Pages
|
||||
updates["render_note"] = res.Render.Note
|
||||
}
|
||||
if err := db.Model(&models.Crawl{}).Where("id = ?", row.ID).Updates(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if m, err := conga.From(svc.App()); err == nil && row.JobID != nil {
|
||||
meta := map[string]any{"crawl_id": row.ID, "status": status}
|
||||
switch status {
|
||||
case models.CrawlFailed:
|
||||
_ = m.FailJob(ctx, *row.JobID, meta)
|
||||
case models.CrawlCancelled:
|
||||
_ = m.StopJob(ctx, *row.JobID, meta)
|
||||
default:
|
||||
_ = m.CompleteJob(ctx, *row.JobID, meta)
|
||||
}
|
||||
}
|
||||
if status == models.CrawlFailed {
|
||||
pluginLog.Warn("golem15.seo: crawl failed", "crawl", row.ID, "error", errText)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Lookup returns the CMS record behind a crawled address (cached per
|
||||
// address): the content type whose Match answers the path exactly, its
|
||||
// descriptor facts in the matched locale, its primary parent's address and
|
||||
// whether its SEO tab stores custom JSON-LD.
|
||||
func Lookup(ctx context.Context, svc *classes.Service, db *gorm.DB) func(key string) *Record {
|
||||
cache := map[string]*Record{}
|
||||
reg := svc.Registry()
|
||||
unprefixed := svc.UnprefixedLocale()
|
||||
return func(key string) *Record {
|
||||
if reg == nil {
|
||||
return nil
|
||||
}
|
||||
path, _, _ := strings.Cut(key, "?")
|
||||
if r, ok := cache[path]; ok {
|
||||
return r
|
||||
}
|
||||
cache[path] = nil
|
||||
typeKey, match, err := reg.MatchPath(ctx, path, unprefixed)
|
||||
if err != nil || match == nil || !match.Exact {
|
||||
return nil
|
||||
}
|
||||
typ, ok := reg.ContentType(typeKey)
|
||||
if !ok || typ.Content == nil {
|
||||
return nil
|
||||
}
|
||||
rec, err := reg.Describe(ctx, typeKey, match.ID, unprefixed)
|
||||
if err != nil || rec == nil {
|
||||
return nil
|
||||
}
|
||||
out := &Record{Type: typeKey, ID: match.ID, Locale: match.Locale, TopHeading: rec.TopHeading, Tools: typ.Content.Tools}
|
||||
var loc *pact.ContentLocale
|
||||
for i := range rec.Locales {
|
||||
if rec.Locales[i].Locale == match.Locale {
|
||||
loc = &rec.Locales[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if loc == nil && len(rec.Locales) > 0 {
|
||||
loc = &rec.Locales[0]
|
||||
}
|
||||
if loc != nil {
|
||||
out.Locale = loc.Locale
|
||||
out.Title, out.Words, out.Headings, out.Images = loc.Title, loc.Words, loc.Headings, loc.Images
|
||||
}
|
||||
if len(rec.Parents) > 0 {
|
||||
if parent, err := reg.Describe(ctx, rec.Parents[0].Type, rec.Parents[0].ID, unprefixed); err == nil && parent != nil {
|
||||
for _, pl := range parent.Locales {
|
||||
if pl.Locale == out.Locale && pl.URL != "" {
|
||||
out.Links = append(out.Links, pl.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if entry, found, err := classes.LoadEntry(db, typ.Content.Model, match.ID, out.Locale); err == nil && found && entry.Data != nil {
|
||||
if raw, _ := entry.Data["schema"].(string); raw != "" {
|
||||
if v, err := schema.ParseValue(raw); err == nil && strings.TrimSpace(v.Custom) != "" {
|
||||
out.CustomJSONLD = true
|
||||
}
|
||||
}
|
||||
}
|
||||
cache[path] = out
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
// errCancelledCrawl refuses cancelling a crawl that is not queued or
|
||||
// running.
|
||||
var errCancelledCrawl = errors.New("golem15.seo: the crawl is not running")
|
||||
|
||||
// Cancel stops a queued or running crawl inside tx: the row is marked
|
||||
// cancelled by name (the crawl keeps what it fetched) and its conga job
|
||||
// cancelled.
|
||||
func Cancel(ctx context.Context, svc *classes.Service, tx *gorm.DB, row models.Crawl, name string) error {
|
||||
if !row.Active() {
|
||||
return errCancelledCrawl
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
updates := map[string]any{"status": models.CrawlCancelled, "cancelled_by": name}
|
||||
if row.Status == models.CrawlQueued {
|
||||
updates["finished_at"] = now
|
||||
}
|
||||
if err := tx.Model(&models.Crawl{}).Where("id = ?", row.ID).Updates(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if row.JobID != nil {
|
||||
if m, err := conga.From(svc.App()); err == nil {
|
||||
if err := m.CancelJob(ctx, *row.JobID); err != nil {
|
||||
pluginLog.Warn("golem15.seo: cancel crawl job", "crawl", row.ID, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrNotRunning is a cancel of a crawl that is not queued or running.
|
||||
var ErrNotRunning = errCancelledCrawl
|
||||
|
||||
// pluginLog is the crawler's logger (component seo).
|
||||
var pluginLog = diary.For("seo")
|
||||
293
classes/crawl/parse.go
Normal file
293
classes/crawl/parse.go
Normal file
@@ -0,0 +1,293 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/net/html/atom"
|
||||
)
|
||||
|
||||
// Heading is one heading of a page in document order.
|
||||
type Heading struct {
|
||||
Level int `json:"level"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// Link is one <a href> of a page, resolved against the page's address.
|
||||
type Link struct {
|
||||
URL *url.URL
|
||||
Href string
|
||||
Text string
|
||||
Rel string
|
||||
}
|
||||
|
||||
// Image is one <img> of a page.
|
||||
type Image struct {
|
||||
Src string
|
||||
Alt string
|
||||
HasAlt bool
|
||||
Width string
|
||||
Height string
|
||||
Loading string
|
||||
}
|
||||
|
||||
// Alternate is one <link rel="alternate" hreflang>.
|
||||
type Alternate struct {
|
||||
Lang string
|
||||
URL *url.URL
|
||||
Href string
|
||||
}
|
||||
|
||||
// PageFacts is what one HTML document says, parsed with
|
||||
// golang.org/x/net/html (no regular expressions over HTML): the language,
|
||||
// title, meta description, canonical, meta robots, og:image, headings,
|
||||
// links, images, JSON-LD scripts, hreflang alternates and the main text
|
||||
// with its word count. The rendered comparison parses the JavaScript
|
||||
// rendered DOM into the same facts.
|
||||
type PageFacts struct {
|
||||
Lang string
|
||||
Title string
|
||||
HasTitle bool
|
||||
Description string
|
||||
HasDescription bool
|
||||
Canonical string
|
||||
Robots string
|
||||
OGImage string
|
||||
Headings []Heading
|
||||
Links []Link
|
||||
Images []Image
|
||||
JSONLD []string
|
||||
Alternates []Alternate
|
||||
Text string
|
||||
Words int
|
||||
}
|
||||
|
||||
// H1 returns the texts of the page's level-1 headings.
|
||||
func (p *PageFacts) H1() []string {
|
||||
var out []string
|
||||
for _, h := range p.Headings {
|
||||
if h.Level == 1 {
|
||||
out = append(out, h.Text)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Noindex reports whether the meta robots value says noindex.
|
||||
func (p *PageFacts) Noindex() bool { return hasNoindex(p.Robots) }
|
||||
|
||||
// hasNoindex reports whether a robots value contains noindex or none.
|
||||
func hasNoindex(v string) bool {
|
||||
for _, part := range strings.FieldsFunc(strings.ToLower(v), func(r rune) bool { return r == ',' || r == ' ' || r == ';' }) {
|
||||
if part == "noindex" || part == "none" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Parse reads the facts of an HTML document fetched from base.
|
||||
func Parse(base *url.URL, body []byte) *PageFacts {
|
||||
out := &PageFacts{}
|
||||
doc, err := html.Parse(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
resolve := func(href string) *url.URL {
|
||||
href = strings.TrimSpace(href)
|
||||
if href == "" {
|
||||
return nil
|
||||
}
|
||||
ref, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if base == nil {
|
||||
return ref
|
||||
}
|
||||
u := base.ResolveReference(ref)
|
||||
u.Fragment = ""
|
||||
return u
|
||||
}
|
||||
var main *html.Node
|
||||
var bodyNode *html.Node
|
||||
var walk func(n *html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.DataAtom {
|
||||
case atom.Html:
|
||||
out.Lang = strings.TrimSpace(attr(n, "lang"))
|
||||
case atom.Title:
|
||||
if !out.HasTitle {
|
||||
out.HasTitle = true
|
||||
out.Title = collapse(textOf(n))
|
||||
}
|
||||
case atom.Meta:
|
||||
name := strings.ToLower(attr(n, "name"))
|
||||
prop := strings.ToLower(attr(n, "property"))
|
||||
switch {
|
||||
case name == "description" && !out.HasDescription:
|
||||
out.HasDescription = true
|
||||
out.Description = collapse(attr(n, "content"))
|
||||
case name == "robots" || name == "googlebot":
|
||||
if v := strings.TrimSpace(attr(n, "content")); v != "" {
|
||||
if out.Robots != "" {
|
||||
out.Robots += ", "
|
||||
}
|
||||
out.Robots += v
|
||||
}
|
||||
case prop == "og:image" && out.OGImage == "":
|
||||
if u := resolve(attr(n, "content")); u != nil {
|
||||
out.OGImage = u.String()
|
||||
}
|
||||
}
|
||||
case atom.Link:
|
||||
rel := strings.ToLower(attr(n, "rel"))
|
||||
switch {
|
||||
case relHas(rel, "canonical") && out.Canonical == "":
|
||||
if u := resolve(attr(n, "href")); u != nil {
|
||||
out.Canonical = u.String()
|
||||
}
|
||||
case relHas(rel, "alternate") && hasAttr(n, "hreflang"):
|
||||
href := attr(n, "href")
|
||||
out.Alternates = append(out.Alternates, Alternate{Lang: strings.TrimSpace(attr(n, "hreflang")), URL: resolve(href), Href: href})
|
||||
}
|
||||
case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6:
|
||||
out.Headings = append(out.Headings, Heading{Level: int(n.Data[1] - '0'), Text: collapse(textOf(n))})
|
||||
case atom.A:
|
||||
if hasAttr(n, "href") && len(out.Links) < MaxLinksPerPage {
|
||||
href := attr(n, "href")
|
||||
low := strings.ToLower(strings.TrimSpace(href))
|
||||
if !strings.HasPrefix(low, "javascript:") && !strings.HasPrefix(low, "mailto:") && !strings.HasPrefix(low, "tel:") && !strings.HasPrefix(low, "#") {
|
||||
if u := resolve(href); u != nil {
|
||||
out.Links = append(out.Links, Link{URL: u, Href: href, Text: collapse(textOf(n)), Rel: strings.ToLower(attr(n, "rel"))})
|
||||
}
|
||||
}
|
||||
}
|
||||
case atom.Img:
|
||||
alt, hasAlt := attrOK(n, "alt")
|
||||
src := attr(n, "src")
|
||||
if u := resolve(src); u != nil {
|
||||
src = u.String()
|
||||
}
|
||||
out.Images = append(out.Images, Image{Src: src, Alt: alt, HasAlt: hasAlt, Width: attr(n, "width"), Height: attr(n, "height"), Loading: strings.ToLower(attr(n, "loading"))})
|
||||
case atom.Script:
|
||||
if strings.EqualFold(strings.TrimSpace(attr(n, "type")), "application/ld+json") {
|
||||
out.JSONLD = append(out.JSONLD, textOf(n))
|
||||
}
|
||||
return
|
||||
case atom.Main:
|
||||
if main == nil {
|
||||
main = n
|
||||
}
|
||||
case atom.Body:
|
||||
bodyNode = n
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(doc)
|
||||
root := main
|
||||
if root == nil {
|
||||
root = bodyNode
|
||||
}
|
||||
if root != nil {
|
||||
var b strings.Builder
|
||||
visibleText(root, &b)
|
||||
out.Text = collapse(b.String())
|
||||
if len(out.Text) > maxText {
|
||||
out.Text = truncateUTF8(out.Text, maxText)
|
||||
}
|
||||
out.Words = len(strings.Fields(out.Text))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// maxText caps the main text kept per page (characters of UTF-8, bytes
|
||||
// bound).
|
||||
const maxText = 100_000
|
||||
|
||||
func truncateUTF8(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
for n > 0 && n < len(s) && (s[n]&0xC0) == 0x80 {
|
||||
n--
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
// visibleText writes the text of n without script, style, noscript and
|
||||
// template content, one space between block parts.
|
||||
func visibleText(n *html.Node, b *strings.Builder) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.DataAtom {
|
||||
case atom.Script, atom.Style, atom.Noscript, atom.Template, atom.Svg:
|
||||
return
|
||||
}
|
||||
}
|
||||
if n.Type == html.TextNode {
|
||||
b.WriteString(n.Data)
|
||||
b.WriteByte(' ')
|
||||
return
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
visibleText(c, b)
|
||||
}
|
||||
}
|
||||
|
||||
func textOf(n *html.Node) string {
|
||||
var b strings.Builder
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.TextNode {
|
||||
b.WriteString(n.Data)
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
if n.Type == html.ElementNode && n.DataAtom == atom.Img {
|
||||
if alt := attr(n, "alt"); alt != "" {
|
||||
b.WriteString(alt)
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
func collapse(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
|
||||
func attrOK(n *html.Node, key string) (string, bool) {
|
||||
for _, a := range n.Attr {
|
||||
if strings.EqualFold(a.Key, key) {
|
||||
return a.Val, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func attr(n *html.Node, key string) string {
|
||||
v, _ := attrOK(n, key)
|
||||
return v
|
||||
}
|
||||
|
||||
func hasAttr(n *html.Node, key string) bool {
|
||||
_, ok := attrOK(n, key)
|
||||
return ok
|
||||
}
|
||||
|
||||
func relHas(rel, want string) bool {
|
||||
for _, part := range strings.Fields(rel) {
|
||||
if part == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
43
classes/crawl/render.go
Normal file
43
classes/crawl/render.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package crawl
|
||||
|
||||
import "context"
|
||||
|
||||
// Renderer renders a page of the site in a browser for the raw versus
|
||||
// rendered comparison (D-70, P33-55): Render answers the page's DOM after
|
||||
// JavaScript ran, Close releases the browser. A crawl without a Renderer
|
||||
// records the comparison as skipped; a Render error records it as failed
|
||||
// with the error and never fails the crawl.
|
||||
type Renderer interface {
|
||||
Render(ctx context.Context, pageURL string) (string, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
// DefaultRenderMax is the number of pages rendered per crawl when
|
||||
// golem15.seo.crawl_render_max is unset.
|
||||
const DefaultRenderMax = 200
|
||||
|
||||
// The render states of a crawl (models.RenderSkipped and the others).
|
||||
const (
|
||||
renderStatusSkipped = "skipped"
|
||||
renderStatusDone = "done"
|
||||
renderStatusFailed = "failed"
|
||||
)
|
||||
|
||||
type rendererKey struct{}
|
||||
|
||||
// WithRenderer returns a context whose crawls use r instead of the
|
||||
// renderer built from config. It is the test seam of the comparison,
|
||||
// settable only from Go code (like fetchguard.WithTransport): no config
|
||||
// key, setting, request or MCP call reaches it.
|
||||
func WithRenderer(ctx context.Context, r Renderer) context.Context {
|
||||
return context.WithValue(ctx, rendererKey{}, r)
|
||||
}
|
||||
|
||||
// rendererFrom returns the renderer of ctx, nil when none.
|
||||
func rendererFrom(ctx context.Context) Renderer {
|
||||
if ctx == nil {
|
||||
return nil
|
||||
}
|
||||
r, _ := ctx.Value(rendererKey{}).(Renderer)
|
||||
return r
|
||||
}
|
||||
248
classes/crawl/store.go
Normal file
248
classes/crawl/store.go
Normal file
@@ -0,0 +1,248 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// insertBatch is the rows written per INSERT.
|
||||
const insertBatch = 200
|
||||
|
||||
// SavePages writes the pages and links one crawl fetched. The crawler
|
||||
// writes only its own tables (D-14).
|
||||
func SavePages(tx *gorm.DB, crawlID uint, res *Result) error {
|
||||
if res == nil {
|
||||
return nil
|
||||
}
|
||||
in := map[string]int{}
|
||||
var links []models.CrawlLink
|
||||
for _, key := range res.Order {
|
||||
p := res.Pages[key]
|
||||
if p.Facts == nil {
|
||||
continue
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, l := range p.Facts.Links {
|
||||
to := ""
|
||||
internal := false
|
||||
if res.fetcher != nil {
|
||||
to = res.fetcher.Key(l.URL)
|
||||
internal = to != ""
|
||||
}
|
||||
if !internal {
|
||||
to = l.URL.String()
|
||||
}
|
||||
links = append(links, models.CrawlLink{CrawlID: crawlID, FromURL: key, ToURL: to, Text: truncateUTF8(l.Text, 500), Rel: l.Rel, Internal: internal})
|
||||
if internal && to != key && !seen[to] {
|
||||
seen[to] = true
|
||||
in[to]++
|
||||
}
|
||||
}
|
||||
}
|
||||
pages := make([]models.CrawlPage, 0, len(res.Order))
|
||||
for _, key := range res.Order {
|
||||
p := res.Pages[key]
|
||||
row := models.CrawlPage{CrawlID: crawlID, URL: key, Depth: p.Depth, InSitemap: p.InSitemap, SitemapFile: p.SitemapFile, LinksIn: in[key]}
|
||||
if !p.FetchedAt.IsZero() {
|
||||
t := p.FetchedAt
|
||||
row.FetchedAt = &t
|
||||
}
|
||||
if r := p.Resp; r != nil {
|
||||
row.Status, row.Location, row.ContentType, row.XRobots = r.Status, r.Location, truncateUTF8(r.ContentType, 200), truncateUTF8(r.XRobots, 500)
|
||||
}
|
||||
if f := p.Facts; f != nil {
|
||||
row.Title, row.Description, row.Canonical, row.Robots = truncateUTF8(f.Title, 1000), truncateUTF8(f.Description, 2000), truncateUTF8(f.Canonical, 2048), truncateUTF8(f.Robots, 500)
|
||||
row.H1 = lagoon.Jsonable[[]string]{Data: f.H1(), Valid: true}
|
||||
hs := make([]models.CrawlHeading, 0, len(f.Headings))
|
||||
for _, h := range f.Headings {
|
||||
hs = append(hs, models.CrawlHeading{Level: h.Level, Text: truncateUTF8(h.Text, 500)})
|
||||
}
|
||||
row.Headings = lagoon.Jsonable[[]models.CrawlHeading]{Data: hs, Valid: true}
|
||||
row.Words, row.Text = f.Words, truncateUTF8(f.Text, models.MaxPageText)
|
||||
row.LinksOut = len(f.Links)
|
||||
}
|
||||
pages = append(pages, row)
|
||||
}
|
||||
if len(pages) > 0 {
|
||||
if err := tx.CreateInBatches(pages, insertBatch).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(links) > 0 {
|
||||
if err := tx.CreateInBatches(links, insertBatch).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// issueRow turns a finding into its stored row.
|
||||
func issueRow(crawlID *uint, source string, f Finding, now time.Time) models.Issue {
|
||||
fix, tool, what := Route(f)
|
||||
details := map[string]any{}
|
||||
for k, v := range f.Details {
|
||||
details[k] = v
|
||||
}
|
||||
if what != "" {
|
||||
details["what"] = what
|
||||
}
|
||||
if f.Check != "" {
|
||||
details["check"] = f.Check
|
||||
}
|
||||
sev := Severity(f.Kind)
|
||||
t := now
|
||||
return models.Issue{
|
||||
CrawlID: crawlID,
|
||||
Source: source,
|
||||
Kind: f.Kind,
|
||||
Severity: sev,
|
||||
Rank: models.SeverityRank(sev),
|
||||
URL: truncateUTF8(f.URL, 2048),
|
||||
Locale: f.Locale,
|
||||
Details: lagoon.Jsonable[map[string]any]{Data: details, Valid: true},
|
||||
Tool: tool,
|
||||
Fix: fix,
|
||||
Fingerprint: Fingerprint(f.Kind, f.URL, f.Locale, f.Check),
|
||||
FoundAt: &t,
|
||||
}
|
||||
}
|
||||
|
||||
// cmsFinding turns a CMS-data issue of the classes report into a finding:
|
||||
// its tool is the one the report names; a CMS issue without one is fixed
|
||||
// in the record's admin form.
|
||||
func cmsFinding(issue classes.Issue) Finding {
|
||||
f := Finding{Kind: issue.Kind, URL: issue.URL, Locale: issue.Locale, Details: issue.Details, Tool: issue.Tool}
|
||||
if f.Tool == "" {
|
||||
f.What = "the record in its admin form"
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// SaveIssues stores a crawl's findings and the CMS-data issues with it
|
||||
// (crawlID nil: the CMS issues of a site not crawled yet). A CMS issue
|
||||
// with the fingerprint of a crawl finding is the same problem and is not
|
||||
// stored twice.
|
||||
func SaveIssues(tx *gorm.DB, crawlID *uint, findings []Finding, cms []classes.Issue) error {
|
||||
now := time.Now().UTC()
|
||||
seen := map[string]bool{}
|
||||
var rows []models.Issue
|
||||
for _, f := range findings {
|
||||
row := issueRow(crawlID, models.SourceCrawl, f, now)
|
||||
if seen[row.Fingerprint] {
|
||||
continue
|
||||
}
|
||||
seen[row.Fingerprint] = true
|
||||
rows = append(rows, row)
|
||||
}
|
||||
for _, issue := range cms {
|
||||
row := issueRow(crawlID, models.SourceCMS, cmsFinding(issue), now)
|
||||
if seen[row.Fingerprint] {
|
||||
continue
|
||||
}
|
||||
seen[row.Fingerprint] = true
|
||||
rows = append(rows, row)
|
||||
}
|
||||
sort.SliceStable(rows, func(i, j int) bool {
|
||||
if rows[i].Rank != rows[j].Rank {
|
||||
return rows[i].Rank < rows[j].Rank
|
||||
}
|
||||
if rows[i].Kind != rows[j].Kind {
|
||||
return rows[i].Kind < rows[j].Kind
|
||||
}
|
||||
return rows[i].URL < rows[j].URL
|
||||
})
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
return tx.CreateInBatches(rows, insertBatch).Error
|
||||
}
|
||||
|
||||
// LatestCrawl returns the newest crawl that is no longer queued or
|
||||
// running, nil when there is none: its issues are the audit's current
|
||||
// ones.
|
||||
func LatestCrawl(db *gorm.DB) (*models.Crawl, error) {
|
||||
var rows []models.Crawl
|
||||
if err := db.Where("status NOT IN ?", []string{models.CrawlQueued, models.CrawlRunning}).Order("id DESC").Limit(1).Find(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &rows[0], nil
|
||||
}
|
||||
|
||||
// ActiveCrawl returns the queued or running crawl, nil when none.
|
||||
func ActiveCrawl(db *gorm.DB) (*models.Crawl, error) {
|
||||
var rows []models.Crawl
|
||||
if err := db.Where("status IN ?", []string{models.CrawlQueued, models.CrawlRunning}).Order("id DESC").Limit(1).Find(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &rows[0], nil
|
||||
}
|
||||
|
||||
// CurrentScope narrows an issues query to the audit's current issues: the
|
||||
// newest finished, failed or cancelled crawl's, or before any crawl the
|
||||
// stored CMS-data issues.
|
||||
func CurrentScope(db *gorm.DB, latest *models.Crawl) *gorm.DB {
|
||||
if latest == nil {
|
||||
return db.Where("golem15_seo_issues.crawl_id IS NULL")
|
||||
}
|
||||
return db.Where("golem15_seo_issues.crawl_id = ?", latest.ID)
|
||||
}
|
||||
|
||||
// IgnoredSet returns which of the fingerprints are ignored.
|
||||
func IgnoredSet(db *gorm.DB, fingerprints []string) (map[string]bool, error) {
|
||||
out := map[string]bool{}
|
||||
if len(fingerprints) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
var rows []models.IssueIgnore
|
||||
for start := 0; start < len(fingerprints); start += 1000 {
|
||||
end := min(start+1000, len(fingerprints))
|
||||
var part []models.IssueIgnore
|
||||
if err := db.Where("fingerprint IN ?", fingerprints[start:end]).Find(&part).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows = append(rows, part...)
|
||||
}
|
||||
for _, r := range rows {
|
||||
out[r.Fingerprint] = true
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Current returns the crawl whose issues are the audit's current ones (nil
|
||||
// before any crawl). Before any crawl, the CMS-data checks run now and
|
||||
// their issues replace the stored ones, so the list shows the CMS data as
|
||||
// it is (UI-SPEC C23: when the list is opened with no crawl yet).
|
||||
func Current(ctx context.Context, svc *classes.Service, db *gorm.DB) (*models.Crawl, error) {
|
||||
latest, err := LatestCrawl(db)
|
||||
if err != nil || latest != nil {
|
||||
return latest, err
|
||||
}
|
||||
return nil, RefreshCMS(ctx, svc, db)
|
||||
}
|
||||
|
||||
// RefreshCMS replaces the stored CMS-data issues of a site not crawled
|
||||
// yet with the report's current ones, in one transaction.
|
||||
func RefreshCMS(ctx context.Context, svc *classes.Service, db *gorm.DB) error {
|
||||
return lagoon.Transaction(ctx, db, func(ctx context.Context, tx *gorm.DB) error {
|
||||
cms, err := svc.Issues(classes.WithTx(ctx, tx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Where("crawl_id IS NULL").Delete(&models.Issue{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return SaveIssues(tx, nil, nil, cms)
|
||||
})
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// The kinds of the CMS-data issue report (D-13, D-14 CMS part). The
|
||||
@@ -309,3 +310,11 @@ func (s *Service) canonicalProblem(ctx context.Context, canonical string) (kind,
|
||||
}
|
||||
return IssueCanonicalNoindex, "", nil
|
||||
}
|
||||
|
||||
// Actor names who runs an audit operation (a crawl start, a cancel, an
|
||||
// ignore): the backend administrator's id and name, and whether the call
|
||||
// came through MCP.
|
||||
func (s *Service) Actor(ctx context.Context, db *gorm.DB) (id *uint, name string, mcp bool) {
|
||||
id, name = s.author(ctx, db, models.ViaAdmin)
|
||||
return id, name, ViaOf(ctx) == models.ViaMCP
|
||||
}
|
||||
|
||||
@@ -44,3 +44,10 @@ route_report_token: ""
|
||||
# WebSite node of the JSON-LD graph carries a SearchAction. Empty means no
|
||||
# SearchAction:
|
||||
# search_url: "/szukaj?q={search_term_string}"
|
||||
#
|
||||
# crawl_trusted lets the site audit's crawler fetch plain http addresses
|
||||
# and addresses inside the network (a staging or local site). Without it
|
||||
# the crawler fetches only https addresses of the site host and its www
|
||||
# twin, and refuses private addresses. Only config (or the environment,
|
||||
# SUMMER_GOLEM15__SEO__CRAWL_TRUSTED) sets it, never a setting or MCP.
|
||||
crawl_trusted: false
|
||||
|
||||
90
controllers/api/crawls.go
Normal file
90
controllers/api/crawls.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// StartCrawlInput is the body of POST /_seo/api/v1/crawls: nothing.
|
||||
type StartCrawlInput struct{}
|
||||
|
||||
// CrawlView is a crawl in an answer.
|
||||
type CrawlView struct {
|
||||
ID uint `json:"id"`
|
||||
Status string `json:"status"`
|
||||
Trigger string `json:"trigger"`
|
||||
TriggerName string `json:"trigger_name,omitempty"`
|
||||
Site string `json:"site"`
|
||||
MaxPages int `json:"max_pages"`
|
||||
Pages int `json:"pages"`
|
||||
LimitHit bool `json:"limit_hit"`
|
||||
Error string `json:"error,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at"`
|
||||
FinishedAt *time.Time `json:"finished_at"`
|
||||
CreatedAt *time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// crawlView is the answer form of a crawl.
|
||||
func crawlView(c models.Crawl) CrawlView {
|
||||
return CrawlView{
|
||||
ID: c.ID, Status: c.Status, Trigger: c.Trigger, TriggerName: c.TriggerName, Site: c.Site,
|
||||
MaxPages: c.MaxPages, Pages: c.Pages, LimitHit: c.LimitHit, Error: c.Error,
|
||||
StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, CreatedAt: c.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
// StartCrawl serves POST /_seo/api/v1/crawls (D-14, P33-39): it queues a
|
||||
// crawl started by the administrator (trigger admin, or mcp through MCP)
|
||||
// and answers 202 with it. A crawl already queued or running refuses with
|
||||
// 409 crawls.already_running, a missing site address with 422
|
||||
// crawls.no_site_url.
|
||||
func StartCrawl(service func() *classes.Service) http.HandlerFunc {
|
||||
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
||||
var row models.Crawl
|
||||
err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error {
|
||||
_, name, viaMCP := svc.Actor(ctx, tx)
|
||||
trigger := models.TriggerAdmin
|
||||
if viaMCP {
|
||||
trigger = models.TriggerMCP
|
||||
}
|
||||
var err error
|
||||
row, err = crawl.Start(ctx, svc, tx, trigger, name)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return 0, nil, StartRefusal(ctx, svc, err)
|
||||
}
|
||||
return http.StatusAccepted, map[string]any{"data": crawlView(row), "message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.started", nil)}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// StartRefusal maps a refused start to its answer.
|
||||
func StartRefusal(ctx context.Context, svc *classes.Service, err error) error {
|
||||
var running *crawl.AlreadyRunningError
|
||||
switch {
|
||||
case errors.As(err, &running):
|
||||
return &apiError{Status: http.StatusConflict, Body: map[string]any{
|
||||
"error": "crawls.already_running",
|
||||
"crawl_id": running.Crawl.ID,
|
||||
"message": RunningMessage(ctx, svc, running),
|
||||
}}
|
||||
case errors.Is(err, crawl.ErrNoSiteURL):
|
||||
return &apiError{Status: http.StatusUnprocessableEntity, Body: map[string]any{
|
||||
"error": "crawls.no_site_url",
|
||||
"message": svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.no_site_url", nil),
|
||||
}}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// RunningMessage is crawls.already_running with the running crawl's start.
|
||||
func RunningMessage(ctx context.Context, svc *classes.Service, running *crawl.AlreadyRunningError) string {
|
||||
return svc.PhraseCtx(ctx, "golem15.seo::lang.crawls.already_running", map[string]string{"time": running.Since().UTC().Format("2006-01-02 15:04 UTC")})
|
||||
}
|
||||
@@ -3,31 +3,96 @@ package api
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// ListIssuesInput is the query of GET /_seo/api/v1/issues.
|
||||
type ListIssuesInput struct {
|
||||
Kind string `json:"kind,omitempty" query:"kind" jsonschema:"one kind only: missing_title, missing_description, duplicate_title, duplicate_description, rule_overrides_record, redirect_chain, canonical_noindex, canonical_redirected, noindex_blocked or sitemap_blocked"`
|
||||
Kind string `json:"kind,omitempty" query:"kind" jsonschema:"one kind only, e.g. h1_multiple, broken_link, missing_title, redirect_chain or sitemap_blocked"`
|
||||
}
|
||||
|
||||
// ListIssues serves GET /_seo/api/v1/issues (D-13, D-14 CMS part): the
|
||||
// CMS-data issue report, each issue with its URL, kind, details and the
|
||||
// tool that fixes it, and a count per kind.
|
||||
// IssueView is a stored issue in an answer: the Phase 32 keys (kind, url,
|
||||
// locale, details, tool) and the audit's id, severity, source, fix,
|
||||
// found_at, ignored and crawl_id. tool is the fixing MCP tool when fix is
|
||||
// tool; for fix code, details.what names the template element.
|
||||
type IssueView struct {
|
||||
ID uint `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
URL string `json:"url"`
|
||||
Locale string `json:"locale,omitempty"`
|
||||
Details map[string]any `json:"details,omitempty"`
|
||||
Tool string `json:"tool"`
|
||||
Severity string `json:"severity"`
|
||||
Source string `json:"source"`
|
||||
Fix string `json:"fix"`
|
||||
FoundAt *time.Time `json:"found_at"`
|
||||
Ignored bool `json:"ignored"`
|
||||
CrawlID *uint `json:"crawl_id"`
|
||||
}
|
||||
|
||||
// issueViews turns stored issues into their answer form.
|
||||
func issueViews(db *gorm.DB, rows []models.Issue) ([]IssueView, error) {
|
||||
fps := make([]string, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
fps = append(fps, r.Fingerprint)
|
||||
}
|
||||
ignored, err := crawl.IgnoredSet(db, fps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]IssueView, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, IssueView{
|
||||
ID: r.ID, Kind: r.Kind, URL: r.URL, Locale: r.Locale, Details: r.DetailsMap(), Tool: r.Tool,
|
||||
Severity: r.Severity, Source: r.Source, Fix: r.Fix, FoundAt: r.FoundAt,
|
||||
Ignored: ignored[r.Fingerprint], CrawlID: r.CrawlID,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListIssues serves GET /_seo/api/v1/issues (D-13, D-14): the audit's
|
||||
// current issues, errors first: the newest crawl's findings with the
|
||||
// CMS-data issues stored with it, or before any crawl the CMS-data issues
|
||||
// as they are now. Each issue names the tool that fixes it, or fix code.
|
||||
// counts holds the number per kind.
|
||||
func ListIssues(service func() *classes.Service) http.HandlerFunc {
|
||||
return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) {
|
||||
issues, err := svc.Issues(ctx)
|
||||
db, err := read(ctx, svc)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
kind := r.URL.Query().Get("kind")
|
||||
latest, err := crawl.Current(ctx, svc, db)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
q := r.URL.Query()
|
||||
base := crawl.CurrentScope(db.Model(&models.Issue{}), latest)
|
||||
if id, err := strconv.ParseUint(strings.TrimSpace(q.Get("crawl")), 10, 64); err == nil && id > 0 {
|
||||
base = db.Model(&models.Issue{}).Where("golem15_seo_issues.crawl_id = ?", id)
|
||||
}
|
||||
var all []models.Issue
|
||||
if err := base.Order("rank ASC, kind ASC, url ASC, id ASC").Find(&all).Error; err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
views, err := issueViews(db, all)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
kind := strings.TrimSpace(q.Get("kind"))
|
||||
counts := map[string]int{}
|
||||
out := make([]classes.Issue, 0, len(issues))
|
||||
for _, issue := range issues {
|
||||
counts[issue.Kind]++
|
||||
if kind == "" || issue.Kind == kind {
|
||||
out = append(out, issue)
|
||||
out := make([]IssueView, 0, len(views))
|
||||
for _, v := range views {
|
||||
counts[v.Kind]++
|
||||
if kind == "" || v.Kind == kind {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return http.StatusOK, map[string]any{"data": out, "counts": counts, "total": len(out)}, nil
|
||||
|
||||
417
crawl_site_test.go
Normal file
417
crawl_site_test.go
Normal file
@@ -0,0 +1,417 @@
|
||||
package seo_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/conga"
|
||||
"git.golem15.com/golem15/summercms/modules/fetchguard"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"git.golem15.com/golem15/summercms/modules/party"
|
||||
"git.golem15.com/golem15/summercms/modules/signpost"
|
||||
)
|
||||
|
||||
// The test site the crawler fetches: a fake http.RoundTripper handed to
|
||||
// the crawler through fetchguard.WithTransport, so the crawler's real
|
||||
// requests (URL checks, User-Agent, no redirects followed) are asserted
|
||||
// offline. Paths it does not know answer 404.
|
||||
|
||||
// sitePage is one answer of the test site.
|
||||
type sitePage struct {
|
||||
status int
|
||||
body string
|
||||
header map[string]string
|
||||
}
|
||||
|
||||
// fakeSite serves the test site and records every request URL.
|
||||
type fakeSite struct {
|
||||
mu sync.Mutex
|
||||
pages map[string]sitePage
|
||||
requests []string
|
||||
agents []string
|
||||
}
|
||||
|
||||
func newFakeSite() *fakeSite { return &fakeSite{pages: map[string]sitePage{}} }
|
||||
|
||||
// set serves body with status at path (path with query).
|
||||
func (s *fakeSite) set(path string, status int, body string, header ...string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
h := map[string]string{"Content-Type": "text/html; charset=utf-8"}
|
||||
for i := 0; i+1 < len(header); i += 2 {
|
||||
h[header[i]] = header[i+1]
|
||||
}
|
||||
s.pages[path] = sitePage{status: status, body: body, header: h}
|
||||
}
|
||||
|
||||
// html serves a 200 HTML page.
|
||||
func (s *fakeSite) html(path, body string) { s.set(path, http.StatusOK, body) }
|
||||
|
||||
func (s *fakeSite) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
s.mu.Lock()
|
||||
s.requests = append(s.requests, r.URL.String())
|
||||
s.agents = append(s.agents, r.Header.Get("User-Agent"))
|
||||
key := r.URL.EscapedPath()
|
||||
if r.URL.RawQuery != "" {
|
||||
key += "?" + r.URL.RawQuery
|
||||
}
|
||||
p, ok := s.pages[key]
|
||||
s.mu.Unlock()
|
||||
if !ok || !strings.EqualFold(r.URL.Hostname(), "www.example.test") {
|
||||
p = sitePage{status: http.StatusNotFound, body: "<html><body>Not found</body></html>", header: map[string]string{"Content-Type": "text/html"}}
|
||||
}
|
||||
header := http.Header{}
|
||||
for k, v := range p.header {
|
||||
header.Set(k, v)
|
||||
}
|
||||
return &http.Response{StatusCode: p.status, Header: header, Body: io.NopCloser(strings.NewReader(p.body)), Request: r}, nil
|
||||
}
|
||||
|
||||
// requested reports whether any request went to an URL containing part.
|
||||
func (s *fakeSite) requested(part string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, u := range s.requests {
|
||||
if strings.Contains(u, part) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sitemap serves /sitemap.xml as an index of one file listing paths.
|
||||
func (s *fakeSite) sitemap(paths ...string) {
|
||||
s.set("/sitemap.xml", http.StatusOK, `<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>`+siteURL+`/sitemap-pages-1.xml</loc></sitemap></sitemapindex>`, "Content-Type", "application/xml")
|
||||
var b strings.Builder
|
||||
b.WriteString(`<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`)
|
||||
for _, p := range paths {
|
||||
b.WriteString("<url><loc>" + siteURL + html.EscapeString(p) + "</loc></url>")
|
||||
}
|
||||
b.WriteString("</urlset>")
|
||||
s.set("/sitemap-pages-1.xml", http.StatusOK, b.String(), "Content-Type", "application/xml")
|
||||
}
|
||||
|
||||
// doc is one planted page: a complete, clean document (unique title and
|
||||
// description, a self canonical, one H1 matching the title) that a test
|
||||
// mutates for the kind it plants.
|
||||
type doc struct {
|
||||
path string
|
||||
lang string
|
||||
title string
|
||||
noTitle bool
|
||||
description string
|
||||
noDesc bool
|
||||
canonical string
|
||||
noCanonical bool
|
||||
robots string
|
||||
head string
|
||||
h1 []string
|
||||
body string
|
||||
links []string
|
||||
}
|
||||
|
||||
// render writes the document's HTML.
|
||||
func (d doc) render() string {
|
||||
var b strings.Builder
|
||||
lang := d.lang
|
||||
if lang == "" {
|
||||
lang = "en"
|
||||
}
|
||||
b.WriteString(`<!doctype html><html lang="` + lang + `"><head><meta charset="utf-8">`)
|
||||
title := d.title
|
||||
if title == "" {
|
||||
title = "Page " + d.path
|
||||
}
|
||||
if !d.noTitle {
|
||||
b.WriteString("<title>" + html.EscapeString(title) + "</title>")
|
||||
}
|
||||
if !d.noDesc {
|
||||
desc := d.description
|
||||
if desc == "" {
|
||||
desc = "About " + d.path
|
||||
}
|
||||
b.WriteString(`<meta name="description" content="` + html.EscapeString(desc) + `">`)
|
||||
}
|
||||
if !d.noCanonical {
|
||||
c := d.canonical
|
||||
if c == "" {
|
||||
c = siteURL + d.path
|
||||
}
|
||||
b.WriteString(`<link rel="canonical" href="` + html.EscapeString(c) + `">`)
|
||||
}
|
||||
if d.robots != "" {
|
||||
b.WriteString(`<meta name="robots" content="` + d.robots + `">`)
|
||||
}
|
||||
b.WriteString(d.head + "</head><body><main>")
|
||||
h1 := d.h1
|
||||
if h1 == nil {
|
||||
h1 = []string{title}
|
||||
}
|
||||
for _, h := range h1 {
|
||||
b.WriteString("<h1>" + html.EscapeString(h) + "</h1>")
|
||||
}
|
||||
b.WriteString(d.body)
|
||||
for _, l := range d.links {
|
||||
b.WriteString(`<p><a href="` + l + `">Read about ` + html.EscapeString(strings.Trim(l, "/")) + `</a></p>`)
|
||||
}
|
||||
b.WriteString("</main></body></html>")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// plant serves the documents on the site.
|
||||
func (s *fakeSite) plant(docs ...doc) {
|
||||
for _, d := range docs {
|
||||
s.html(d.path, d.render())
|
||||
}
|
||||
}
|
||||
|
||||
// The fake content plugin acme.crawl: in-memory records behind two types
|
||||
// with descriptor facts and fixing tools, so the audit's content routes
|
||||
// (P33-38) are exercised. acme-crawl-post (/c/:slug) names edit, headings
|
||||
// and image_alt tools; acme-crawl-category (/cat/:slug) names only a
|
||||
// headings tool.
|
||||
const (
|
||||
crawlPostType = "acme-crawl-post"
|
||||
crawlCategoryType = "acme-crawl-category"
|
||||
)
|
||||
|
||||
// crawlRecord is one in-memory record.
|
||||
type crawlRecord struct {
|
||||
id string
|
||||
path string
|
||||
title string
|
||||
words int
|
||||
headings []pact.ContentHeading
|
||||
images []pact.ContentImage
|
||||
topHeading int
|
||||
parents []pact.ContentRef
|
||||
}
|
||||
|
||||
var (
|
||||
crawlRecordsMu sync.Mutex
|
||||
crawlRecords = map[string][]crawlRecord{}
|
||||
)
|
||||
|
||||
// setCrawlRecords replaces the records of the fake types for one test.
|
||||
func setCrawlRecords(t *testing.T, byType map[string][]crawlRecord) {
|
||||
t.Helper()
|
||||
crawlRecordsMu.Lock()
|
||||
crawlRecords = byType
|
||||
crawlRecordsMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
crawlRecordsMu.Lock()
|
||||
crawlRecords = map[string][]crawlRecord{}
|
||||
crawlRecordsMu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func crawlRecordsOf(typeKey string) []crawlRecord {
|
||||
crawlRecordsMu.Lock()
|
||||
defer crawlRecordsMu.Unlock()
|
||||
return slices.Clone(crawlRecords[typeKey])
|
||||
}
|
||||
|
||||
func (r crawlRecord) content() pact.ContentRecord {
|
||||
words := r.words
|
||||
var wp *int
|
||||
if words > 0 {
|
||||
wp = &words
|
||||
}
|
||||
return pact.ContentRecord{
|
||||
ID: r.id, Published: true, TopHeading: r.topHeading, Parents: r.parents,
|
||||
Locales: []pact.ContentLocale{{Locale: "en", URL: r.path, Title: r.title, Words: wp, Headings: r.headings, Images: r.images}},
|
||||
}
|
||||
}
|
||||
|
||||
type crawlContentPlugin struct{}
|
||||
|
||||
func (crawlContentPlugin) ID() string { return "acme.crawl" }
|
||||
func (crawlContentPlugin) Requires() []string { return nil }
|
||||
func (crawlContentPlugin) Register(*backpack.App) error { return nil }
|
||||
func (crawlContentPlugin) Boot(*backpack.App) error { return nil }
|
||||
func (crawlContentPlugin) MenuItemTypes() []pact.MenuItemType {
|
||||
return []pact.MenuItemType{
|
||||
crawlType(crawlPostType, "/c/:slug", map[string]string{
|
||||
pact.ContentToolEdit: "acme_edit_post", pact.ContentToolHeadings: "acme_fix_headings", pact.ContentToolImageAlt: "acme_set_alt",
|
||||
}),
|
||||
crawlType(crawlCategoryType, "/cat/:slug", map[string]string{pact.ContentToolHeadings: "acme_category_headings"}),
|
||||
}
|
||||
}
|
||||
|
||||
// crawlType is one fake type over the in-memory records.
|
||||
func crawlType(key, pattern string, tools map[string]string) pact.MenuItemType {
|
||||
find := func(match func(crawlRecord) bool) *crawlRecord {
|
||||
for _, r := range crawlRecordsOf(key) {
|
||||
if match(r) {
|
||||
return &r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return pact.MenuItemType{
|
||||
Key: key, Label: key, DefaultPattern: pattern, Placeholders: []string{"slug"},
|
||||
Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { return nil, nil },
|
||||
Content: &pact.ContentDescriptor{
|
||||
Model: "acme_crawl_" + strings.ReplaceAll(key, "-", "_"),
|
||||
Tools: tools,
|
||||
Describe: func(_ context.Context, in pact.ContentDescribe) (*pact.ContentRecord, error) {
|
||||
r := find(func(r crawlRecord) bool { return r.id == in.ID })
|
||||
if r == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rec := r.content()
|
||||
return &rec, nil
|
||||
},
|
||||
Records: func(_ context.Context, in pact.ContentList) (pact.ContentPage, error) {
|
||||
var out pact.ContentPage
|
||||
if in.After != "" {
|
||||
return out, nil
|
||||
}
|
||||
for _, r := range crawlRecordsOf(key) {
|
||||
out.Records = append(out.Records, r.content())
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Match: func(_ context.Context, in pact.ContentMatch) (*pact.ContentMatchResult, error) {
|
||||
if _, ok := signpost.MatchPattern(in.Pattern, in.Path); !ok {
|
||||
return nil, nil
|
||||
}
|
||||
r := find(func(r crawlRecord) bool { return r.path == in.Path })
|
||||
if r == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return &pact.ContentMatchResult{ID: r.id, Locale: "en", Exact: true}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
var _ pact.HasMenuItemTypes = crawlContentPlugin{}
|
||||
|
||||
func init() {
|
||||
party.Register(crawlContentPlugin{})
|
||||
}
|
||||
|
||||
// bootCrawlEnv boots acme.pages, acme.crawl and golem15.seo.
|
||||
func bootCrawlEnv(t *testing.T, extraEnv ...string) *testEnv {
|
||||
t.Helper()
|
||||
return bootPlugins(t, []string{"acme.pages", "acme.crawl", "golem15.seo"}, extraEnv...)
|
||||
}
|
||||
|
||||
// crawlJob returns the SEO plugin's crawl job.
|
||||
func (e *testEnv) crawlJob(t *testing.T) pact.Job {
|
||||
t.Helper()
|
||||
for _, j := range e.seoPlugin(t).Jobs() {
|
||||
if info, ok := conga.Describe(j); ok && info.Kind == crawl.CrawlKind {
|
||||
return j
|
||||
}
|
||||
}
|
||||
t.Fatal("the SEO plugin registers no crawl job")
|
||||
return nil
|
||||
}
|
||||
|
||||
// siteCtx is a context whose crawler requests go to site.
|
||||
func siteCtx(t *testing.T, site *fakeSite) context.Context {
|
||||
return fetchguard.WithTransport(t.Context(), site)
|
||||
}
|
||||
|
||||
// startCrawl starts a crawl through the API and returns its id.
|
||||
func (e *testEnv) startCrawl(t *testing.T, token string) uint {
|
||||
t.Helper()
|
||||
rec := e.expect(t, http.StatusAccepted, http.MethodPost, "/_seo/api/v1/crawls", token, map[string]any{})
|
||||
var out struct {
|
||||
Data struct {
|
||||
ID uint `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil || out.Data.ID == 0 {
|
||||
t.Fatalf("start crawl: %s (%v)", rec.Body.String(), err)
|
||||
}
|
||||
return out.Data.ID
|
||||
}
|
||||
|
||||
// runCrawl runs the crawl job of id against the site.
|
||||
func (e *testEnv) runCrawl(t *testing.T, ctx context.Context, id uint) {
|
||||
t.Helper()
|
||||
if err := e.crawlJob(t).Work(ctx, crawl.CrawlArgs{CrawlID: id}); err != nil {
|
||||
t.Fatalf("crawl job: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// crawlOnce starts a crawl and runs it against the site.
|
||||
func (e *testEnv) crawlOnce(t *testing.T, token string, ctx context.Context) uint {
|
||||
t.Helper()
|
||||
id := e.startCrawl(t, token)
|
||||
e.runCrawl(t, ctx, id)
|
||||
return id
|
||||
}
|
||||
|
||||
// apiIssue is one issue of GET /_seo/api/v1/issues.
|
||||
type apiIssue struct {
|
||||
ID uint `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
URL string `json:"url"`
|
||||
Locale string `json:"locale"`
|
||||
Details map[string]any `json:"details"`
|
||||
Tool string `json:"tool"`
|
||||
Severity string `json:"severity"`
|
||||
Source string `json:"source"`
|
||||
Fix string `json:"fix"`
|
||||
Ignored bool `json:"ignored"`
|
||||
CrawlID *uint `json:"crawl_id"`
|
||||
}
|
||||
|
||||
// issues lists the audit's issues through the API (query appended).
|
||||
func (e *testEnv) issues(t *testing.T, token, query string) []apiIssue {
|
||||
t.Helper()
|
||||
path := "/_seo/api/v1/issues"
|
||||
if query != "" {
|
||||
path += "?" + query
|
||||
}
|
||||
rec := e.expect(t, http.StatusOK, http.MethodGet, path, token, nil)
|
||||
var out struct {
|
||||
Data []apiIssue `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Data
|
||||
}
|
||||
|
||||
// issueSet is "kind url" for every issue of source, sorted.
|
||||
func issueSet(list []apiIssue, source string) []string {
|
||||
var out []string
|
||||
for _, i := range list {
|
||||
if source == "" || i.Source == source {
|
||||
out = append(out, i.Kind+" "+i.URL)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// findIssue returns the one issue of kind at url, failing when there is
|
||||
// not exactly one.
|
||||
func findIssue(t *testing.T, list []apiIssue, kind, url string) apiIssue {
|
||||
t.Helper()
|
||||
var found []apiIssue
|
||||
for _, i := range list {
|
||||
if i.Kind == kind && i.URL == url {
|
||||
found = append(found, i)
|
||||
}
|
||||
}
|
||||
if len(found) != 1 {
|
||||
t.Fatalf("%d %s issues for %s, want 1; all: %v", len(found), kind, url, issueSet(list, ""))
|
||||
}
|
||||
return found[0]
|
||||
}
|
||||
48
crawler_test.go
Normal file
48
crawler_test.go
Normal file
@@ -0,0 +1,48 @@
|
||||
package seo_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
)
|
||||
|
||||
// TestCrawlerTracer is the end-to-end path (D-14): an administrator
|
||||
// starts a crawl through the API, the crawl job fetches the site with the
|
||||
// crawler's User-Agent, and GET /issues lists the page with two H1
|
||||
// headings as an error with how to fix it: code for H1 headings of the
|
||||
// template, the record type's headings tool for an H1 inside the record's
|
||||
// own content.
|
||||
func TestCrawlerTracer(t *testing.T) {
|
||||
env := bootCrawlEnv(t)
|
||||
token := env.admin(t, true)
|
||||
setCrawlRecords(t, map[string][]crawlRecord{
|
||||
crawlPostType: {{id: "1", path: "/c/two-h1", title: "Post title", headings: []pact.ContentHeading{{Level: 1, Text: "Inside the body"}}}},
|
||||
})
|
||||
site := newFakeSite()
|
||||
site.sitemap("/", "/p/two-h1", "/c/two-h1")
|
||||
site.plant(
|
||||
doc{path: "/", title: "Home", links: []string{"/p/two-h1", "/c/two-h1"}},
|
||||
doc{path: "/p/two-h1", title: "Template page", h1: []string{"Template page", "Another H1"}},
|
||||
doc{path: "/c/two-h1", title: "Post title", h1: []string{"Post title", "Inside the body"}},
|
||||
)
|
||||
|
||||
env.crawlOnce(t, token, siteCtx(t, site))
|
||||
|
||||
list := env.issues(t, token, "")
|
||||
tpl := findIssue(t, list, "h1_multiple", "/p/two-h1")
|
||||
if tpl.Severity != "error" || tpl.Fix != "code" || tpl.Tool != "" || tpl.Source != "crawl" || tpl.Details["what"] == nil || tpl.Details["count"] != float64(2) {
|
||||
t.Fatalf("template h1_multiple = %+v", tpl)
|
||||
}
|
||||
content := findIssue(t, list, "h1_multiple", "/c/two-h1")
|
||||
if content.Fix != "tool" || content.Tool != "acme_fix_headings" || content.CrawlID == nil || content.ID == 0 {
|
||||
t.Fatalf("content h1_multiple = %+v", content)
|
||||
}
|
||||
if !site.requested("/sitemap-pages-1.xml") || !site.requested("/c/two-h1") {
|
||||
t.Fatalf("crawler requests = %v", site.requests)
|
||||
}
|
||||
for _, ua := range site.agents {
|
||||
if ua != "SummerCMS-SEO-Crawler/1 (+https://www.example.test)" {
|
||||
t.Fatalf("User-Agent = %q", ua)
|
||||
}
|
||||
}
|
||||
}
|
||||
2
go.mod
2
go.mod
@@ -12,6 +12,7 @@ require (
|
||||
github.com/testcontainers/testcontainers-go v0.44.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0
|
||||
gocloud.dev v0.46.0
|
||||
golang.org/x/net v0.58.0
|
||||
gorm.io/driver/postgres v1.6.3
|
||||
gorm.io/gorm v1.31.2
|
||||
)
|
||||
@@ -162,7 +163,6 @@ require (
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/image v0.46.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
|
||||
16
jobs.go
16
jobs.go
@@ -2,19 +2,33 @@ package seo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes"
|
||||
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
|
||||
"git.golem15.com/golem15/summercms/modules/conga"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
)
|
||||
|
||||
// Jobs registers the entry generation job (D-94) on conga.
|
||||
// Jobs registers the entry generation job (D-94) and the crawl job
|
||||
// (golem15.seo.crawl, D-14) on conga. A crawl is not retried: a failed run
|
||||
// records its error and keeps what it fetched.
|
||||
func (p *Plugin) Jobs() []pact.Job {
|
||||
return []pact.Job{
|
||||
conga.Job(p.generateEntries, conga.MaxAttempts(3)),
|
||||
conga.Job(p.crawl, conga.MaxAttempts(1), conga.Timeout(6*time.Hour)),
|
||||
}
|
||||
}
|
||||
|
||||
// crawl runs one crawl job (golem15.seo.crawl).
|
||||
func (p *Plugin) crawl(ctx context.Context, args crawl.CrawlArgs) error {
|
||||
svc := p.Service()
|
||||
if svc == nil {
|
||||
return classes.ErrNotBooted
|
||||
}
|
||||
return crawl.Run(ctx, svc, args.CrawlID)
|
||||
}
|
||||
|
||||
// generateEntries runs one generation job.
|
||||
func (p *Plugin) generateEntries(ctx context.Context, args classes.GenerateEntriesArgs) error {
|
||||
svc := p.Service()
|
||||
|
||||
@@ -311,6 +311,7 @@ history:
|
||||
kind_notfound: 404 log entry
|
||||
kind_page: registry page
|
||||
jobs:
|
||||
crawl: 'SEO: crawl the site'
|
||||
generate_entries: 'SEO: generate entries'
|
||||
effective:
|
||||
property: Property
|
||||
@@ -520,3 +521,7 @@ schema:
|
||||
price_invalid: "Enter a price such as 150 or 149.99."
|
||||
currency_invalid: "Use a three-letter currency code, e.g. PLN."
|
||||
merged: "Saved. Your custom JSON-LD replaces :props in the generated :node node."
|
||||
crawls:
|
||||
started: Crawl started. Follow it in SEO › Crawls; the issues appear in Site audit when it finishes.
|
||||
already_running: A crawl is already running (started :time). Wait for it, or cancel it in SEO › Crawls.
|
||||
no_site_url: The site address for crawls is not configured (golem15.seo.site_url). Ask the administrator to set it.
|
||||
|
||||
@@ -317,6 +317,7 @@ history:
|
||||
kind_notfound: Wpis dziennika 404
|
||||
kind_page: strona rejestru
|
||||
jobs:
|
||||
crawl: 'SEO: skanowanie witryny'
|
||||
generate_entries: 'SEO: generowanie wpisów'
|
||||
effective:
|
||||
property: Właściwość
|
||||
@@ -534,3 +535,7 @@ schema:
|
||||
price_invalid: "Wpisz cenę, np. 150 lub 149,99."
|
||||
currency_invalid: "Użyj trzyliterowego kodu waluty, np. PLN."
|
||||
merged: "Zapisano. Twój JSON-LD zastępuje :props w wygenerowanym węźle :node."
|
||||
crawls:
|
||||
started: Rozpoczęto skanowanie. Postęp znajdziesz w SEO › Skanowania; problemy pojawią się w Audycie witryny po jego zakończeniu.
|
||||
already_running: Skanowanie już trwa (rozpoczęte :time). Poczekaj na nie lub anuluj je w SEO › Skanowania.
|
||||
no_site_url: Adres witryny do skanowania nie jest skonfigurowany (golem15.seo.site_url). Poproś administratora o jego ustawienie.
|
||||
|
||||
103
models/crawl.go
Normal file
103
models/crawl.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// The states of a crawl (UI-SPEC C24).
|
||||
const (
|
||||
CrawlQueued = "queued"
|
||||
CrawlRunning = "running"
|
||||
CrawlFinished = "finished"
|
||||
CrawlFailed = "failed"
|
||||
CrawlCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// CrawlStatuses lists the states in filter order.
|
||||
var CrawlStatuses = []string{CrawlQueued, CrawlRunning, CrawlFinished, CrawlFailed, CrawlCancelled}
|
||||
|
||||
// What started a crawl.
|
||||
const (
|
||||
TriggerAdmin = "admin"
|
||||
TriggerSchedule = "schedule"
|
||||
TriggerMCP = "mcp"
|
||||
)
|
||||
|
||||
// The states of the rendered comparison of a crawl (P33-55).
|
||||
const (
|
||||
RenderSkipped = "skipped"
|
||||
RenderDone = "done"
|
||||
RenderFailed = "failed"
|
||||
)
|
||||
|
||||
// Crawl is one run of the site audit's crawler on golem15_seo_crawls
|
||||
// (D-14, P33-37): its state, what started it and who, the site address,
|
||||
// the page limit, the pages fetched and whether the limit stopped it, the
|
||||
// error of a failed run, the state of the rendered comparison, the conga
|
||||
// job and its times.
|
||||
type Crawl struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
Status string `gorm:"column:status"`
|
||||
Trigger string `gorm:"column:trigger"`
|
||||
TriggerName string `gorm:"column:trigger_name"`
|
||||
Site string `gorm:"column:site"`
|
||||
MaxPages int `gorm:"column:max_pages"`
|
||||
Pages int `gorm:"column:pages"`
|
||||
LimitHit bool `gorm:"column:limit_hit"`
|
||||
Error string `gorm:"column:error"`
|
||||
RenderStatus string `gorm:"column:render_status"`
|
||||
RenderPages int `gorm:"column:render_pages"`
|
||||
RenderNote string `gorm:"column:render_note"`
|
||||
CancelledBy string `gorm:"column:cancelled_by"`
|
||||
JobID *uint `gorm:"column:job_id"`
|
||||
StartedAt *time.Time `gorm:"column:started_at"`
|
||||
FinishedAt *time.Time `gorm:"column:finished_at"`
|
||||
CreatedAt *time.Time `gorm:"column:created_at"`
|
||||
// The list's values, filled by the Crawls controller's
|
||||
// ListExtendRecords and never saved.
|
||||
IssuesCell string `gorm:"-:all;column:issues"`
|
||||
PagesCell string `gorm:"-:all;column:pages_cell"`
|
||||
DurationCell string `gorm:"-:all;column:duration"`
|
||||
TriggerCell string `gorm:"-:all;column:trigger_cell"`
|
||||
}
|
||||
|
||||
func (Crawl) TableName() string { return "golem15_seo_crawls" }
|
||||
|
||||
// MorphName names a crawl in the admin.
|
||||
func (Crawl) MorphName() string { return `Golem15\SEO\Models\Crawl` }
|
||||
|
||||
// Fillable is empty: crawls are written by the crawler only.
|
||||
func (Crawl) Fillable() []string { return []string{} }
|
||||
|
||||
// Rules is empty: nothing is edited.
|
||||
func (Crawl) Rules() map[string]string { return map[string]string{} }
|
||||
|
||||
// Active reports whether the crawl is queued or running.
|
||||
func (c Crawl) Active() bool { return c.Status == CrawlQueued || c.Status == CrawlRunning }
|
||||
|
||||
// The Crawls list's filter scope.
|
||||
const FilterCrawlStatus = "FilterCrawlStatus"
|
||||
|
||||
// FilterScopes lists the Crawls list's filter scopes.
|
||||
func (Crawl) FilterScopes() []string { return []string{FilterCrawlStatus} }
|
||||
|
||||
// FilterScope narrows the Crawls list to one state; an unknown value
|
||||
// matches nothing.
|
||||
func (Crawl) FilterScope(name string, db *gorm.DB, value any) *gorm.DB {
|
||||
if db == nil || name != FilterCrawlStatus {
|
||||
return db
|
||||
}
|
||||
s, _ := value.(string)
|
||||
for _, st := range CrawlStatuses {
|
||||
if s == st {
|
||||
return db.Where("golem15_seo_crawls.status = ?", s)
|
||||
}
|
||||
}
|
||||
return db.Where("1 = 0")
|
||||
}
|
||||
|
||||
func init() {
|
||||
Register(Crawl{})
|
||||
}
|
||||
66
models/crawl_page.go
Normal file
66
models/crawl_page.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
)
|
||||
|
||||
// CrawlHeading is one heading of a crawled page in document order.
|
||||
type CrawlHeading struct {
|
||||
Level int `json:"level"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// MaxPageText caps the stored main text of a crawled page, in characters.
|
||||
const MaxPageText = 100_000
|
||||
|
||||
// CrawlPage is one address a crawl fetched, on golem15_seo_crawl_pages
|
||||
// (P33-37): the answer (status, Location, content type) and what the raw
|
||||
// HTML says (title, description, canonical, robots, headings, word count,
|
||||
// main text), its depth from the home page, whether a sitemap lists it and
|
||||
// which file, and its outgoing and incoming internal links.
|
||||
type CrawlPage struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
CrawlID uint `gorm:"column:crawl_id"`
|
||||
URL string `gorm:"column:url"`
|
||||
Status int `gorm:"column:status"`
|
||||
Location string `gorm:"column:location"`
|
||||
ContentType string `gorm:"column:content_type"`
|
||||
Title string `gorm:"column:title"`
|
||||
Description string `gorm:"column:description"`
|
||||
Canonical string `gorm:"column:canonical"`
|
||||
Robots string `gorm:"column:robots"`
|
||||
XRobots string `gorm:"column:x_robots"`
|
||||
H1 lagoon.Jsonable[[]string] `gorm:"column:h1"`
|
||||
Headings lagoon.Jsonable[[]CrawlHeading] `gorm:"column:headings"`
|
||||
Words int `gorm:"column:words"`
|
||||
Text string `gorm:"column:text"`
|
||||
Depth int `gorm:"column:depth"`
|
||||
InSitemap bool `gorm:"column:in_sitemap"`
|
||||
SitemapFile string `gorm:"column:sitemap_file"`
|
||||
LinksOut int `gorm:"column:links_out"`
|
||||
LinksIn int `gorm:"column:links_in"`
|
||||
FetchedAt *time.Time `gorm:"column:fetched_at"`
|
||||
}
|
||||
|
||||
func (CrawlPage) TableName() string { return "golem15_seo_crawl_pages" }
|
||||
|
||||
// CrawlLink is one link between two addresses of a crawl, on
|
||||
// golem15_seo_crawl_links: the linking page, the target, its text and rel,
|
||||
// and whether the target is on the site.
|
||||
type CrawlLink struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
CrawlID uint `gorm:"column:crawl_id"`
|
||||
FromURL string `gorm:"column:from_url"`
|
||||
ToURL string `gorm:"column:to_url"`
|
||||
Text string `gorm:"column:text"`
|
||||
Rel string `gorm:"column:rel"`
|
||||
Internal bool `gorm:"column:internal"`
|
||||
}
|
||||
|
||||
func (CrawlLink) TableName() string { return "golem15_seo_crawl_links" }
|
||||
|
||||
func init() {
|
||||
Register(CrawlPage{}, CrawlLink{})
|
||||
}
|
||||
156
models/issue.go
Normal file
156
models/issue.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Where a stored issue was found (UI-SPEC C23).
|
||||
const (
|
||||
SourceCrawl = "crawl"
|
||||
SourceCMS = "cms"
|
||||
)
|
||||
|
||||
// How an issue is fixed (P33-38): through an MCP tool that changes CMS
|
||||
// data, or in the site's template code.
|
||||
const (
|
||||
FixTool = "tool"
|
||||
FixCode = "code"
|
||||
)
|
||||
|
||||
// The severities of an issue and their sort rank (errors first).
|
||||
const (
|
||||
SeverityError = "error"
|
||||
SeverityWarning = "warning"
|
||||
SeverityNotice = "notice"
|
||||
)
|
||||
|
||||
// SeverityRank orders the severities: error 1, warning 2, notice 3.
|
||||
func SeverityRank(severity string) int {
|
||||
switch severity {
|
||||
case SeverityError:
|
||||
return 1
|
||||
case SeverityWarning:
|
||||
return 2
|
||||
}
|
||||
return 3
|
||||
}
|
||||
|
||||
// Issue is one stored finding of the site audit on golem15_seo_issues
|
||||
// (P33-37): the crawl it belongs to (nil for CMS-data issues stored before
|
||||
// any crawl), where it was found, its kind and severity, the address and
|
||||
// locale, the kind's details, the fixing MCP tool or fix code, and the
|
||||
// fingerprint an ignore is keyed by.
|
||||
type Issue struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
CrawlID *uint `gorm:"column:crawl_id"`
|
||||
Source string `gorm:"column:source"`
|
||||
Kind string `gorm:"column:kind"`
|
||||
Severity string `gorm:"column:severity"`
|
||||
Rank int `gorm:"column:rank"`
|
||||
URL string `gorm:"column:url"`
|
||||
Locale string `gorm:"column:locale"`
|
||||
Details lagoon.Jsonable[map[string]any] `gorm:"column:details"`
|
||||
Tool string `gorm:"column:tool"`
|
||||
Fix string `gorm:"column:fix"`
|
||||
Fingerprint string `gorm:"column:fingerprint"`
|
||||
FoundAt *time.Time `gorm:"column:found_at"`
|
||||
// The list's values, filled by the Site audit controller's
|
||||
// ListExtendRecords and never saved.
|
||||
Summary string `gorm:"-:all;column:summary"`
|
||||
FixCell string `gorm:"-:all;column:fix_cell"`
|
||||
KindCell string `gorm:"-:all;column:kind_cell"`
|
||||
}
|
||||
|
||||
func (Issue) TableName() string { return "golem15_seo_issues" }
|
||||
|
||||
// MorphName names an issue in the admin.
|
||||
func (Issue) MorphName() string { return `Golem15\SEO\Models\Issue` }
|
||||
|
||||
// Fillable is empty: issues are written by the audit only.
|
||||
func (Issue) Fillable() []string { return []string{} }
|
||||
|
||||
// Rules is empty: nothing is edited.
|
||||
func (Issue) Rules() map[string]string { return map[string]string{} }
|
||||
|
||||
// DetailsMap returns the issue's details, never nil.
|
||||
func (i Issue) DetailsMap() map[string]any {
|
||||
if d := i.Details.Get(); d != nil {
|
||||
return d
|
||||
}
|
||||
return map[string]any{}
|
||||
}
|
||||
|
||||
// IssueIgnore is one ignored fingerprint on golem15_seo_issue_ignores: an
|
||||
// issue with this fingerprint stays hidden in later crawls until it is no
|
||||
// longer ignored (UI-SPEC C23).
|
||||
type IssueIgnore struct {
|
||||
ID uint `gorm:"column:id;primaryKey"`
|
||||
Fingerprint string `gorm:"column:fingerprint"`
|
||||
AdminID *uint `gorm:"column:admin_id"`
|
||||
IgnoredAt *time.Time `gorm:"column:ignored_at"`
|
||||
}
|
||||
|
||||
func (IssueIgnore) TableName() string { return "golem15_seo_issue_ignores" }
|
||||
|
||||
// The Site audit list's filter scopes and the status and fix values.
|
||||
const (
|
||||
FilterIssueSeverity = "FilterIssueSeverity"
|
||||
FilterIssueKind = "FilterIssueKind"
|
||||
FilterIssueSource = "FilterIssueSource"
|
||||
FilterIssueFix = "FilterIssueFix"
|
||||
FilterIssueStatus = "FilterIssueStatus"
|
||||
IssueOpen = "open"
|
||||
IssueIgnored = "ignored"
|
||||
)
|
||||
|
||||
// IgnoredSQL is true for an issue whose fingerprint is ignored.
|
||||
const IgnoredSQL = `EXISTS (SELECT 1 FROM golem15_seo_issue_ignores AS ig WHERE ig.fingerprint = golem15_seo_issues.fingerprint)`
|
||||
|
||||
// FilterScopes lists the Site audit list's filter scopes.
|
||||
func (Issue) FilterScopes() []string {
|
||||
return []string{FilterIssueSeverity, FilterIssueKind, FilterIssueSource, FilterIssueFix, FilterIssueStatus}
|
||||
}
|
||||
|
||||
// FilterScope narrows the Site audit list; an unknown value matches
|
||||
// nothing.
|
||||
func (Issue) FilterScope(name string, db *gorm.DB, value any) *gorm.DB {
|
||||
if db == nil {
|
||||
return db
|
||||
}
|
||||
s, _ := value.(string)
|
||||
switch name {
|
||||
case FilterIssueSeverity:
|
||||
if s == SeverityError || s == SeverityWarning || s == SeverityNotice {
|
||||
return db.Where("golem15_seo_issues.severity = ?", s)
|
||||
}
|
||||
case FilterIssueKind:
|
||||
if s != "" {
|
||||
return db.Where("golem15_seo_issues.kind = ?", s)
|
||||
}
|
||||
case FilterIssueSource:
|
||||
if s == SourceCrawl || s == SourceCMS {
|
||||
return db.Where("golem15_seo_issues.source = ?", s)
|
||||
}
|
||||
case FilterIssueFix:
|
||||
if s == FixTool || s == FixCode {
|
||||
return db.Where("golem15_seo_issues.fix = ?", s)
|
||||
}
|
||||
case FilterIssueStatus:
|
||||
switch s {
|
||||
case IssueOpen:
|
||||
return db.Where("NOT " + IgnoredSQL)
|
||||
case IssueIgnored:
|
||||
return db.Where(IgnoredSQL)
|
||||
}
|
||||
default:
|
||||
return db
|
||||
}
|
||||
return db.Where("1 = 0")
|
||||
}
|
||||
|
||||
func init() {
|
||||
Register(Issue{}, IssueIgnore{})
|
||||
}
|
||||
@@ -81,6 +81,7 @@ func (p *Plugin) Routes(r pact.Router) error {
|
||||
g.Post("/import", apicontrollers.ImportCSV(p.Service), api)
|
||||
g.Post("/batch", apicontrollers.ApplyBatch(p.Service), api)
|
||||
g.Get("/issues", apicontrollers.ListIssues(p.Service), api)
|
||||
g.Post("/crawls", apicontrollers.StartCrawl(p.Service), api)
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
111
updates/202610280005_create_golem15_seo_crawls.go
Normal file
111
updates/202610280005_create_golem15_seo_crawls.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"github.com/go-gormigrate/gormigrate/v2"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// createCrawls creates the site audit's tables (D-14, P33-37): the crawls,
|
||||
// the pages and links each crawl fetched, the stored issues (crawl and
|
||||
// CMS-data findings, with the fixing tool or fix code and the fingerprint)
|
||||
// and the ignored fingerprints. Deleting a crawl deletes its pages, links
|
||||
// and issues.
|
||||
var createCrawls = []*gormigrate.Migration{
|
||||
{
|
||||
ID: "202610280005_create_golem15_seo_crawls",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
for _, stmt := range []string{
|
||||
`CREATE TABLE golem15_seo_crawls (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
trigger TEXT NOT NULL DEFAULT 'admin',
|
||||
trigger_name TEXT NOT NULL DEFAULT '',
|
||||
site TEXT NOT NULL DEFAULT '',
|
||||
max_pages INTEGER NOT NULL DEFAULT 0,
|
||||
pages INTEGER NOT NULL DEFAULT 0,
|
||||
limit_hit BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
render_status TEXT NOT NULL DEFAULT 'skipped',
|
||||
render_pages INTEGER NOT NULL DEFAULT 0,
|
||||
render_note TEXT NOT NULL DEFAULT '',
|
||||
cancelled_by TEXT NOT NULL DEFAULT '',
|
||||
job_id BIGINT NULL,
|
||||
started_at TIMESTAMPTZ NULL,
|
||||
finished_at TIMESTAMPTZ NULL,
|
||||
created_at TIMESTAMPTZ NULL
|
||||
)`,
|
||||
`CREATE INDEX golem15_seo_crawls_status ON golem15_seo_crawls (status)`,
|
||||
`CREATE TABLE golem15_seo_crawl_pages (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
crawl_id BIGINT NOT NULL REFERENCES golem15_seo_crawls (id) ON DELETE CASCADE,
|
||||
url TEXT NOT NULL,
|
||||
status INTEGER NOT NULL DEFAULT 0,
|
||||
location TEXT NOT NULL DEFAULT '',
|
||||
content_type TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
canonical TEXT NOT NULL DEFAULT '',
|
||||
robots TEXT NOT NULL DEFAULT '',
|
||||
x_robots TEXT NOT NULL DEFAULT '',
|
||||
h1 JSONB NULL,
|
||||
headings JSONB NULL,
|
||||
words INTEGER NOT NULL DEFAULT 0,
|
||||
text TEXT NOT NULL DEFAULT '',
|
||||
depth INTEGER NOT NULL DEFAULT 0,
|
||||
in_sitemap BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
sitemap_file TEXT NOT NULL DEFAULT '',
|
||||
links_out INTEGER NOT NULL DEFAULT 0,
|
||||
links_in INTEGER NOT NULL DEFAULT 0,
|
||||
fetched_at TIMESTAMPTZ NULL
|
||||
)`,
|
||||
`CREATE UNIQUE INDEX golem15_seo_crawl_pages_url ON golem15_seo_crawl_pages (crawl_id, url)`,
|
||||
`CREATE TABLE golem15_seo_crawl_links (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
crawl_id BIGINT NOT NULL REFERENCES golem15_seo_crawls (id) ON DELETE CASCADE,
|
||||
from_url TEXT NOT NULL,
|
||||
to_url TEXT NOT NULL,
|
||||
text TEXT NOT NULL DEFAULT '',
|
||||
rel TEXT NOT NULL DEFAULT '',
|
||||
internal BOOLEAN NOT NULL DEFAULT TRUE
|
||||
)`,
|
||||
`CREATE INDEX golem15_seo_crawl_links_to ON golem15_seo_crawl_links (crawl_id, to_url)`,
|
||||
`CREATE TABLE golem15_seo_issues (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
crawl_id BIGINT NULL REFERENCES golem15_seo_crawls (id) ON DELETE CASCADE,
|
||||
source TEXT NOT NULL DEFAULT 'crawl',
|
||||
kind TEXT NOT NULL,
|
||||
severity TEXT NOT NULL DEFAULT 'warning',
|
||||
rank SMALLINT NOT NULL DEFAULT 2,
|
||||
url TEXT NOT NULL DEFAULT '',
|
||||
locale TEXT NOT NULL DEFAULT '',
|
||||
details JSONB NULL,
|
||||
tool TEXT NOT NULL DEFAULT '',
|
||||
fix TEXT NOT NULL DEFAULT 'tool',
|
||||
fingerprint TEXT NOT NULL,
|
||||
found_at TIMESTAMPTZ NULL
|
||||
)`,
|
||||
`CREATE INDEX golem15_seo_issues_crawl ON golem15_seo_issues (crawl_id, rank, kind)`,
|
||||
`CREATE INDEX golem15_seo_issues_fingerprint ON golem15_seo_issues (fingerprint)`,
|
||||
`CREATE TABLE golem15_seo_issue_ignores (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
fingerprint TEXT NOT NULL,
|
||||
admin_id BIGINT NULL,
|
||||
ignored_at TIMESTAMPTZ NULL
|
||||
)`,
|
||||
`CREATE UNIQUE INDEX golem15_seo_issue_ignores_fingerprint ON golem15_seo_issue_ignores (fingerprint)`,
|
||||
} {
|
||||
if err := tx.Exec(stmt).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Exec(`DROP TABLE IF EXISTS golem15_seo_issue_ignores, golem15_seo_issues, golem15_seo_crawl_links, golem15_seo_crawl_pages, golem15_seo_crawls`).Error
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
Register(createCrawls...)
|
||||
}
|
||||
Reference in New Issue
Block a user