d7597bb1a5
The crawl job's context ends on a cancel or at the job timeout. Storing the pages, links and issues now runs on a context detached from it with its own 10-minute bound, a timeout ends the crawl failed with a time-limit error, a LoadLimits failure falls back to the defaults, and a fetch cut short by the cancellation is recorded as canceled.
436 lines
11 KiB
Go
436 lines
11 KiB
Go
package crawl
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/xml"
|
|
"errors"
|
|
"io"
|
|
"net/url"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// ErrNoSiteURL is a crawl started without a site address
|
|
// (golem15.seo.site_url, else app.url).
|
|
var ErrNoSiteURL = errors.New("golem15.seo: the site address for crawls is not configured")
|
|
|
|
// The paths the crawler seeds from (RESEARCH Pattern 7).
|
|
const (
|
|
HomePath = "/"
|
|
SitemapPath = "/sitemap.xml"
|
|
RobotsPath = "/robots.txt"
|
|
)
|
|
|
|
// maxSitemapFiles caps the sitemap files read from an index.
|
|
const maxSitemapFiles = 500
|
|
|
|
// How a page entered the frontier.
|
|
const (
|
|
viaHome = "home"
|
|
viaSitemap = "sitemap"
|
|
viaLink = "link"
|
|
viaRedirect = "redirect"
|
|
viaCanonical = "canonical"
|
|
viaAlternate = "alternate"
|
|
)
|
|
|
|
// Page is one fetched address of a crawl: the answer, the facts of its raw
|
|
// HTML, the facts of its JavaScript-rendered DOM when the comparison ran,
|
|
// its depth from the home page and the sitemap file that lists it.
|
|
type Page struct {
|
|
Key string
|
|
Depth int
|
|
Via string
|
|
Resp *Response
|
|
Facts *PageFacts
|
|
Rendered *PageFacts
|
|
InSitemap bool
|
|
SitemapFile string
|
|
FetchedAt time.Time
|
|
}
|
|
|
|
// OK reports whether the page answered 200 with HTML.
|
|
func (p *Page) OK() bool {
|
|
return p != nil && p.Resp != nil && p.Resp.Err == "" && p.Resp.Status == 200 && p.Facts != nil
|
|
}
|
|
|
|
// SitemapFile is one sitemap the crawl read: its address, status and the
|
|
// page addresses it lists.
|
|
type SitemapFile struct {
|
|
Key string
|
|
Status int
|
|
URLs []string
|
|
}
|
|
|
|
// RenderState is the outcome of the rendered comparison (P33-55).
|
|
type RenderState struct {
|
|
Status string
|
|
Pages int
|
|
Note string
|
|
}
|
|
|
|
// Result is what one crawl fetched, in frontier order.
|
|
type Result struct {
|
|
Pages map[string]*Page
|
|
Order []string
|
|
Sitemaps []SitemapFile
|
|
// Listed maps a page address to the sitemap file listing it.
|
|
Listed map[string]string
|
|
Robots string
|
|
RobotsStatus int
|
|
LimitHit bool
|
|
Canceled bool
|
|
Render RenderState
|
|
fetcher *Fetcher
|
|
}
|
|
|
|
// Page returns the fetched page at key, nil when none.
|
|
func (r *Result) Page(key string) *Page {
|
|
if r == nil {
|
|
return nil
|
|
}
|
|
return r.Pages[key]
|
|
}
|
|
|
|
// Options are the limits and seams of one crawl.
|
|
type Options struct {
|
|
// MaxPages caps the fetched addresses (D-14, default 5000).
|
|
MaxPages int
|
|
// Delay is the pause between two requests (default 250 ms).
|
|
Delay time.Duration
|
|
// Renderer renders pages for the comparison; nil skips it.
|
|
Renderer Renderer
|
|
// RenderMax caps the rendered pages (default 200, crawl order).
|
|
RenderMax int
|
|
// RenderErr is why the renderer could not be built: the comparison is
|
|
// recorded as failed with it, and the raw checks run.
|
|
RenderErr error
|
|
// Canceled is asked between requests; true stops the crawl and keeps
|
|
// what was fetched.
|
|
Canceled func(ctx context.Context) bool
|
|
// Progress is told the number of fetched pages.
|
|
Progress func(ctx context.Context, fetched int)
|
|
}
|
|
|
|
// item is one frontier entry.
|
|
type item struct {
|
|
key string
|
|
depth int
|
|
via string
|
|
}
|
|
|
|
// pacer spaces requests at least delay apart across the workers.
|
|
type pacer struct {
|
|
mu sync.Mutex
|
|
next time.Time
|
|
delay time.Duration
|
|
}
|
|
|
|
func (p *pacer) wait(ctx context.Context) error {
|
|
p.mu.Lock()
|
|
now := time.Now()
|
|
at := p.next
|
|
if at.Before(now) {
|
|
at = now
|
|
}
|
|
p.next = at.Add(p.delay)
|
|
p.mu.Unlock()
|
|
if d := time.Until(at); d > 0 {
|
|
t := time.NewTimer(d)
|
|
defer t.Stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-t.C:
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Crawl fetches the site through f, a fetchguard client limited to the
|
|
// site host and its www twin (fetchguard.AllowHostsMode, HTTPS, no
|
|
// redirects followed; TrustedMode only from Go config): robots.txt, the
|
|
// sitemap index and its files, then breadth-first from the home page and
|
|
// the sitemap addresses over same-host links, redirect targets, canonicals
|
|
// and hreflang alternates, two requests at a time with the delay between
|
|
// requests, up to MaxPages. robots.txt rules are read for the report, not
|
|
// obeyed (P33-41). Cancelling keeps what was fetched. After each HTML page
|
|
// answering 200, a configured Renderer renders it (one tab at a time, up
|
|
// to RenderMax pages) for the raw versus rendered comparison.
|
|
func Crawl(ctx context.Context, f *Fetcher, opt Options) (*Result, error) {
|
|
if opt.MaxPages <= 0 {
|
|
opt.MaxPages = 5000
|
|
}
|
|
if opt.RenderMax <= 0 {
|
|
opt.RenderMax = DefaultRenderMax
|
|
}
|
|
res := &Result{Pages: map[string]*Page{}, Listed: map[string]string{}, Render: RenderState{Status: renderStatusSkipped}, fetcher: f}
|
|
switch {
|
|
case opt.Renderer != nil:
|
|
res.Render.Status = renderStatusDone
|
|
case opt.RenderErr != nil:
|
|
res.Render = RenderState{Status: renderStatusFailed, Note: opt.RenderErr.Error()}
|
|
}
|
|
pace := &pacer{delay: opt.Delay}
|
|
canceled := func() bool {
|
|
if ctx.Err() != nil {
|
|
return true
|
|
}
|
|
return opt.Canceled != nil && opt.Canceled(ctx)
|
|
}
|
|
|
|
// robots.txt, for the sitemap_blocked check.
|
|
if err := pace.wait(ctx); err != nil {
|
|
return res, err
|
|
}
|
|
if r := f.Get(ctx, RobotsPath); r.Err == "" {
|
|
res.RobotsStatus = r.Status
|
|
if r.Status == 200 {
|
|
res.Robots = string(r.Body)
|
|
}
|
|
}
|
|
|
|
// The sitemap index and its files.
|
|
if err := readSitemaps(ctx, f, pace, res); err != nil {
|
|
return res, err
|
|
}
|
|
|
|
seen := map[string]bool{}
|
|
var queue []item
|
|
push := func(key string, depth int, via string) {
|
|
if key == "" || seen[key] {
|
|
return
|
|
}
|
|
seen[key] = true
|
|
queue = append(queue, item{key: key, depth: depth, via: via})
|
|
}
|
|
push(HomePath, 0, viaHome)
|
|
for _, sm := range res.Sitemaps {
|
|
for _, key := range sm.URLs {
|
|
push(key, 1, viaSitemap)
|
|
}
|
|
}
|
|
|
|
var renderMu sync.Mutex
|
|
rendered := 0
|
|
renderOff := opt.Renderer == nil
|
|
for len(queue) > 0 {
|
|
if canceled() {
|
|
res.Canceled = true
|
|
break
|
|
}
|
|
if len(res.Order) >= opt.MaxPages {
|
|
res.LimitHit = true
|
|
break
|
|
}
|
|
n := Workers
|
|
if room := opt.MaxPages - len(res.Order); n > room {
|
|
n = room
|
|
}
|
|
if n > len(queue) {
|
|
n = len(queue)
|
|
}
|
|
batch := queue[:n]
|
|
queue = queue[n:]
|
|
pages := make([]*Page, len(batch))
|
|
var wg sync.WaitGroup
|
|
for i, it := range batch {
|
|
wg.Add(1)
|
|
go func(i int, it item) {
|
|
defer wg.Done()
|
|
p := &Page{Key: it.key, Depth: it.depth, Via: it.via}
|
|
if err := pace.wait(ctx); err != nil {
|
|
p.Resp = &Response{Err: "canceled"}
|
|
pages[i] = p
|
|
return
|
|
}
|
|
p.Resp = f.Get(ctx, it.key)
|
|
if p.Resp.Err != "" && ctx.Err() != nil {
|
|
// The fetch was cut short by the cancellation.
|
|
p.Resp = &Response{Err: "canceled"}
|
|
}
|
|
p.FetchedAt = time.Now().UTC()
|
|
if p.Resp.Err == "" && p.Resp.Status == 200 && p.Resp.HTML() {
|
|
base, _ := url.Parse(f.Absolute(it.key))
|
|
p.Facts = Parse(base, p.Resp.Body)
|
|
renderMu.Lock()
|
|
if !renderOff && rendered < opt.RenderMax && ctx.Err() == nil {
|
|
rendered++
|
|
dom, err := opt.Renderer.Render(ctx, f.Absolute(it.key))
|
|
if err != nil {
|
|
renderOff = true
|
|
rendered--
|
|
res.Render = RenderState{Status: renderStatusFailed, Note: err.Error()}
|
|
} else {
|
|
p.Rendered = Parse(base, []byte(dom))
|
|
}
|
|
}
|
|
renderMu.Unlock()
|
|
}
|
|
pages[i] = p
|
|
}(i, it)
|
|
}
|
|
wg.Wait()
|
|
for _, p := range pages {
|
|
if file, ok := res.Listed[p.Key]; ok {
|
|
p.InSitemap, p.SitemapFile = true, file
|
|
}
|
|
res.Pages[p.Key] = p
|
|
res.Order = append(res.Order, p.Key)
|
|
p.Resp.Body = nil
|
|
for _, next := range discovered(f, p) {
|
|
push(next.key, p.Depth+1, next.via)
|
|
}
|
|
}
|
|
if opt.Progress != nil {
|
|
opt.Progress(ctx, len(res.Order))
|
|
}
|
|
}
|
|
if len(queue) > 0 && !res.Canceled && len(res.Order) >= opt.MaxPages {
|
|
res.LimitHit = true
|
|
}
|
|
res.Render.Pages = rendered
|
|
return res, nil
|
|
}
|
|
|
|
// discovered lists the same-host addresses a fetched page leads to: its
|
|
// redirect target, its links, canonical and hreflang alternates.
|
|
func discovered(f *Fetcher, p *Page) []item {
|
|
var out []item
|
|
add := func(raw string, via string) {
|
|
u, err := url.Parse(raw)
|
|
if err != nil {
|
|
return
|
|
}
|
|
u.Fragment = ""
|
|
if key := f.Key(u); key != "" {
|
|
out = append(out, item{key: key, via: via})
|
|
}
|
|
}
|
|
if p.Resp != nil && p.Resp.Location != "" {
|
|
add(p.Resp.Location, viaRedirect)
|
|
}
|
|
if p.Facts == nil {
|
|
return out
|
|
}
|
|
for _, l := range p.Facts.Links {
|
|
if l.URL != nil {
|
|
add(l.URL.String(), viaLink)
|
|
}
|
|
}
|
|
if p.Facts.Canonical != "" {
|
|
add(p.Facts.Canonical, viaCanonical)
|
|
}
|
|
for _, a := range p.Facts.Alternates {
|
|
if a.URL != nil {
|
|
add(a.URL.String(), viaAlternate)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// readSitemaps reads /sitemap.xml: an index's files (same host only) or a
|
|
// single urlset.
|
|
func readSitemaps(ctx context.Context, f *Fetcher, pace *pacer, res *Result) error {
|
|
if err := pace.wait(ctx); err != nil {
|
|
return err
|
|
}
|
|
r := f.Get(ctx, SitemapPath)
|
|
if r.Err != "" || r.Status != 200 {
|
|
res.Sitemaps = append(res.Sitemaps, SitemapFile{Key: SitemapPath, Status: r.Status})
|
|
return nil
|
|
}
|
|
index, locs := parseSitemap(r.Body)
|
|
if !index {
|
|
res.addSitemap(f, SitemapFile{Key: SitemapPath, Status: 200}, locs)
|
|
return nil
|
|
}
|
|
for i, loc := range locs {
|
|
if i >= maxSitemapFiles {
|
|
break
|
|
}
|
|
u, err := url.Parse(loc)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
key := f.Key(u)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if err := pace.wait(ctx); err != nil {
|
|
return err
|
|
}
|
|
sr := f.Get(ctx, key)
|
|
file := SitemapFile{Key: key, Status: sr.Status}
|
|
if sr.Err != "" || sr.Status != 200 {
|
|
res.Sitemaps = append(res.Sitemaps, file)
|
|
continue
|
|
}
|
|
_, urls := parseSitemap(sr.Body)
|
|
res.addSitemap(f, file, urls)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (res *Result) addSitemap(f *Fetcher, file SitemapFile, locs []string) {
|
|
for _, loc := range locs {
|
|
u, err := url.Parse(loc)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
key := f.Key(u)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if _, dup := res.Listed[key]; !dup {
|
|
res.Listed[key] = file.Key
|
|
}
|
|
file.URLs = append(file.URLs, key)
|
|
}
|
|
res.Sitemaps = append(res.Sitemaps, file)
|
|
}
|
|
|
|
// parseSitemap reads the <loc> values of a sitemap document; index is true
|
|
// for a <sitemapindex>.
|
|
func parseSitemap(body []byte) (index bool, locs []string) {
|
|
dec := xml.NewDecoder(io.LimitReader(bytes.NewReader(body), MaxBody))
|
|
depth := 0
|
|
inLoc := false
|
|
var parent string
|
|
var stack []string
|
|
for {
|
|
tok, err := dec.Token()
|
|
if err != nil {
|
|
break
|
|
}
|
|
switch t := tok.(type) {
|
|
case xml.StartElement:
|
|
depth++
|
|
name := strings.ToLower(t.Name.Local)
|
|
if depth == 1 && name == "sitemapindex" {
|
|
index = true
|
|
}
|
|
stack = append(stack, name)
|
|
if name == "loc" && len(stack) >= 2 {
|
|
parent = stack[len(stack)-2]
|
|
inLoc = parent == "url" || parent == "sitemap"
|
|
}
|
|
case xml.EndElement:
|
|
depth--
|
|
if len(stack) > 0 {
|
|
stack = stack[:len(stack)-1]
|
|
}
|
|
inLoc = false
|
|
case xml.CharData:
|
|
if inLoc {
|
|
if s := strings.TrimSpace(string(t)); s != "" {
|
|
locs = append(locs, s)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return index, locs
|
|
}
|