sm-seo-plugin/classes/crawl/crawler.go

package crawl

import (
	"bytes"
	"context"
	"encoding/xml"
	"errors"
	"io"
	"net/url"
	"strings"
	"sync"
	"time"
)

// ErrNoSiteURL is a crawl started without a site address
// (golem15.seo.site_url, else app.url).
var ErrNoSiteURL = errors.New("golem15.seo: the site address for crawls is not configured")

// The paths the crawler seeds from (RESEARCH Pattern 7).
const (
	HomePath    = "/"
	SitemapPath = "/sitemap.xml"
	RobotsPath  = "/robots.txt"
)

// maxSitemapFiles caps the sitemap files read from an index.
const maxSitemapFiles = 500

// How a page entered the frontier.
const (
	viaHome      = "home"
	viaSitemap   = "sitemap"
	viaLink      = "link"
	viaRedirect  = "redirect"
	viaCanonical = "canonical"
	viaAlternate = "alternate"
)

// Page is one fetched address of a crawl: the answer, the facts of its raw
// HTML, the facts of its JavaScript-rendered DOM when the comparison ran,
// its depth from the home page and the sitemap file that lists it.
type Page struct {
	Key         string
	Depth       int
	Via         string
	Resp        *Response
	Facts       *PageFacts
	Rendered    *PageFacts
	InSitemap   bool
	SitemapFile string
	FetchedAt   time.Time
}

// OK reports whether the page answered 200 with HTML.
func (p *Page) OK() bool {
	return p != nil && p.Resp != nil && p.Resp.Err == "" && p.Resp.Status == 200 && p.Facts != nil
}

// SitemapFile is one sitemap the crawl read: its address, status and the
// page addresses it lists.
type SitemapFile struct {
	Key    string
	Status int
	URLs   []string
}

// RenderState is the outcome of the rendered comparison (P33-55).
type RenderState struct {
	Status string
	Pages  int
	Note   string
}

// Result is what one crawl fetched, in frontier order.
type Result struct {
	Pages    map[string]*Page
	Order    []string
	Sitemaps []SitemapFile
	// Listed maps a page address to the sitemap file listing it.
	Listed       map[string]string
	Robots       string
	RobotsStatus int
	LimitHit     bool
	Canceled     bool
	Render       RenderState
	fetcher      *Fetcher
}

// Page returns the fetched page at key, nil when none.
func (r *Result) Page(key string) *Page {
	if r == nil {
		return nil
	}
	return r.Pages[key]
}

// Options are the limits and seams of one crawl.
type Options struct {
	// MaxPages caps the fetched addresses (D-14, default 5000).
	MaxPages int
	// Delay is the pause between two requests (default 250 ms).
	Delay time.Duration
	// Renderer renders pages for the comparison; nil skips it.
	Renderer Renderer
	// RenderMax caps the rendered pages (default 200, crawl order).
	RenderMax int
	// RenderErr is why the renderer could not be built: the comparison is
	// recorded as failed with it, and the raw checks run.
	RenderErr error
	// Canceled is asked between requests; true stops the crawl and keeps
	// what was fetched.
	Canceled func(ctx context.Context) bool
	// Progress is told the number of fetched pages.
	Progress func(ctx context.Context, fetched int)
}

// item is one frontier entry.
type item struct {
	key   string
	depth int
	via   string
}

// pacer spaces requests at least delay apart across the workers.
type pacer struct {
	mu    sync.Mutex
	next  time.Time
	delay time.Duration
}

func (p *pacer) wait(ctx context.Context) error {
	p.mu.Lock()
	now := time.Now()
	at := p.next
	if at.Before(now) {
		at = now
	}
	p.next = at.Add(p.delay)
	p.mu.Unlock()
	if d := time.Until(at); d > 0 {
		t := time.NewTimer(d)
		defer t.Stop()
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-t.C:
		}
	}
	return nil
}

// Crawl fetches the site through f, a fetchguard client limited to the
// site host and its www twin (fetchguard.AllowHostsMode, HTTPS, no
// redirects followed; TrustedMode only from Go config): robots.txt, the
// sitemap index and its files, then breadth-first from the home page and
// the sitemap addresses over same-host links, redirect targets, canonicals
// and hreflang alternates, two requests at a time with the delay between
// requests, up to MaxPages. robots.txt rules are read for the report, not
// obeyed (P33-41). Cancelling keeps what was fetched. After each HTML page
// answering 200, a configured Renderer renders it (one tab at a time, up
// to RenderMax pages) for the raw versus rendered comparison.
func Crawl(ctx context.Context, f *Fetcher, opt Options) (*Result, error) {
	if opt.MaxPages <= 0 {
		opt.MaxPages = 5000
	}
	if opt.RenderMax <= 0 {
		opt.RenderMax = DefaultRenderMax
	}
	res := &Result{Pages: map[string]*Page{}, Listed: map[string]string{}, Render: RenderState{Status: renderStatusSkipped}, fetcher: f}
	switch {
	case opt.Renderer != nil:
		res.Render.Status = renderStatusDone
	case opt.RenderErr != nil:
		res.Render = RenderState{Status: renderStatusFailed, Note: opt.RenderErr.Error()}
	}
	pace := &pacer{delay: opt.Delay}
	canceled := func() bool {
		if ctx.Err() != nil {
			return true
		}
		return opt.Canceled != nil && opt.Canceled(ctx)
	}

	// robots.txt, for the sitemap_blocked check.
	if err := pace.wait(ctx); err != nil {
		return res, err
	}
	if r := f.Get(ctx, RobotsPath); r.Err == "" {
		res.RobotsStatus = r.Status
		if r.Status == 200 {
			res.Robots = string(r.Body)
		}
	}

	// The sitemap index and its files.
	if err := readSitemaps(ctx, f, pace, res); err != nil {
		return res, err
	}

	seen := map[string]bool{}
	var queue []item
	push := func(key string, depth int, via string) {
		if key == "" || seen[key] {
			return
		}
		seen[key] = true
		queue = append(queue, item{key: key, depth: depth, via: via})
	}
	push(HomePath, 0, viaHome)
	for _, sm := range res.Sitemaps {
		for _, key := range sm.URLs {
			push(key, 1, viaSitemap)
		}
	}

	var renderMu sync.Mutex
	rendered := 0
	renderOff := opt.Renderer == nil
	for len(queue) > 0 {
		if canceled() {
			res.Canceled = true
			break
		}
		if len(res.Order) >= opt.MaxPages {
			res.LimitHit = true
			break
		}
		n := Workers
		if room := opt.MaxPages - len(res.Order); n > room {
			n = room
		}
		if n > len(queue) {
			n = len(queue)
		}
		batch := queue[:n]
		queue = queue[n:]
		pages := make([]*Page, len(batch))
		var wg sync.WaitGroup
		for i, it := range batch {
			wg.Add(1)
			go func(i int, it item) {
				defer wg.Done()
				p := &Page{Key: it.key, Depth: it.depth, Via: it.via}
				if err := pace.wait(ctx); err != nil {
					p.Resp = &Response{Err: "canceled"}
					pages[i] = p
					return
				}
				p.Resp = f.Get(ctx, it.key)
				p.FetchedAt = time.Now().UTC()
				if p.Resp.Err == "" && p.Resp.Status == 200 && p.Resp.HTML() {
					base, _ := url.Parse(f.Absolute(it.key))
					p.Facts = Parse(base, p.Resp.Body)
					renderMu.Lock()
					if !renderOff && rendered < opt.RenderMax && ctx.Err() == nil {
						rendered++
						dom, err := opt.Renderer.Render(ctx, f.Absolute(it.key))
						if err != nil {
							renderOff = true
							rendered--
							res.Render = RenderState{Status: renderStatusFailed, Note: err.Error()}
						} else {
							p.Rendered = Parse(base, []byte(dom))
						}
					}
					renderMu.Unlock()
				}
				pages[i] = p
			}(i, it)
		}
		wg.Wait()
		for _, p := range pages {
			if file, ok := res.Listed[p.Key]; ok {
				p.InSitemap, p.SitemapFile = true, file
			}
			res.Pages[p.Key] = p
			res.Order = append(res.Order, p.Key)
			p.Resp.Body = nil
			for _, next := range discovered(f, p) {
				push(next.key, p.Depth+1, next.via)
			}
		}
		if opt.Progress != nil {
			opt.Progress(ctx, len(res.Order))
		}
	}
	if len(queue) > 0 && !res.Canceled && len(res.Order) >= opt.MaxPages {
		res.LimitHit = true
	}
	res.Render.Pages = rendered
	return res, nil
}

// discovered lists the same-host addresses a fetched page leads to: its
// redirect target, its links, canonical and hreflang alternates.
func discovered(f *Fetcher, p *Page) []item {
	var out []item
	add := func(raw string, via string) {
		u, err := url.Parse(raw)
		if err != nil {
			return
		}
		u.Fragment = ""
		if key := f.Key(u); key != "" {
			out = append(out, item{key: key, via: via})
		}
	}
	if p.Resp != nil && p.Resp.Location != "" {
		add(p.Resp.Location, viaRedirect)
	}
	if p.Facts == nil {
		return out
	}
	for _, l := range p.Facts.Links {
		if l.URL != nil {
			add(l.URL.String(), viaLink)
		}
	}
	if p.Facts.Canonical != "" {
		add(p.Facts.Canonical, viaCanonical)
	}
	for _, a := range p.Facts.Alternates {
		if a.URL != nil {
			add(a.URL.String(), viaAlternate)
		}
	}
	return out
}

// readSitemaps reads /sitemap.xml: an index's files (same host only) or a
// single urlset.
func readSitemaps(ctx context.Context, f *Fetcher, pace *pacer, res *Result) error {
	if err := pace.wait(ctx); err != nil {
		return err
	}
	r := f.Get(ctx, SitemapPath)
	if r.Err != "" || r.Status != 200 {
		res.Sitemaps = append(res.Sitemaps, SitemapFile{Key: SitemapPath, Status: r.Status})
		return nil
	}
	index, locs := parseSitemap(r.Body)
	if !index {
		res.addSitemap(f, SitemapFile{Key: SitemapPath, Status: 200}, locs)
		return nil
	}
	for i, loc := range locs {
		if i >= maxSitemapFiles {
			break
		}
		u, err := url.Parse(loc)
		if err != nil {
			continue
		}
		key := f.Key(u)
		if key == "" {
			continue
		}
		if err := pace.wait(ctx); err != nil {
			return err
		}
		sr := f.Get(ctx, key)
		file := SitemapFile{Key: key, Status: sr.Status}
		if sr.Err != "" || sr.Status != 200 {
			res.Sitemaps = append(res.Sitemaps, file)
			continue
		}
		_, urls := parseSitemap(sr.Body)
		res.addSitemap(f, file, urls)
	}
	return nil
}

func (res *Result) addSitemap(f *Fetcher, file SitemapFile, locs []string) {
	for _, loc := range locs {
		u, err := url.Parse(loc)
		if err != nil {
			continue
		}
		key := f.Key(u)
		if key == "" {
			continue
		}
		if _, dup := res.Listed[key]; !dup {
			res.Listed[key] = file.Key
		}
		file.URLs = append(file.URLs, key)
	}
	res.Sitemaps = append(res.Sitemaps, file)
}

// parseSitemap reads the <loc> values of a sitemap document; index is true
// for a <sitemapindex>.
func parseSitemap(body []byte) (index bool, locs []string) {
	dec := xml.NewDecoder(io.LimitReader(bytes.NewReader(body), MaxBody))
	depth := 0
	inLoc := false
	var parent string
	var stack []string
	for {
		tok, err := dec.Token()
		if err != nil {
			break
		}
		switch t := tok.(type) {
		case xml.StartElement:
			depth++
			name := strings.ToLower(t.Name.Local)
			if depth == 1 && name == "sitemapindex" {
				index = true
			}
			stack = append(stack, name)
			if name == "loc" && len(stack) >= 2 {
				parent = stack[len(stack)-2]
				inLoc = parent == "url" || parent == "sitemap"
			}
		case xml.EndElement:
			depth--
			if len(stack) > 0 {
				stack = stack[:len(stack)-1]
			}
			inLoc = false
		case xml.CharData:
			if inLoc {
				if s := strings.TrimSpace(string(t)); s != "" {
					locs = append(locs, s)
				}
			}
		}
	}
	return index, locs
}