sm-seo-plugin/classes/crawl/parse.go

package crawl

import (
	"bytes"
	"net/url"
	"strings"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

// Heading is one heading of a page in document order.
type Heading struct {
	Level int    `json:"level"`
	Text  string `json:"text"`
}

// Link is one <a href> of a page, resolved against the page's address.
type Link struct {
	URL  *url.URL
	Href string
	Text string
	Rel  string
}

// Image is one <img> of a page.
type Image struct {
	Src     string
	Alt     string
	HasAlt  bool
	Width   string
	Height  string
	Loading string
}

// Alternate is one <link rel="alternate" hreflang>.
type Alternate struct {
	Lang string
	URL  *url.URL
	Href string
}

// PageFacts is what one HTML document says, parsed with
// golang.org/x/net/html (no regular expressions over HTML): the language,
// title, meta description, canonical, meta robots, og:image, headings,
// links, images, JSON-LD scripts, hreflang alternates and the main text
// with its word count. The rendered comparison parses the JavaScript
// rendered DOM into the same facts.
type PageFacts struct {
	Lang           string
	Title          string
	HasTitle       bool
	Description    string
	HasDescription bool
	Canonical      string
	Robots         string
	OGImage        string
	Headings       []Heading
	Links          []Link
	Images         []Image
	JSONLD         []string
	Alternates     []Alternate
	Text           string
	Words          int
}

// H1 returns the texts of the page's level-1 headings.
func (p *PageFacts) H1() []string {
	var out []string
	for _, h := range p.Headings {
		if h.Level == 1 {
			out = append(out, h.Text)
		}
	}
	return out
}

// Noindex reports whether the meta robots value says noindex.
func (p *PageFacts) Noindex() bool { return hasNoindex(p.Robots) }

// hasNoindex reports whether a robots value contains noindex or none.
func hasNoindex(v string) bool {
	for _, part := range strings.FieldsFunc(strings.ToLower(v), func(r rune) bool { return r == ',' || r == ' ' || r == ';' }) {
		if part == "noindex" || part == "none" {
			return true
		}
	}
	return false
}

// Parse reads the facts of an HTML document fetched from base.
func Parse(base *url.URL, body []byte) *PageFacts {
	out := &PageFacts{}
	doc, err := html.Parse(bytes.NewReader(body))
	if err != nil {
		return out
	}
	resolve := func(href string) *url.URL {
		href = strings.TrimSpace(href)
		if href == "" {
			return nil
		}
		ref, err := url.Parse(href)
		if err != nil {
			return nil
		}
		if base == nil {
			return ref
		}
		u := base.ResolveReference(ref)
		u.Fragment = ""
		return u
	}
	var main *html.Node
	var bodyNode *html.Node
	var walk func(n *html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode {
			switch n.DataAtom {
			case atom.Html:
				out.Lang = strings.TrimSpace(attr(n, "lang"))
			case atom.Title:
				if !out.HasTitle {
					out.HasTitle = true
					out.Title = collapse(textOf(n))
				}
			case atom.Meta:
				name := strings.ToLower(attr(n, "name"))
				prop := strings.ToLower(attr(n, "property"))
				switch {
				case name == "description" && !out.HasDescription:
					out.HasDescription = true
					out.Description = collapse(attr(n, "content"))
				case name == "robots" || name == "googlebot":
					if v := strings.TrimSpace(attr(n, "content")); v != "" {
						if out.Robots != "" {
							out.Robots += ", "
						}
						out.Robots += v
					}
				case prop == "og:image" && out.OGImage == "":
					if u := resolve(attr(n, "content")); u != nil {
						out.OGImage = u.String()
					}
				}
			case atom.Link:
				rel := strings.ToLower(attr(n, "rel"))
				switch {
				case relHas(rel, "canonical") && out.Canonical == "":
					if u := resolve(attr(n, "href")); u != nil {
						out.Canonical = u.String()
					}
				case relHas(rel, "alternate") && hasAttr(n, "hreflang"):
					href := attr(n, "href")
					out.Alternates = append(out.Alternates, Alternate{Lang: strings.TrimSpace(attr(n, "hreflang")), URL: resolve(href), Href: href})
				}
			case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6:
				out.Headings = append(out.Headings, Heading{Level: int(n.Data[1] - '0'), Text: collapse(textOf(n))})
			case atom.A:
				if hasAttr(n, "href") && len(out.Links) < MaxLinksPerPage {
					href := attr(n, "href")
					low := strings.ToLower(strings.TrimSpace(href))
					if !strings.HasPrefix(low, "javascript:") && !strings.HasPrefix(low, "mailto:") && !strings.HasPrefix(low, "tel:") && !strings.HasPrefix(low, "#") {
						if u := resolve(href); u != nil {
							out.Links = append(out.Links, Link{URL: u, Href: href, Text: collapse(textOf(n)), Rel: strings.ToLower(attr(n, "rel"))})
						}
					}
				}
			case atom.Img:
				alt, hasAlt := attrOK(n, "alt")
				src := attr(n, "src")
				if u := resolve(src); u != nil {
					src = u.String()
				}
				out.Images = append(out.Images, Image{Src: src, Alt: alt, HasAlt: hasAlt, Width: attr(n, "width"), Height: attr(n, "height"), Loading: strings.ToLower(attr(n, "loading"))})
			case atom.Script:
				if strings.EqualFold(strings.TrimSpace(attr(n, "type")), "application/ld+json") {
					out.JSONLD = append(out.JSONLD, textOf(n))
				}
				return
			case atom.Main:
				if main == nil {
					main = n
				}
			case atom.Body:
				bodyNode = n
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(doc)
	root := main
	if root == nil {
		root = bodyNode
	}
	if root != nil {
		var b strings.Builder
		visibleText(root, &b)
		out.Text = collapse(b.String())
		if len(out.Text) > maxText {
			out.Text = truncateUTF8(out.Text, maxText)
		}
		out.Words = len(strings.Fields(out.Text))
	}
	return out
}

// maxText caps the main text kept per page (characters of UTF-8, bytes
// bound).
const maxText = 100_000

func truncateUTF8(s string, n int) string {
	if len(s) <= n {
		return s
	}
	for n > 0 && n < len(s) && (s[n]&0xC0) == 0x80 {
		n--
	}
	return s[:n]
}

// visibleText writes the text of n without script, style, noscript and
// template content, one space between block parts.
func visibleText(n *html.Node, b *strings.Builder) {
	if n.Type == html.ElementNode {
		switch n.DataAtom {
		case atom.Script, atom.Style, atom.Noscript, atom.Template, atom.Svg:
			return
		}
	}
	if n.Type == html.TextNode {
		b.WriteString(n.Data)
		b.WriteByte(' ')
		return
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		visibleText(c, b)
	}
}

func textOf(n *html.Node) string {
	var b strings.Builder
	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.TextNode {
			b.WriteString(n.Data)
			b.WriteByte(' ')
		}
		if n.Type == html.ElementNode && n.DataAtom == atom.Img {
			if alt := attr(n, "alt"); alt != "" {
				b.WriteString(alt)
				b.WriteByte(' ')
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(n)
	return strings.TrimSpace(b.String())
}

func collapse(s string) string { return strings.Join(strings.Fields(s), " ") }

func attrOK(n *html.Node, key string) (string, bool) {
	for _, a := range n.Attr {
		if strings.EqualFold(a.Key, key) {
			return a.Val, true
		}
	}
	return "", false
}

func attr(n *html.Node, key string) string {
	v, _ := attrOK(n, key)
	return v
}

func hasAttr(n *html.Node, key string) bool {
	_, ok := attrOK(n, key)
	return ok
}

func relHas(rel, want string) bool {
	for _, part := range strings.Fields(rel) {
		if part == want {
			return true
		}
	}
	return false
}