feat: crawler finds a page with two H1 headings and stores the issue (tracer)
- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37) - classes/crawl: fetchguard client on the site host and its www twin, sitemap index and robots.txt seeds, two workers with a delay, limits - h1_missing and h1_multiple with the P33-38 fix route - POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues answers the stored issues with id, severity, source, fix, found_at, ignored and crawl_id
This commit is contained in:
Jakub Zych
293
classes/crawl/parse.go
Normal file
293
classes/crawl/parse.go
Normal file
@@ -0,0 +1,293 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/net/html/atom"
|
||||
)
|
||||
|
||||
// Heading is one heading of a page in document order.
|
||||
type Heading struct {
|
||||
Level int `json:"level"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// Link is one <a href> of a page, resolved against the page's address.
|
||||
type Link struct {
|
||||
URL *url.URL
|
||||
Href string
|
||||
Text string
|
||||
Rel string
|
||||
}
|
||||
|
||||
// Image is one <img> of a page.
|
||||
type Image struct {
|
||||
Src string
|
||||
Alt string
|
||||
HasAlt bool
|
||||
Width string
|
||||
Height string
|
||||
Loading string
|
||||
}
|
||||
|
||||
// Alternate is one <link rel="alternate" hreflang>.
|
||||
type Alternate struct {
|
||||
Lang string
|
||||
URL *url.URL
|
||||
Href string
|
||||
}
|
||||
|
||||
// PageFacts is what one HTML document says, parsed with
|
||||
// golang.org/x/net/html (no regular expressions over HTML): the language,
|
||||
// title, meta description, canonical, meta robots, og:image, headings,
|
||||
// links, images, JSON-LD scripts, hreflang alternates and the main text
|
||||
// with its word count. The rendered comparison parses the JavaScript
|
||||
// rendered DOM into the same facts.
|
||||
type PageFacts struct {
|
||||
Lang string
|
||||
Title string
|
||||
HasTitle bool
|
||||
Description string
|
||||
HasDescription bool
|
||||
Canonical string
|
||||
Robots string
|
||||
OGImage string
|
||||
Headings []Heading
|
||||
Links []Link
|
||||
Images []Image
|
||||
JSONLD []string
|
||||
Alternates []Alternate
|
||||
Text string
|
||||
Words int
|
||||
}
|
||||
|
||||
// H1 returns the texts of the page's level-1 headings.
|
||||
func (p *PageFacts) H1() []string {
|
||||
var out []string
|
||||
for _, h := range p.Headings {
|
||||
if h.Level == 1 {
|
||||
out = append(out, h.Text)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Noindex reports whether the meta robots value says noindex.
|
||||
func (p *PageFacts) Noindex() bool { return hasNoindex(p.Robots) }
|
||||
|
||||
// hasNoindex reports whether a robots value contains noindex or none.
|
||||
func hasNoindex(v string) bool {
|
||||
for _, part := range strings.FieldsFunc(strings.ToLower(v), func(r rune) bool { return r == ',' || r == ' ' || r == ';' }) {
|
||||
if part == "noindex" || part == "none" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Parse reads the facts of an HTML document fetched from base.
|
||||
func Parse(base *url.URL, body []byte) *PageFacts {
|
||||
out := &PageFacts{}
|
||||
doc, err := html.Parse(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
resolve := func(href string) *url.URL {
|
||||
href = strings.TrimSpace(href)
|
||||
if href == "" {
|
||||
return nil
|
||||
}
|
||||
ref, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if base == nil {
|
||||
return ref
|
||||
}
|
||||
u := base.ResolveReference(ref)
|
||||
u.Fragment = ""
|
||||
return u
|
||||
}
|
||||
var main *html.Node
|
||||
var bodyNode *html.Node
|
||||
var walk func(n *html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.DataAtom {
|
||||
case atom.Html:
|
||||
out.Lang = strings.TrimSpace(attr(n, "lang"))
|
||||
case atom.Title:
|
||||
if !out.HasTitle {
|
||||
out.HasTitle = true
|
||||
out.Title = collapse(textOf(n))
|
||||
}
|
||||
case atom.Meta:
|
||||
name := strings.ToLower(attr(n, "name"))
|
||||
prop := strings.ToLower(attr(n, "property"))
|
||||
switch {
|
||||
case name == "description" && !out.HasDescription:
|
||||
out.HasDescription = true
|
||||
out.Description = collapse(attr(n, "content"))
|
||||
case name == "robots" || name == "googlebot":
|
||||
if v := strings.TrimSpace(attr(n, "content")); v != "" {
|
||||
if out.Robots != "" {
|
||||
out.Robots += ", "
|
||||
}
|
||||
out.Robots += v
|
||||
}
|
||||
case prop == "og:image" && out.OGImage == "":
|
||||
if u := resolve(attr(n, "content")); u != nil {
|
||||
out.OGImage = u.String()
|
||||
}
|
||||
}
|
||||
case atom.Link:
|
||||
rel := strings.ToLower(attr(n, "rel"))
|
||||
switch {
|
||||
case relHas(rel, "canonical") && out.Canonical == "":
|
||||
if u := resolve(attr(n, "href")); u != nil {
|
||||
out.Canonical = u.String()
|
||||
}
|
||||
case relHas(rel, "alternate") && hasAttr(n, "hreflang"):
|
||||
href := attr(n, "href")
|
||||
out.Alternates = append(out.Alternates, Alternate{Lang: strings.TrimSpace(attr(n, "hreflang")), URL: resolve(href), Href: href})
|
||||
}
|
||||
case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6:
|
||||
out.Headings = append(out.Headings, Heading{Level: int(n.Data[1] - '0'), Text: collapse(textOf(n))})
|
||||
case atom.A:
|
||||
if hasAttr(n, "href") && len(out.Links) < MaxLinksPerPage {
|
||||
href := attr(n, "href")
|
||||
low := strings.ToLower(strings.TrimSpace(href))
|
||||
if !strings.HasPrefix(low, "javascript:") && !strings.HasPrefix(low, "mailto:") && !strings.HasPrefix(low, "tel:") && !strings.HasPrefix(low, "#") {
|
||||
if u := resolve(href); u != nil {
|
||||
out.Links = append(out.Links, Link{URL: u, Href: href, Text: collapse(textOf(n)), Rel: strings.ToLower(attr(n, "rel"))})
|
||||
}
|
||||
}
|
||||
}
|
||||
case atom.Img:
|
||||
alt, hasAlt := attrOK(n, "alt")
|
||||
src := attr(n, "src")
|
||||
if u := resolve(src); u != nil {
|
||||
src = u.String()
|
||||
}
|
||||
out.Images = append(out.Images, Image{Src: src, Alt: alt, HasAlt: hasAlt, Width: attr(n, "width"), Height: attr(n, "height"), Loading: strings.ToLower(attr(n, "loading"))})
|
||||
case atom.Script:
|
||||
if strings.EqualFold(strings.TrimSpace(attr(n, "type")), "application/ld+json") {
|
||||
out.JSONLD = append(out.JSONLD, textOf(n))
|
||||
}
|
||||
return
|
||||
case atom.Main:
|
||||
if main == nil {
|
||||
main = n
|
||||
}
|
||||
case atom.Body:
|
||||
bodyNode = n
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(doc)
|
||||
root := main
|
||||
if root == nil {
|
||||
root = bodyNode
|
||||
}
|
||||
if root != nil {
|
||||
var b strings.Builder
|
||||
visibleText(root, &b)
|
||||
out.Text = collapse(b.String())
|
||||
if len(out.Text) > maxText {
|
||||
out.Text = truncateUTF8(out.Text, maxText)
|
||||
}
|
||||
out.Words = len(strings.Fields(out.Text))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// maxText caps the main text kept per page (characters of UTF-8, bytes
|
||||
// bound).
|
||||
const maxText = 100_000
|
||||
|
||||
func truncateUTF8(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
for n > 0 && n < len(s) && (s[n]&0xC0) == 0x80 {
|
||||
n--
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
// visibleText writes the text of n without script, style, noscript and
|
||||
// template content, one space between block parts.
|
||||
func visibleText(n *html.Node, b *strings.Builder) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.DataAtom {
|
||||
case atom.Script, atom.Style, atom.Noscript, atom.Template, atom.Svg:
|
||||
return
|
||||
}
|
||||
}
|
||||
if n.Type == html.TextNode {
|
||||
b.WriteString(n.Data)
|
||||
b.WriteByte(' ')
|
||||
return
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
visibleText(c, b)
|
||||
}
|
||||
}
|
||||
|
||||
func textOf(n *html.Node) string {
|
||||
var b strings.Builder
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.TextNode {
|
||||
b.WriteString(n.Data)
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
if n.Type == html.ElementNode && n.DataAtom == atom.Img {
|
||||
if alt := attr(n, "alt"); alt != "" {
|
||||
b.WriteString(alt)
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
func collapse(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
|
||||
func attrOK(n *html.Node, key string) (string, bool) {
|
||||
for _, a := range n.Attr {
|
||||
if strings.EqualFold(a.Key, key) {
|
||||
return a.Val, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func attr(n *html.Node, key string) string {
|
||||
v, _ := attrOK(n, key)
|
||||
return v
|
||||
}
|
||||
|
||||
func hasAttr(n *html.Node, key string) bool {
|
||||
_, ok := attrOK(n, key)
|
||||
return ok
|
||||
}
|
||||
|
||||
func relHas(rel, want string) bool {
|
||||
for _, part := range strings.Fields(rel) {
|
||||
if part == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user