edc03cae4b
- orphan_page for listed indexable pages no crawled page links to, with pages mentioning the title or focus keyphrase as suggestions (D-82) - CMS-data image_alt and h1_in_content from the content descriptors (TopHeading 2 only), stored with the crawl as source cms (D-85, D-50) - seo-page names Tools edit seo_update_page; every stored issue carries a tool or fix code with details.what (P33-38) - TestIssueMerge (ignored fingerprints survive the next crawl, fixed pages lose their issues) and TestStagingCrawl (Pitfall 12)
414 lines
13 KiB
Go
414 lines
13 KiB
Go
package classes
|
|
|
|
import (
|
|
"context"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
|
|
"git.golem15.com/golem15/sm-seo-plugin/models"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// The kinds of the CMS-data issue report (D-13, D-14 CMS part). The
|
|
// crawler's checks of the rendered pages are Phase 33.
|
|
const (
|
|
IssueMissingTitle = "missing_title"
|
|
IssueMissingDescription = "missing_description"
|
|
IssueDuplicateTitle = "duplicate_title"
|
|
IssueDuplicateDesc = "duplicate_description"
|
|
IssueRuleOverridesRecord = "rule_overrides_record"
|
|
IssueRedirectChain = "redirect_chain"
|
|
IssueCanonicalNoindex = "canonical_noindex"
|
|
IssueCanonicalRedirected = "canonical_redirected"
|
|
IssueNoindexBlocked = "noindex_blocked"
|
|
// IssueSitemapBlocked is a sitemap address the production robots.txt
|
|
// keeps Googlebot from (D-24): an error, since the sitemap asks for a
|
|
// page the crawler may not fetch.
|
|
IssueSitemapBlocked = "sitemap_blocked"
|
|
// IssueImageAlt is a record image without alternative text that is
|
|
// not marked decorative (D-85), from the content descriptors.
|
|
IssueImageAlt = "image_alt"
|
|
// IssueH1InContent is an H1 in the body of a record whose page renders
|
|
// the title as the only H1 (descriptor TopHeading 2, D-50; Journal's
|
|
// H1 setting on).
|
|
IssueH1InContent = "h1_in_content"
|
|
)
|
|
|
|
// The severities an issue may name.
|
|
const (
|
|
SeverityError = "error"
|
|
)
|
|
|
|
// Issue is one finding of the issue report: the address, the kind, the
|
|
// details of the finding and the MCP tool that fixes it.
|
|
type Issue struct {
|
|
Kind string `json:"kind"`
|
|
URL string `json:"url"`
|
|
Locale string `json:"locale,omitempty"`
|
|
Details map[string]any `json:"details,omitempty"`
|
|
Tool string `json:"tool"`
|
|
// Severity is set for kinds that are errors (sitemap_blocked); "" for
|
|
// the others.
|
|
Severity string `json:"severity,omitempty"`
|
|
}
|
|
|
|
// The tools the issues point to (pier prefixes them with seo_).
|
|
const (
|
|
toolSetURLSEO = "seo_set_url_seo"
|
|
toolUpdateRule = "seo_update_rule"
|
|
toolUpdateRedirect = "seo_update_redirect"
|
|
toolUpdateRobots = "seo_update_robots"
|
|
)
|
|
|
|
// Issues builds the CMS-data issue report (D-13, D-14): missing effective
|
|
// titles and descriptions per record and locale, effective titles and
|
|
// descriptions shared by more than one address (records and URL rules),
|
|
// URL rules overriding a record, redirect chains with their final
|
|
// targets, canonicals pointing to a noindex or redirected address, and
|
|
// noindex addresses robots.txt keeps crawlers from. Records are read
|
|
// through their SEO entries (the generation job creates one per
|
|
// published record and locale).
|
|
func (s *Service) Issues(ctx context.Context) ([]Issue, error) {
|
|
db, err := s.DB(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var data issueData
|
|
if err := db.Where("url <> ''").Order("url ASC, locale ASC").Find(&data.entries).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
if err := db.Order("url ASC").Find(&data.rules).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
data.locale = s.DefaultLocale(ctx, db)
|
|
if data.redirects, err = LoadRedirects(db); err != nil {
|
|
return nil, err
|
|
}
|
|
if data.robots, err = LoadRobots(db); err != nil {
|
|
return nil, err
|
|
}
|
|
if data.noindex, err = noindexAddresses(db); err != nil {
|
|
return nil, err
|
|
}
|
|
listed, _, err := s.SitemapListed(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for address := range listed {
|
|
data.listed = append(data.listed, address)
|
|
}
|
|
sort.Strings(data.listed)
|
|
out, err := s.issuesOf(ctx, data, s.canonicalProblem)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
content, err := s.ContentIssues(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return append(out, content...), nil
|
|
}
|
|
|
|
// contentIssuesPage is the page size of the content walk.
|
|
const contentIssuesPage = 200
|
|
|
|
// ContentIssues are the CMS-data checks on the content descriptors of
|
|
// every published record (D-85, D-50): image_alt for images with an empty
|
|
// alt that are not decorative, and h1_in_content for an H1 in the body of
|
|
// a record whose descriptor sets TopHeading 2 (off when the type sets no
|
|
// rule, as Journal does with its H1 setting off). Each names the record
|
|
// type's fitting tool (ContentDescriptor.Tools: image_alt or headings,
|
|
// else edit); "" when the type names none.
|
|
func (s *Service) ContentIssues(ctx context.Context) ([]Issue, error) {
|
|
reg := s.Registry()
|
|
if reg == nil {
|
|
return nil, nil
|
|
}
|
|
var out []Issue
|
|
for _, typ := range reg.ContentTypes() {
|
|
if typ.Content == nil || typ.Content.Records == nil {
|
|
continue
|
|
}
|
|
tool := func(ops ...string) string {
|
|
for _, op := range ops {
|
|
if t := strings.TrimSpace(typ.Content.Tools[op]); t != "" {
|
|
return t
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
after := ""
|
|
for {
|
|
page, err := reg.Records(ctx, typ.Key, after, contentIssuesPage, s.UnprefixedLocale())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, rec := range page.Records {
|
|
if !rec.Published {
|
|
continue
|
|
}
|
|
for _, l := range rec.Locales {
|
|
if strings.TrimSpace(l.URL) == "" {
|
|
continue
|
|
}
|
|
var missing []string
|
|
for _, img := range l.Images {
|
|
if !img.Decorative && strings.TrimSpace(img.Alt) == "" && img.URL != "" {
|
|
missing = append(missing, img.URL)
|
|
}
|
|
}
|
|
if len(missing) > 0 {
|
|
out = append(out, Issue{Kind: IssueImageAlt, URL: l.URL, Locale: l.Locale, Tool: tool(pact.ContentToolImageAlt, pact.ContentToolEdit), Details: map[string]any{
|
|
"count": len(missing), "images": missing, "type": typ.Key, "id": rec.ID,
|
|
}})
|
|
}
|
|
if rec.TopHeading == 2 {
|
|
var h1 []string
|
|
for _, h := range l.Headings {
|
|
if h.Level == 1 {
|
|
h1 = append(h1, h.Text)
|
|
}
|
|
}
|
|
if len(h1) > 0 {
|
|
out = append(out, Issue{Kind: IssueH1InContent, URL: l.URL, Locale: l.Locale, Tool: tool(pact.ContentToolHeadings, pact.ContentToolEdit), Details: map[string]any{
|
|
"count": len(h1), "h1": h1, "type": typ.Key, "id": rec.ID,
|
|
}})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if page.Next == "" || page.Next == after || len(page.Records) == 0 {
|
|
break
|
|
}
|
|
after = page.Next
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// issueData is what the issue report reads from the database: the SEO
|
|
// entries with an address (by url and locale), the URL rules (by url), the
|
|
// default content locale, the redirects, the robots row and the noindex
|
|
// addresses.
|
|
type issueData struct {
|
|
entries []models.Entry
|
|
rules []models.Rule
|
|
locale string
|
|
redirects []models.Redirect
|
|
robots models.Robots
|
|
noindex []string
|
|
// listed are the addresses the sitemaps list (path with query).
|
|
listed []string
|
|
}
|
|
|
|
// issuesOf builds the report from data; canonical classifies a canonical
|
|
// value (canonicalProblem).
|
|
func (s *Service) issuesOf(ctx context.Context, data issueData, canonical func(context.Context, string) (string, string, error)) ([]Issue, error) {
|
|
var out []Issue
|
|
entries, rules := data.entries, data.rules
|
|
|
|
// Missing and duplicate effective values.
|
|
type place struct{ url, locale string }
|
|
titles := map[string][]place{}
|
|
descs := map[string][]place{}
|
|
for _, e := range entries {
|
|
if strings.TrimSpace(e.EffectiveTitle) == "" {
|
|
out = append(out, Issue{Kind: IssueMissingTitle, URL: e.URL, Locale: e.Locale, Tool: toolSetURLSEO})
|
|
} else {
|
|
k := e.Locale + "|" + strings.ToLower(strings.TrimSpace(e.EffectiveTitle))
|
|
titles[k] = append(titles[k], place{e.URL, e.Locale})
|
|
}
|
|
if strings.TrimSpace(e.EffectiveDescription) == "" {
|
|
out = append(out, Issue{Kind: IssueMissingDescription, URL: e.URL, Locale: e.Locale, Tool: toolSetURLSEO})
|
|
} else {
|
|
k := e.Locale + "|" + strings.ToLower(strings.TrimSpace(e.EffectiveDescription))
|
|
descs[k] = append(descs[k], place{e.URL, e.Locale})
|
|
}
|
|
}
|
|
locale := data.locale
|
|
for _, r := range rules {
|
|
if r.Match != models.MatchExact {
|
|
continue
|
|
}
|
|
if t := strings.TrimSpace(r.Title); t != "" {
|
|
k := locale + "|" + strings.ToLower(t)
|
|
titles[k] = append(titles[k], place{r.URL, ""})
|
|
}
|
|
if d := strings.TrimSpace(r.Description); d != "" {
|
|
k := locale + "|" + strings.ToLower(d)
|
|
descs[k] = append(descs[k], place{r.URL, ""})
|
|
}
|
|
}
|
|
duplicates := func(groups map[string][]place, kind string) {
|
|
for key, list := range groups {
|
|
if len(list) < 2 {
|
|
continue
|
|
}
|
|
_, text, _ := strings.Cut(key, "|")
|
|
for i, p := range list {
|
|
others := make([]string, 0, len(list)-1)
|
|
for j, o := range list {
|
|
if j != i && o.url != p.url {
|
|
others = append(others, o.url)
|
|
}
|
|
}
|
|
if len(others) == 0 {
|
|
continue
|
|
}
|
|
out = append(out, Issue{Kind: kind, URL: p.url, Locale: p.locale, Tool: toolSetURLSEO, Details: map[string]any{"value": text, "also": others}})
|
|
}
|
|
}
|
|
}
|
|
duplicates(titles, IssueDuplicateTitle)
|
|
duplicates(descs, IssueDuplicateDesc)
|
|
|
|
// URL rules overriding a record.
|
|
if reg := s.Registry(); reg != nil {
|
|
for _, r := range rules {
|
|
if r.Match != models.MatchExact || strings.Contains(r.URL, "?") {
|
|
continue
|
|
}
|
|
var fields []string
|
|
for _, p := range models.RuleProperties {
|
|
if strings.TrimSpace(r.Value(p.Column)) != "" {
|
|
fields = append(fields, p.Column)
|
|
}
|
|
}
|
|
if len(fields) == 0 {
|
|
continue
|
|
}
|
|
key, match, err := reg.MatchPath(ctx, trimSlash(r.URL), s.UnprefixedLocale())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if match == nil || !match.Exact {
|
|
continue
|
|
}
|
|
out = append(out, Issue{Kind: IssueRuleOverridesRecord, URL: r.URL, Locale: match.Locale, Tool: toolUpdateRule, Details: map[string]any{
|
|
"rule_id": r.ID, "type": key, "id": match.ID, "fields": fields,
|
|
}})
|
|
}
|
|
}
|
|
|
|
// Redirect chains.
|
|
redirects := data.redirects
|
|
set := NewRedirectSet(redirects)
|
|
site := s.SiteURL()
|
|
for _, r := range redirects {
|
|
if r.Status == models.Status410 {
|
|
continue
|
|
}
|
|
path, query, ok := redirectProbe(r)
|
|
if !ok {
|
|
continue
|
|
}
|
|
ch, err := set.Follow(site, path, query, MaxRedirectHops)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if ch.Loop || len(ch.Hops) < 2 || ch.Hops[0].Rule.ID != r.ID {
|
|
continue
|
|
}
|
|
out = append(out, Issue{Kind: IssueRedirectChain, URL: r.Source, Tool: toolUpdateRedirect, Details: map[string]any{
|
|
"redirect_id": r.ID, "hops": len(ch.Hops), "final": displayURL(site, ch.Final), "path": displayPath(site, r, ch),
|
|
}})
|
|
}
|
|
|
|
// Canonicals pointing to a noindex or redirected address.
|
|
addCanonical := func(u, loc, value, tool string, details map[string]any) error {
|
|
kind, target, err := canonical(ctx, value)
|
|
if err != nil || kind == "" {
|
|
return err
|
|
}
|
|
if details == nil {
|
|
details = map[string]any{}
|
|
}
|
|
details["canonical"] = value
|
|
if target != "" {
|
|
details["target"] = target
|
|
}
|
|
out = append(out, Issue{Kind: kind, URL: u, Locale: loc, Tool: tool, Details: details})
|
|
return nil
|
|
}
|
|
for _, e := range entries {
|
|
if err := addCanonical(e.URL, e.Locale, e.Canonical, toolSetURLSEO, nil); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
for _, r := range rules {
|
|
if err := addCanonical(r.URL, "", r.Canonical, toolUpdateRule, map[string]any{"rule_id": r.ID}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// Noindex addresses robots.txt blocks.
|
|
file := ComposeRobots(data.robots.Content, data.robots.Bots, s.SitemapLines(ctx))
|
|
for _, u := range s.blockedNoindex(data.noindex, data.robots.Content, data.robots.Bots) {
|
|
out = append(out, Issue{Kind: IssueNoindexBlocked, URL: u, Tool: toolUpdateRobots})
|
|
}
|
|
|
|
// Sitemap addresses the production robots.txt blocks for Googlebot.
|
|
for _, u := range data.listed {
|
|
test := s.testRobotsFile(file, data.robots.Content, "Googlebot", u)
|
|
if test.Allowed {
|
|
continue
|
|
}
|
|
out = append(out, Issue{Kind: IssueSitemapBlocked, URL: u, Tool: toolUpdateRobots, Severity: SeverityError, Details: map[string]any{
|
|
"line": test.Line, "rule": test.Rule,
|
|
}})
|
|
}
|
|
|
|
sort.SliceStable(out, func(i, j int) bool {
|
|
if out[i].Kind != out[j].Kind {
|
|
return out[i].Kind < out[j].Kind
|
|
}
|
|
if out[i].URL != out[j].URL {
|
|
return out[i].URL < out[j].URL
|
|
}
|
|
return out[i].Locale < out[j].Locale
|
|
})
|
|
return out, nil
|
|
}
|
|
|
|
// canonicalProblem checks a canonical on this site (D-06): a redirect
|
|
// answers canonical_redirected with the final address, a page served
|
|
// noindex (in production) canonical_noindex. Other sites and empty values
|
|
// answer no kind.
|
|
func (s *Service) canonicalProblem(ctx context.Context, canonical string) (kind, target string, err error) {
|
|
canonical = strings.TrimSpace(canonical)
|
|
if canonical == "" {
|
|
return "", "", nil
|
|
}
|
|
ref := canonical
|
|
if u, err := url.Parse(canonical); err == nil && u.IsAbs() {
|
|
if !sameSite(s.SiteURL(), u.Host) {
|
|
return "", "", nil
|
|
}
|
|
ref = u.String()
|
|
}
|
|
res, err := s.Resolver().Resolve(ctx, ref)
|
|
if err != nil {
|
|
if err == ErrBadURL {
|
|
return "", "", nil
|
|
}
|
|
return "", "", err
|
|
}
|
|
if res.Action == ActionRedirect {
|
|
return IssueCanonicalRedirected, res.Location, nil
|
|
}
|
|
if s.Staging() || res.SEO == nil || !strings.Contains(res.SEO.Robots, "noindex") {
|
|
return "", "", nil
|
|
}
|
|
return IssueCanonicalNoindex, "", nil
|
|
}
|
|
|
|
// Actor names who runs an audit operation (a crawl start, a cancel, an
|
|
// ignore): the backend administrator's id and name, and whether the call
|
|
// came through MCP.
|
|
func (s *Service) Actor(ctx context.Context, db *gorm.DB) (id *uint, name string, mcp bool) {
|
|
id, name = s.author(ctx, db, models.ViaAdmin)
|
|
return id, name, ViaOf(ctx) == models.ViaMCP
|
|
}
|