package classes import ( "context" "net/url" "sort" "strings" "git.golem15.com/golem15/sm-seo-plugin/models" "gorm.io/gorm" ) // The kinds of the CMS-data issue report (D-13, D-14 CMS part). The // crawler's checks of the rendered pages are Phase 33. const ( IssueMissingTitle = "missing_title" IssueMissingDescription = "missing_description" IssueDuplicateTitle = "duplicate_title" IssueDuplicateDesc = "duplicate_description" IssueRuleOverridesRecord = "rule_overrides_record" IssueRedirectChain = "redirect_chain" IssueCanonicalNoindex = "canonical_noindex" IssueCanonicalRedirected = "canonical_redirected" IssueNoindexBlocked = "noindex_blocked" // IssueSitemapBlocked is a sitemap address the production robots.txt // keeps Googlebot from (D-24): an error, since the sitemap asks for a // page the crawler may not fetch. IssueSitemapBlocked = "sitemap_blocked" ) // The severities an issue may name. const ( SeverityError = "error" ) // Issue is one finding of the issue report: the address, the kind, the // details of the finding and the MCP tool that fixes it. type Issue struct { Kind string `json:"kind"` URL string `json:"url"` Locale string `json:"locale,omitempty"` Details map[string]any `json:"details,omitempty"` Tool string `json:"tool"` // Severity is set for kinds that are errors (sitemap_blocked); "" for // the others. Severity string `json:"severity,omitempty"` } // The tools the issues point to (pier prefixes them with seo_). const ( toolSetURLSEO = "seo_set_url_seo" toolUpdateRule = "seo_update_rule" toolUpdateRedirect = "seo_update_redirect" toolUpdateRobots = "seo_update_robots" ) // Issues builds the CMS-data issue report (D-13, D-14): missing effective // titles and descriptions per record and locale, effective titles and // descriptions shared by more than one address (records and URL rules), // URL rules overriding a record, redirect chains with their final // targets, canonicals pointing to a noindex or redirected address, and // noindex addresses robots.txt keeps crawlers from. Records are read // through their SEO entries (the generation job creates one per // published record and locale). func (s *Service) Issues(ctx context.Context) ([]Issue, error) { db, err := s.DB(ctx) if err != nil { return nil, err } var data issueData if err := db.Where("url <> ''").Order("url ASC, locale ASC").Find(&data.entries).Error; err != nil { return nil, err } if err := db.Order("url ASC").Find(&data.rules).Error; err != nil { return nil, err } data.locale = s.DefaultLocale(ctx, db) if data.redirects, err = LoadRedirects(db); err != nil { return nil, err } if data.robots, err = LoadRobots(db); err != nil { return nil, err } if data.noindex, err = noindexAddresses(db); err != nil { return nil, err } listed, _, err := s.SitemapListed(ctx) if err != nil { return nil, err } for address := range listed { data.listed = append(data.listed, address) } sort.Strings(data.listed) return s.issuesOf(ctx, data, s.canonicalProblem) } // issueData is what the issue report reads from the database: the SEO // entries with an address (by url and locale), the URL rules (by url), the // default content locale, the redirects, the robots row and the noindex // addresses. type issueData struct { entries []models.Entry rules []models.Rule locale string redirects []models.Redirect robots models.Robots noindex []string // listed are the addresses the sitemaps list (path with query). listed []string } // issuesOf builds the report from data; canonical classifies a canonical // value (canonicalProblem). func (s *Service) issuesOf(ctx context.Context, data issueData, canonical func(context.Context, string) (string, string, error)) ([]Issue, error) { var out []Issue entries, rules := data.entries, data.rules // Missing and duplicate effective values. type place struct{ url, locale string } titles := map[string][]place{} descs := map[string][]place{} for _, e := range entries { if strings.TrimSpace(e.EffectiveTitle) == "" { out = append(out, Issue{Kind: IssueMissingTitle, URL: e.URL, Locale: e.Locale, Tool: toolSetURLSEO}) } else { k := e.Locale + "|" + strings.ToLower(strings.TrimSpace(e.EffectiveTitle)) titles[k] = append(titles[k], place{e.URL, e.Locale}) } if strings.TrimSpace(e.EffectiveDescription) == "" { out = append(out, Issue{Kind: IssueMissingDescription, URL: e.URL, Locale: e.Locale, Tool: toolSetURLSEO}) } else { k := e.Locale + "|" + strings.ToLower(strings.TrimSpace(e.EffectiveDescription)) descs[k] = append(descs[k], place{e.URL, e.Locale}) } } locale := data.locale for _, r := range rules { if r.Match != models.MatchExact { continue } if t := strings.TrimSpace(r.Title); t != "" { k := locale + "|" + strings.ToLower(t) titles[k] = append(titles[k], place{r.URL, ""}) } if d := strings.TrimSpace(r.Description); d != "" { k := locale + "|" + strings.ToLower(d) descs[k] = append(descs[k], place{r.URL, ""}) } } duplicates := func(groups map[string][]place, kind string) { for key, list := range groups { if len(list) < 2 { continue } _, text, _ := strings.Cut(key, "|") for i, p := range list { others := make([]string, 0, len(list)-1) for j, o := range list { if j != i && o.url != p.url { others = append(others, o.url) } } if len(others) == 0 { continue } out = append(out, Issue{Kind: kind, URL: p.url, Locale: p.locale, Tool: toolSetURLSEO, Details: map[string]any{"value": text, "also": others}}) } } } duplicates(titles, IssueDuplicateTitle) duplicates(descs, IssueDuplicateDesc) // URL rules overriding a record. if reg := s.Registry(); reg != nil { for _, r := range rules { if r.Match != models.MatchExact || strings.Contains(r.URL, "?") { continue } var fields []string for _, p := range models.RuleProperties { if strings.TrimSpace(r.Value(p.Column)) != "" { fields = append(fields, p.Column) } } if len(fields) == 0 { continue } key, match, err := reg.MatchPath(ctx, trimSlash(r.URL), s.UnprefixedLocale()) if err != nil { return nil, err } if match == nil || !match.Exact { continue } out = append(out, Issue{Kind: IssueRuleOverridesRecord, URL: r.URL, Locale: match.Locale, Tool: toolUpdateRule, Details: map[string]any{ "rule_id": r.ID, "type": key, "id": match.ID, "fields": fields, }}) } } // Redirect chains. redirects := data.redirects set := NewRedirectSet(redirects) site := s.SiteURL() for _, r := range redirects { if r.Status == models.Status410 { continue } path, query, ok := redirectProbe(r) if !ok { continue } ch, err := set.Follow(site, path, query, MaxRedirectHops) if err != nil { return nil, err } if ch.Loop || len(ch.Hops) < 2 || ch.Hops[0].Rule.ID != r.ID { continue } out = append(out, Issue{Kind: IssueRedirectChain, URL: r.Source, Tool: toolUpdateRedirect, Details: map[string]any{ "redirect_id": r.ID, "hops": len(ch.Hops), "final": displayURL(site, ch.Final), "path": displayPath(site, r, ch), }}) } // Canonicals pointing to a noindex or redirected address. addCanonical := func(u, loc, value, tool string, details map[string]any) error { kind, target, err := canonical(ctx, value) if err != nil || kind == "" { return err } if details == nil { details = map[string]any{} } details["canonical"] = value if target != "" { details["target"] = target } out = append(out, Issue{Kind: kind, URL: u, Locale: loc, Tool: tool, Details: details}) return nil } for _, e := range entries { if err := addCanonical(e.URL, e.Locale, e.Canonical, toolSetURLSEO, nil); err != nil { return nil, err } } for _, r := range rules { if err := addCanonical(r.URL, "", r.Canonical, toolUpdateRule, map[string]any{"rule_id": r.ID}); err != nil { return nil, err } } // Noindex addresses robots.txt blocks. file := ComposeRobots(data.robots.Content, data.robots.Bots, s.SitemapLines(ctx)) for _, u := range s.blockedNoindex(data.noindex, data.robots.Content, data.robots.Bots) { out = append(out, Issue{Kind: IssueNoindexBlocked, URL: u, Tool: toolUpdateRobots}) } // Sitemap addresses the production robots.txt blocks for Googlebot. for _, u := range data.listed { test := s.testRobotsFile(file, data.robots.Content, "Googlebot", u) if test.Allowed { continue } out = append(out, Issue{Kind: IssueSitemapBlocked, URL: u, Tool: toolUpdateRobots, Severity: SeverityError, Details: map[string]any{ "line": test.Line, "rule": test.Rule, }}) } sort.SliceStable(out, func(i, j int) bool { if out[i].Kind != out[j].Kind { return out[i].Kind < out[j].Kind } if out[i].URL != out[j].URL { return out[i].URL < out[j].URL } return out[i].Locale < out[j].Locale }) return out, nil } // canonicalProblem checks a canonical on this site (D-06): a redirect // answers canonical_redirected with the final address, a page served // noindex (in production) canonical_noindex. Other sites and empty values // answer no kind. func (s *Service) canonicalProblem(ctx context.Context, canonical string) (kind, target string, err error) { canonical = strings.TrimSpace(canonical) if canonical == "" { return "", "", nil } ref := canonical if u, err := url.Parse(canonical); err == nil && u.IsAbs() { if !sameSite(s.SiteURL(), u.Host) { return "", "", nil } ref = u.String() } res, err := s.Resolver().Resolve(ctx, ref) if err != nil { if err == ErrBadURL { return "", "", nil } return "", "", err } if res.Action == ActionRedirect { return IssueCanonicalRedirected, res.Location, nil } if s.Staging() || res.SEO == nil || !strings.Contains(res.SEO.Robots, "noindex") { return "", "", nil } return IssueCanonicalNoindex, "", nil } // Actor names who runs an audit operation (a crawl start, a cancel, an // ignore): the backend administrator's id and name, and whether the call // came through MCP. func (s *Service) Actor(ctx context.Context, db *gorm.DB) (id *uint, name string, mcp bool) { id, name = s.author(ctx, db, models.ViaAdmin) return id, name, ViaOf(ctx) == models.ViaMCP }