feat: sitemap-aware batch conflict and the sitemap_blocked issue

- noindex_published fires only when a sitemap file lists the address now
  (read once per batch through contract.Listing)
- CMS issue sitemap_blocked (severity error, tool seo_update_robots,
  details line and rule): a listed address the production robots.txt
  keeps Googlebot from
- the SEO tab's P1 shows status.excluded_title with the reason (noindex,
  canonical elsewhere, redirect, or the type switched off in the sitemap
  settings via ListingSettings.TypesOff) as a plain callout
- robots.txt is composed once per check of many addresses

classes/issues.go

@@ -21,6 +21,15 @@ const (
 	IssueCanonicalNoindex    = "canonical_noindex"
 	IssueCanonicalRedirected = "canonical_redirected"
 	IssueNoindexBlocked      = "noindex_blocked"
+	// IssueSitemapBlocked is a sitemap address the production robots.txt
+	// keeps Googlebot from (D-24): an error, since the sitemap asks for a
+	// page the crawler may not fetch.
+	IssueSitemapBlocked = "sitemap_blocked"
+)
+
+// The severities an issue may name.
+const (
+	SeverityError = "error"
 )
 
 // Issue is one finding of the issue report: the address, the kind, the
@@ -31,6 +40,9 @@ type Issue struct {
 	Locale  string         `json:"locale,omitempty"`
 	Details map[string]any `json:"details,omitempty"`
 	Tool    string         `json:"tool"`
+	// Severity is set for kinds that are errors (sitemap_blocked); "" for
+	// the others.
+	Severity string `json:"severity,omitempty"`
 }
 
 // The tools the issues point to (pier prefixes them with seo_).
@@ -71,6 +83,14 @@ func (s *Service) Issues(ctx context.Context) ([]Issue, error) {
 	if data.noindex, err = noindexAddresses(db); err != nil {
 		return nil, err
 	}
+	listed, _, err := s.SitemapListed(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for address := range listed {
+		data.listed = append(data.listed, address)
+	}
+	sort.Strings(data.listed)
 	return s.issuesOf(ctx, data, s.canonicalProblem)
 }
 
@@ -85,6 +105,8 @@ type issueData struct {
 	redirects []models.Redirect
 	robots    models.Robots
 	noindex   []string
+	// listed are the addresses the sitemaps list (path with query).
+	listed []string
 }
 
 // issuesOf builds the report from data; canonical classifies a canonical
@@ -228,10 +250,22 @@ func (s *Service) issuesOf(ctx context.Context, data issueData, canonical func(c
 	}
 
 	// Noindex addresses robots.txt blocks.
+	file := ComposeRobots(data.robots.Content, data.robots.Bots, s.SitemapLines(ctx))
 	for _, u := range s.blockedNoindex(data.noindex, data.robots.Content, data.robots.Bots) {
 		out = append(out, Issue{Kind: IssueNoindexBlocked, URL: u, Tool: toolUpdateRobots})
 	}
 
+	// Sitemap addresses the production robots.txt blocks for Googlebot.
+	for _, u := range data.listed {
+		test := s.testRobotsFile(file, data.robots.Content, "Googlebot", u)
+		if test.Allowed {
+			continue
+		}
+		out = append(out, Issue{Kind: IssueSitemapBlocked, URL: u, Tool: toolUpdateRobots, Severity: SeverityError, Details: map[string]any{
+			"line": test.Line, "rule": test.Rule,
+		}})
+	}
+
 	sort.SliceStable(out, func(i, j int) bool {
 		if out[i].Kind != out[j].Kind {
 			return out[i].Kind < out[j].Kind