feat: sitemap-aware batch conflict and the sitemap_blocked issue

- noindex_published fires only when a sitemap file lists the address now
  (read once per batch through contract.Listing)
- CMS issue sitemap_blocked (severity error, tool seo_update_robots,
  details line and rule): a listed address the production robots.txt
  keeps Googlebot from
- the SEO tab's P1 shows status.excluded_title with the reason (noindex,
  canonical elsewhere, redirect, or the type switched off in the sitemap
  settings via ListingSettings.TypesOff) as a plain callout
- robots.txt is composed once per check of many addresses
This commit is contained in:
Jakub Zych
2026-11-04 13:39:31 +01:00
parent 742a5b930b
commit 59f0457c69
11 changed files with 370 additions and 22 deletions

View File

@@ -205,7 +205,12 @@ type RobotsTest struct {
// grobotstxt, Google's matcher (D-22, D-99): longest match, Allow wins a
// tie, * and $.
func (s *Service) TestRobots(content string, bots models.BotSwitches, agent, path string) RobotsTest {
file := ComposeRobots(content, bots, s.SitemapLines(context.Background()))
return s.testRobotsFile(ComposeRobots(content, bots, s.SitemapLines(context.Background())), content, agent, path)
}
// testRobotsFile is TestRobots on a composed file; content is the editor's
// part of it (the generated bot rules follow it).
func (s *Service) testRobotsFile(file, content, agent, path string) RobotsTest {
uri := AbsoluteURL(s.SiteURL(), path)
if s.SiteURL() == "" {
uri = "http://localhost" + path
@@ -269,8 +274,12 @@ func (s *Service) BlockedNoindex(db *gorm.DB, content string, bots models.BotSwi
// the production file composed from content and bots.
func (s *Service) blockedNoindex(urls []string, content string, bots models.BotSwitches) []string {
var out []string
if len(urls) == 0 {
return out
}
file := ComposeRobots(content, bots, s.SitemapLines(context.Background()))
for _, u := range urls {
if !s.TestRobots(content, bots, "Googlebot", u).Allowed {
if !s.testRobotsFile(file, content, "Googlebot", u).Allowed {
out = append(out, u)
}
}