feat: sitemap-aware batch conflict and the sitemap_blocked issue
- noindex_published fires only when a sitemap file lists the address now (read once per batch through contract.Listing) - CMS issue sitemap_blocked (severity error, tool seo_update_robots, details line and rule): a listed address the production robots.txt keeps Googlebot from - the SEO tab's P1 shows status.excluded_title with the reason (noindex, canonical elsewhere, redirect, or the type switched off in the sitemap settings via ListingSettings.TypesOff) as a plain callout - robots.txt is composed once per check of many addresses
This commit is contained in:
@@ -205,7 +205,12 @@ type RobotsTest struct {
|
||||
// grobotstxt, Google's matcher (D-22, D-99): longest match, Allow wins a
|
||||
// tie, * and $.
|
||||
func (s *Service) TestRobots(content string, bots models.BotSwitches, agent, path string) RobotsTest {
|
||||
file := ComposeRobots(content, bots, s.SitemapLines(context.Background()))
|
||||
return s.testRobotsFile(ComposeRobots(content, bots, s.SitemapLines(context.Background())), content, agent, path)
|
||||
}
|
||||
|
||||
// testRobotsFile is TestRobots on a composed file; content is the editor's
|
||||
// part of it (the generated bot rules follow it).
|
||||
func (s *Service) testRobotsFile(file, content, agent, path string) RobotsTest {
|
||||
uri := AbsoluteURL(s.SiteURL(), path)
|
||||
if s.SiteURL() == "" {
|
||||
uri = "http://localhost" + path
|
||||
@@ -269,8 +274,12 @@ func (s *Service) BlockedNoindex(db *gorm.DB, content string, bots models.BotSwi
|
||||
// the production file composed from content and bots.
|
||||
func (s *Service) blockedNoindex(urls []string, content string, bots models.BotSwitches) []string {
|
||||
var out []string
|
||||
if len(urls) == 0 {
|
||||
return out
|
||||
}
|
||||
file := ComposeRobots(content, bots, s.SitemapLines(context.Background()))
|
||||
for _, u := range urls {
|
||||
if !s.TestRobots(content, bots, "Googlebot", u).Allowed {
|
||||
if !s.testRobotsFile(file, content, "Googlebot", u).Allowed {
|
||||
out = append(out, u)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user