package classes import ( "context" "sort" "strings" "git.golem15.com/golem15/sm-seo-plugin/models" "github.com/jimsmart/grobotstxt" "gorm.io/gorm" ) // RobotsSitemapsConfigKey lists the sitemap addresses robots.txt names, // one Sitemap line each, absolute on the site URL (P-23). Unset, robots.txt // names the sitemap plugin's index (D-22, SitemapLines). const RobotsSitemapsConfigKey = "golem15.seo.robots_sitemaps" // MaxRobotsBytes is the largest robots.txt the editor stores: Google // ignores everything after 500 KiB. const MaxRobotsBytes = 500 << 10 // The two fixed robots.txt bodies. const ( // NonProductionRobots is served outside production whatever is stored // (D-22). NonProductionRobots = "User-agent: *\nDisallow: /\n" // FallbackRobots is served when the stored file cannot be read and no // earlier body is cached: allow everything, never a 5xx. FallbackRobots = "User-agent: *\nDisallow:\n" ) // SitemapLines are the addresses of robots.txt's Sitemap lines (D-22): // golem15.seo.robots_sitemaps when it is set, else the sitemap index of // the sitemap plugin's contract.Listing (its IndexURL), else /sitemap.xml // on the site URL. The per-type files are never listed: the index names // them. func (s *Service) SitemapLines(ctx context.Context) []string { if s.robotsSitemapsSet() { return s.RobotsSitemaps() } if l := s.listing(); l != nil { if st, err := l.Settings(ctx); err == nil && strings.TrimSpace(st.IndexURL) != "" { return []string{strings.TrimSpace(st.IndexURL)} } else if err != nil { log.Warn("golem15.seo: sitemap index address", "error", err) } } return s.RobotsSitemaps() } // robotsSitemapsSet reports whether golem15.seo.robots_sitemaps is set to // a non-empty list. func (s *Service) robotsSitemapsSet() bool { cfg := s.Config() if cfg == nil { return false } raw, ok := cfg.Lookup(RobotsSitemapsConfigKey) if !ok || raw == nil { return false } switch v := raw.(type) { case []any: return len(v) > 0 case []string: return len(v) > 0 case string: return strings.TrimSpace(v) != "" } return false } // RobotsSitemaps returns the configured sitemap addresses, absolute on // the site URL; /sitemap.xml when none is configured. func (s *Service) RobotsSitemaps() []string { var list []string if cfg := s.Config(); cfg != nil { if raw, ok := cfg.Lookup(RobotsSitemapsConfigKey); ok { switch v := raw.(type) { case []any: for _, item := range v { if str, ok := item.(string); ok { list = append(list, str) } } case []string: list = append(list, v...) case string: list = strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' }) } } } if list == nil { list = []string{"/sitemap.xml"} } out := make([]string, 0, len(list)) for _, item := range list { if item = strings.TrimSpace(item); item != "" { out = append(out, AbsoluteURL(s.SiteURL(), item)) } } return out } // LoadRobots reads robots row 1; without one the defaults apply (no // rules, every bot allowed). func LoadRobots(db *gorm.DB) (models.Robots, error) { var rows []models.Robots if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil { return models.Robots{}, err } if len(rows) == 0 { return models.Robots{ID: 1, Bots: models.BotSwitches{}}, nil } return rows[0], nil } // ComposeRobots builds the production robots.txt (D-22, D-25, P-23): the // stored rules, one "User-agent: " / "Disallow: /" group per // switched-off bot, then one Sitemap line per sitemap address. func ComposeRobots(content string, bots models.BotSwitches, sitemaps []string) string { var b strings.Builder content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t") if content != "" { b.WriteString(content) b.WriteString("\n") } for _, bot := range Bots { if bots.Allowed(bot.Token) { continue } if b.Len() > 0 { b.WriteString("\n") } b.WriteString("User-agent: " + bot.Token + "\nDisallow: /\n") } if len(sitemaps) > 0 && b.Len() > 0 { b.WriteString("\n") } for _, sm := range sitemaps { b.WriteString("Sitemap: " + sm + "\n") } return b.String() } // generatedStart is the 1-based line where the generated part of a // composed file starts: the lines after the stored rules. func generatedStart(content string) int { content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t") if content == "" { return 1 } return strings.Count(content, "\n") + 2 } // ServedFile is the robots.txt crawlers get (D-22): outside production // always NonProductionRobots, else the composed file from the stored rules // and switches. It caches the last body it served, which LastServedRobots // answers when the store fails. func (s *Service) ServedFile(ctx context.Context, production bool) (string, error) { if !production { return NonProductionRobots, nil } db, err := s.DB(ctx) if err != nil { return "", err } row, err := LoadRobots(db) if err != nil { return "", err } body := ComposeRobots(row.Content, row.Bots, s.SitemapLines(ctx)) if s != nil { s.robots.Store(&body) } return body, nil } // LastServedRobots returns the last production body ServedFile answered, // "" when there is none. func (s *Service) LastServedRobots() string { if s == nil { return "" } if p := s.robots.Load(); p != nil { return *p } return "" } // RobotsTest is the tester's answer (UI-SPEC E5): whether agent may fetch // the address, the deciding line (0 and "" when no rule matched) and // whether a bot switch decided it. type RobotsTest struct { Allowed bool `json:"allowed"` Line int `json:"line"` Rule string `json:"rule"` Bot bool `json:"bot"` Agent string `json:"agent"` Staging bool `json:"staging"` } // TestRobots answers whether agent may fetch path under the production // file composed from content and bots (unsaved values included), through // grobotstxt, Google's matcher (D-22, D-99): longest match, Allow wins a // tie, * and $. func (s *Service) TestRobots(content string, bots models.BotSwitches, agent, path string) RobotsTest { return s.testRobotsFile(ComposeRobots(content, bots, s.SitemapLines(context.Background())), content, agent, path) } // testRobotsFile is TestRobots on a composed file; content is the editor's // part of it (the generated bot rules follow it). func (s *Service) testRobotsFile(file, content, agent, path string) RobotsTest { uri := AbsoluteURL(s.SiteURL(), path) if s.SiteURL() == "" { uri = "http://localhost" + path } m := grobotstxt.NewRobotsMatcher() out := RobotsTest{Agent: agent, Staging: s.Staging()} out.Allowed = m.AgentAllowed(file, agent, uri) if line := m.MatchingLine(); line > 0 { out.Line = line lines := strings.Split(file, "\n") if line <= len(lines) { out.Rule = strings.TrimSpace(lines[line-1]) } out.Bot = !out.Allowed && line >= generatedStart(content) } return out } // noindexAddresses lists the addresses the site serves noindex that a // robots.txt could block (D-24): exact URL rules and records whose own // robots value says noindex. func noindexAddresses(db *gorm.DB) ([]string, error) { seen := map[string]bool{} var out []string var rules []models.Rule if err := db.Where("match = ? AND robots LIKE ?", models.MatchExact, "%noindex%").Find(&rules).Error; err != nil { return nil, err } for _, r := range rules { if !seen[r.URL] { seen[r.URL] = true out = append(out, r.URL) } } var entries []models.Entry if err := db.Where("robots LIKE ? AND url <> ''", "%noindex%").Find(&entries).Error; err != nil { return nil, err } for _, e := range entries { if !seen[e.URL] { seen[e.URL] = true out = append(out, e.URL) } } sort.Strings(out) return out, nil } // BlockedNoindex lists the noindex addresses Googlebot may not fetch under // the production file composed from content and bots (D-24): a crawler // that cannot fetch a page never sees its noindex. func (s *Service) BlockedNoindex(db *gorm.DB, content string, bots models.BotSwitches) ([]string, error) { urls, err := noindexAddresses(db) if err != nil { return nil, err } return s.blockedNoindex(urls, content, bots), nil } // blockedNoindex lists the addresses of urls Googlebot may not fetch under // the production file composed from content and bots. func (s *Service) blockedNoindex(urls []string, content string, bots models.BotSwitches) []string { var out []string if len(urls) == 0 { return out } file := ComposeRobots(content, bots, s.SitemapLines(context.Background())) for _, u := range urls { if !s.testRobotsFile(file, content, "Googlebot", u).Allowed { out = append(out, u) } } return out }