sm-seo-plugin/classes/robots.go

package classes

import (
	"context"
	"sort"
	"strings"

	"git.golem15.com/golem15/sm-seo-plugin/models"
	"github.com/jimsmart/grobotstxt"
	"gorm.io/gorm"
)

// RobotsSitemapsConfigKey lists the sitemap addresses robots.txt names,
// one Sitemap line each, absolute on the site URL (P-23); default
// ["/sitemap.xml"].
const RobotsSitemapsConfigKey = "golem15.seo.robots_sitemaps"

// MaxRobotsBytes is the largest robots.txt the editor stores: Google
// ignores everything after 500 KiB.
const MaxRobotsBytes = 500 << 10

// The two fixed robots.txt bodies.
const (
	// NonProductionRobots is served outside production whatever is stored
	// (D-22).
	NonProductionRobots = "User-agent: *\nDisallow: /\n"
	// FallbackRobots is served when the stored file cannot be read and no
	// earlier body is cached: allow everything, never a 5xx.
	FallbackRobots = "User-agent: *\nDisallow:\n"
)

// RobotsSitemaps returns the configured sitemap addresses, absolute on
// the site URL.
func (s *Service) RobotsSitemaps() []string {
	var list []string
	if cfg := s.Config(); cfg != nil {
		if raw, ok := cfg.Lookup(RobotsSitemapsConfigKey); ok {
			switch v := raw.(type) {
			case []any:
				for _, item := range v {
					if str, ok := item.(string); ok {
						list = append(list, str)
					}
				}
			case []string:
				list = append(list, v...)
			case string:
				list = strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' })
			}
		}
	}
	if list == nil {
		list = []string{"/sitemap.xml"}
	}
	out := make([]string, 0, len(list))
	for _, item := range list {
		if item = strings.TrimSpace(item); item != "" {
			out = append(out, AbsoluteURL(s.SiteURL(), item))
		}
	}
	return out
}

// LoadRobots reads robots row 1; without one the defaults apply (no
// rules, every bot allowed).
func LoadRobots(db *gorm.DB) (models.Robots, error) {
	var rows []models.Robots
	if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil {
		return models.Robots{}, err
	}
	if len(rows) == 0 {
		return models.Robots{ID: 1, Bots: models.BotSwitches{}}, nil
	}
	return rows[0], nil
}

// ComposeRobots builds the production robots.txt (D-22, D-25, P-23): the
// stored rules, one "User-agent: <token>" / "Disallow: /" group per
// switched-off bot, then one Sitemap line per sitemap address.
func ComposeRobots(content string, bots models.BotSwitches, sitemaps []string) string {
	var b strings.Builder
	content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t")
	if content != "" {
		b.WriteString(content)
		b.WriteString("\n")
	}
	for _, bot := range Bots {
		if bots.Allowed(bot.Token) {
			continue
		}
		if b.Len() > 0 {
			b.WriteString("\n")
		}
		b.WriteString("User-agent: " + bot.Token + "\nDisallow: /\n")
	}
	if len(sitemaps) > 0 && b.Len() > 0 {
		b.WriteString("\n")
	}
	for _, sm := range sitemaps {
		b.WriteString("Sitemap: " + sm + "\n")
	}
	return b.String()
}

// generatedStart is the 1-based line where the generated part of a
// composed file starts: the lines after the stored rules.
func generatedStart(content string) int {
	content = strings.TrimRight(strings.ReplaceAll(content, "\r\n", "\n"), "\n \t")
	if content == "" {
		return 1
	}
	return strings.Count(content, "\n") + 2
}

// ServedFile is the robots.txt crawlers get (D-22): outside production
// always NonProductionRobots, else the composed file from the stored rules
// and switches. It caches the last body it served, which LastServedRobots
// answers when the store fails.
func (s *Service) ServedFile(ctx context.Context, production bool) (string, error) {
	if !production {
		return NonProductionRobots, nil
	}
	db, err := s.DB(ctx)
	if err != nil {
		return "", err
	}
	row, err := LoadRobots(db)
	if err != nil {
		return "", err
	}
	body := ComposeRobots(row.Content, row.Bots, s.RobotsSitemaps())
	if s != nil {
		s.robots.Store(&body)
	}
	return body, nil
}

// LastServedRobots returns the last production body ServedFile answered,
// "" when there is none.
func (s *Service) LastServedRobots() string {
	if s == nil {
		return ""
	}
	if p := s.robots.Load(); p != nil {
		return *p
	}
	return ""
}

// RobotsTest is the tester's answer (UI-SPEC E5): whether agent may fetch
// the address, the deciding line (0 and "" when no rule matched) and
// whether a bot switch decided it.
type RobotsTest struct {
	Allowed bool   `json:"allowed"`
	Line    int    `json:"line"`
	Rule    string `json:"rule"`
	Bot     bool   `json:"bot"`
	Agent   string `json:"agent"`
	Staging bool   `json:"staging"`
}

// TestRobots answers whether agent may fetch path under the production
// file composed from content and bots (unsaved values included), through
// grobotstxt, Google's matcher (D-22, D-99): longest match, Allow wins a
// tie, * and $.
func (s *Service) TestRobots(content string, bots models.BotSwitches, agent, path string) RobotsTest {
	file := ComposeRobots(content, bots, s.RobotsSitemaps())
	uri := AbsoluteURL(s.SiteURL(), path)
	if s.SiteURL() == "" {
		uri = "http://localhost" + path
	}
	m := grobotstxt.NewRobotsMatcher()
	out := RobotsTest{Agent: agent, Staging: s.Staging()}
	out.Allowed = m.AgentAllowed(file, agent, uri)
	if line := m.MatchingLine(); line > 0 {
		out.Line = line
		lines := strings.Split(file, "\n")
		if line <= len(lines) {
			out.Rule = strings.TrimSpace(lines[line-1])
		}
		out.Bot = !out.Allowed && line >= generatedStart(content)
	}
	return out
}

// noindexAddresses lists the addresses the site serves noindex that a
// robots.txt could block (D-24): exact URL rules and records whose own
// robots value says noindex.
func noindexAddresses(db *gorm.DB) ([]string, error) {
	seen := map[string]bool{}
	var out []string
	var rules []models.Rule
	if err := db.Where("match = ? AND robots LIKE ?", models.MatchExact, "%noindex%").Find(&rules).Error; err != nil {
		return nil, err
	}
	for _, r := range rules {
		if !seen[r.URL] {
			seen[r.URL] = true
			out = append(out, r.URL)
		}
	}
	var entries []models.Entry
	if err := db.Where("robots LIKE ? AND url <> ''", "%noindex%").Find(&entries).Error; err != nil {
		return nil, err
	}
	for _, e := range entries {
		if !seen[e.URL] {
			seen[e.URL] = true
			out = append(out, e.URL)
		}
	}
	sort.Strings(out)
	return out, nil
}

// BlockedNoindex lists the noindex addresses Googlebot may not fetch under
// the production file composed from content and bots (D-24): a crawler
// that cannot fetch a page never sees its noindex.
func (s *Service) BlockedNoindex(db *gorm.DB, content string, bots models.BotSwitches) ([]string, error) {
	urls, err := noindexAddresses(db)
	if err != nil {
		return nil, err
	}
	return s.blockedNoindex(urls, content, bots), nil
}

// blockedNoindex lists the addresses of urls Googlebot may not fetch under
// the production file composed from content and bots.
func (s *Service) blockedNoindex(urls []string, content string, bots models.BotSwitches) []string {
	var out []string
	for _, u := range urls {
		if !s.TestRobots(content, bots, "Googlebot", u).Allowed {
			out = append(out, u)
		}
	}
	return out
}