sm-sitemap-plugin/models/settings.go

package models

import (
	"net/url"
	"strconv"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"

	"git.golem15.com/golem15/summercms/modules/cabana"
	"git.golem15.com/golem15/summercms/modules/lagoon"
)

// errFrontendURL is the phrase key of the site address refusal (the
// Phase 17 frontend_url phrase, unchanged).
const errFrontendURL = "golem15.sitemap::lang.errors.frontend_url"

// Settings is the sitemap settings singleton (row id 1) on
// golem15_sitemap_settings (UI-SPEC C26). SiteURL is the public site
// address every <loc> starts with (empty: golem15.sitemap.site_url, then
// app.url), FileSize the addresses per file, DefaultLocale the site's
// default locale (the unprefixed one, and the target of x-default) and
// ExtraURLs the additional addresses. RebuildPendingUntil is the debounce
// time of the next queued rebuild, NumberedSize the file size the files are
// numbered for, BuiltAt the end of the last rebuild and RebuildRequestedAt
// the last rebuild asked for from the admin. The Phase 17 columns
// (frontend_url, priorities, tiers, the update manifest) stay in the table
// and are no longer read (D-32).
//
// The image and News sitemaps, IndexNow and the HTML sitemap (D-31, D-34):
// ImagesEnabled lists the sitemap-images files, NewsEnabled sitemap-news.xml
// with NewsPublication (empty: app.name) and NewsSources (type:<key> and
// category:<key>:<id> values; empty: every article type). IndexNowEnabled
// notifies api.indexnow.org with IndexNowKey after each rebuild, and the
// last answer is IndexNowLastAt, IndexNowLastCount and IndexNowLastStatus.
// HTMLEnabled serves the HTML sitemap API, HTMLPaths is the site's HTML
// sitemap address per locale and HTMLPageSize the links per page.
type Settings struct {
	ID                  uint                        `gorm:"column:id;primaryKey"`
	SiteURL             *string                     `gorm:"column:site_url"`
	FileSize            int                         `gorm:"column:file_size"`
	DefaultLocale       *string                     `gorm:"column:default_locale"`
	ExtraURLs           lagoon.Jsonable[[]ExtraURL] `gorm:"column:extra_urls"`
	RebuildPendingUntil *time.Time                  `gorm:"column:rebuild_pending_until"`
	NumberedSize        *int                        `gorm:"column:numbered_size"`
	BuiltAt             *time.Time                  `gorm:"column:built_at"`
	RebuildRequestedAt  *time.Time                  `gorm:"column:rebuild_requested_at"`

	ImagesEnabled      bool                               `gorm:"column:images_enabled"`
	NewsEnabled        bool                               `gorm:"column:news_enabled"`
	NewsPublication    *string                            `gorm:"column:news_publication"`
	NewsSources        lagoon.Jsonable[[]string]          `gorm:"column:news_sources"`
	IndexNowEnabled    bool                               `gorm:"column:indexnow_enabled"`
	IndexNowKey        *string                            `gorm:"column:indexnow_key"`
	IndexNowLastAt     *time.Time                         `gorm:"column:indexnow_last_at"`
	IndexNowLastCount  *int                               `gorm:"column:indexnow_last_count"`
	IndexNowLastStatus *int                               `gorm:"column:indexnow_last_status"`
	HTMLEnabled        bool                               `gorm:"column:html_enabled"`
	HTMLPaths          lagoon.Jsonable[map[string]string] `gorm:"column:html_paths"`
	HTMLPageSize       int                                `gorm:"column:html_page_size"`
}

// DefaultFileSize is the number of addresses per sitemap file when the
// setting is not stored (D-30); MaxFileSize is the protocol's limit.
const (
	DefaultFileSize = 1000
	MaxFileSize     = 50000
)

// ExtraURL is one additional address of the settings: URL is a path (or an
// absolute http(s) URL on the site), LastMod its optional last change as
// the admin's date picker sends it.
type ExtraURL struct {
	URL     string `json:"url"`
	LastMod string `json:"lastmod"`
}

// EffectiveFileSize is the stored file size, or DefaultFileSize when it is
// outside 1..MaxFileSize.
func (s Settings) EffectiveFileSize() int {
	if s.FileSize < 1 || s.FileSize > MaxFileSize {
		return DefaultFileSize
	}
	return s.FileSize
}

// Extra returns the additional addresses.
func (s Settings) Extra() []ExtraURL { return s.ExtraURLs.Get() }

func (Settings) TableName() string { return "golem15_sitemap_settings" }

// Fillable lists the columns the settings form edits.
func (Settings) Fillable() []string {
	return []string{"site_url", "file_size", "extra_urls", "images_enabled", "news_enabled", "news_publication", "news_sources", "indexnow_enabled", "html_enabled", "html_page_size"}
}

// Rules is empty: the settings controller runs Validate before a save.
func (Settings) Rules() map[string]string { return map[string]string{} }

// The phrase keys of the other settings refusals.
const (
	errFileSize  = "golem15.sitemap::lang.errors.file_size"
	errURLScheme = "golem15.sitemap::lang.errors.url_scheme"
	errLastmod   = "golem15.sitemap::lang.errors.lastmod"
	errNewsName  = "golem15.sitemap::lang.errors.news_publication"
	errNewsSrc   = "golem15.sitemap::lang.errors.news_source"
	errHTMLPath  = "golem15.sitemap::lang.errors.html_path"
	errPageSize  = "golem15.sitemap::lang.errors.page_size"
)

// MaxNewsPublication is the longest publication name.
const MaxNewsPublication = 255

// Validate answers every problem Problems finds as a validation error on
// its field; the settings controller calls it before a save, so the save
// answers 422 and the row is left as it was.
func (s Settings) Validate() error {
	problems := s.Problems()
	if len(problems) == 0 {
		return nil
	}
	details := make(map[string]any, len(problems))
	for field, keys := range problems {
		details[field] = keys
	}
	return &cabana.ValidationError{Details: details}
}

// Problems lists the settings' refusals by field (UI-SPEC C26): a site
// address that AllowedFrontendURL refuses, a file size outside 1..MaxFileSize and an additional
// address that is not a path or an http(s) URL (extra_urls.<i>.url) or
// whose lastmod is not a date (extra_urls.<i>.lastmod), a publication name
// over 255 characters, a malformed News source, an HTML sitemap address
// that ValidHTMLPath refuses (html_path) and a page size outside 50..500.
func (s Settings) Problems() map[string][]string {
	out := map[string][]string{}
	if !AllowedFrontendURL(Text(s.SiteURL)) {
		out["site_url"] = []string{errFrontendURL}
	}
	if s.FileSize < 1 || s.FileSize > MaxFileSize {
		out["file_size"] = []string{errFileSize}
	}
	for i, e := range s.Extra() {
		if !AllowedAddress(strings.TrimSpace(e.URL)) {
			out["extra_urls."+strconv.Itoa(i)+".url"] = []string{errURLScheme}
		}
		if mod := strings.TrimSpace(e.LastMod); mod != "" && !ValidDate(mod) {
			out["extra_urls."+strconv.Itoa(i)+".lastmod"] = []string{errLastmod}
		}
	}
	if name := Text(s.NewsPublication); utf8.RuneCountInString(name) > MaxNewsPublication || strings.ContainsFunc(name, unicode.IsControl) {
		out["news_publication"] = []string{errNewsName}
	}
	for _, v := range s.NewsSources.Get() {
		if !ValidNewsSource(strings.TrimSpace(v)) {
			out["news_sources"] = []string{errNewsSrc}
			break
		}
	}
	for _, p := range s.HTMLPaths.Get() {
		if p = strings.TrimSpace(p); p != "" && !ValidHTMLPath(p) {
			out["html_path"] = []string{errHTMLPath}
			break
		}
	}
	if s.HTMLPageSize < MinHTMLPageSize || s.HTMLPageSize > MaxHTMLPageSize {
		out["html_page_size"] = []string{errPageSize}
	}
	return out
}

// HTMLPathProblem is the refusal of an HTML sitemap address (errors.html_path).
const HTMLPathProblem = errHTMLPath

// ValidHTMLPath reports whether p may be the HTML sitemap's address: a
// path that starts with a single /, at most 255 characters of letters,
// digits and - _ . ~ / (no query, fragment, "//" or ".." segment), and not
// the site root.
func ValidHTMLPath(p string) bool {
	if len(p) < 2 || len(p) > 255 || p[0] != '/' || strings.Contains(p, "//") {
		return false
	}
	for _, seg := range strings.Split(p[1:], "/") {
		if seg == "." || seg == ".." {
			return false
		}
	}
	for _, r := range p {
		if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || strings.ContainsRune("-_.~/", r)) {
			return false
		}
	}
	return true
}

// The prefixes of the news_sources values (P33-15): type:<type key> and
// category:<type key>:<id>.
const (
	NewsSourceType     = "type:"
	NewsSourceCategory = "category:"
)

// ValidNewsSource reports whether v is a news_sources value:
// type:<type key> or category:<type key>:<id>, each part 1 to 128 letters,
// digits, '-', '_' or '.'.
func ValidNewsSource(v string) bool {
	switch {
	case strings.HasPrefix(v, NewsSourceType):
		return validSourcePart(strings.TrimPrefix(v, NewsSourceType))
	case strings.HasPrefix(v, NewsSourceCategory):
		rest := strings.TrimPrefix(v, NewsSourceCategory)
		i := strings.LastIndex(rest, ":")
		return i > 0 && validSourcePart(rest[:i]) && validSourcePart(rest[i+1:])
	}
	return false
}

func validSourcePart(s string) bool {
	if s == "" || len(s) > 128 {
		return false
	}
	for _, r := range s {
		if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.') {
			return false
		}
	}
	return true
}

// AllowedFrontendURL reports whether u may be the site address: the empty
// string, or an absolute http or https URL with a host and without
// whitespace, control characters, a double quote, <, > or &. The value
// starts every <loc> and hreflang link of the sitemaps, so a value outside
// this form (a javascript: URL, or text that closes the attribute it is
// written into) would reach their readers (T-17-15, T-33-12).
func AllowedFrontendURL(u string) bool {
	if u == "" {
		return true
	}
	if !utf8.ValidString(u) {
		return false
	}
	if strings.ContainsFunc(u, func(r rune) bool {
		return unicode.IsControl(r) || unicode.IsSpace(r) || r == '"' || r == '<' || r == '>' || r == '&'
	}) {
		return false
	}
	parsed, err := url.Parse(u)
	if err != nil {
		return false
	}
	if parsed.Scheme != "http" && parsed.Scheme != "https" {
		return false
	}
	return parsed.Opaque == "" && parsed.Hostname() != ""
}

// DefaultSettings is the settings read when no row is stored.
func DefaultSettings() Settings {
	return Settings{ID: 1, FileSize: DefaultFileSize, ImagesEnabled: true, HTMLEnabled: true, HTMLPageSize: DefaultHTMLPageSize}
}

// The HTML sitemap's links per page: DefaultHTMLPageSize when not stored,
// MinHTMLPageSize to MaxHTMLPageSize when set (UI-SPEC C26).
const (
	DefaultHTMLPageSize = 150
	MinHTMLPageSize     = 50
	MaxHTMLPageSize     = 500
)

// DefaultHTMLPaths are the HTML sitemap's addresses when none is stored.
func DefaultHTMLPaths() map[string]string {
	return map[string]string{"pl": "/mapa-strony", "en": "/sitemap"}
}

// EffectiveHTMLPageSize is the stored page size, or DefaultHTMLPageSize
// when it is outside MinHTMLPageSize..MaxHTMLPageSize.
func (s Settings) EffectiveHTMLPageSize() int {
	if s.HTMLPageSize < MinHTMLPageSize || s.HTMLPageSize > MaxHTMLPageSize {
		return DefaultHTMLPageSize
	}
	return s.HTMLPageSize
}

// EffectiveHTMLPaths are the stored HTML sitemap addresses without blank
// ones, or DefaultHTMLPaths when none is stored.
func (s Settings) EffectiveHTMLPaths() map[string]string {
	out := map[string]string{}
	for locale, path := range s.HTMLPaths.Get() {
		if p := strings.TrimSpace(path); p != "" && strings.TrimSpace(locale) != "" {
			out[locale] = p
		}
	}
	if len(out) == 0 {
		return DefaultHTMLPaths()
	}
	return out
}

// Sources returns the News sources without blank values.
func (s Settings) Sources() []string {
	var out []string
	for _, v := range s.NewsSources.Get() {
		if v = strings.TrimSpace(v); v != "" {
			out = append(out, v)
		}
	}
	return out
}

func init() {
	Register(Settings{})
}

// dateLayouts are the lastmod forms an additional address may carry.
var dateLayouts = []string{time.RFC3339Nano, "2006-01-02 15:04:05", "2006-01-02T15:04:05", "2006-01-02"}

// ParseDate reads a lastmod as the settings form or an import gives it
// (YYYY-MM-DD, "YYYY-MM-DD hh:mm:ss" or RFC 3339; UTC unless it names a
// zone); nil when empty or unreadable.
func ParseDate(s string) *time.Time {
	s = strings.TrimSpace(s)
	if s == "" {
		return nil
	}
	for _, layout := range dateLayouts {
		if t, err := time.Parse(layout, s); err == nil {
			t = t.UTC()
			return &t
		}
	}
	return nil
}

// ValidDate reports whether s is a lastmod ParseDate reads.
func ValidDate(s string) bool { return ParseDate(s) != nil }