package models import ( "net/url" "strconv" "strings" "time" "unicode" "unicode/utf8" "git.golem15.com/golem15/summercms/modules/cabana" "git.golem15.com/golem15/summercms/modules/lagoon" ) // errFrontendURL is the phrase key of the site address refusal (the // Phase 17 frontend_url phrase, unchanged). const errFrontendURL = "golem15.sitemap::lang.errors.frontend_url" // Settings is the sitemap settings singleton (row id 1) on // golem15_sitemap_settings (UI-SPEC C26). SiteURL is the public site // address every starts with (empty: golem15.sitemap.site_url, then // app.url), FileSize the addresses per file, DefaultLocale the site's // default locale (the unprefixed one, and the target of x-default) and // ExtraURLs the additional addresses. RebuildPendingUntil is the debounce // time of the next queued rebuild, NumberedSize the file size the files are // numbered for, BuiltAt the end of the last rebuild and RebuildRequestedAt // the last rebuild asked for from the admin. The Phase 17 columns // (frontend_url, priorities, tiers, the update manifest) stay in the table // and are no longer read (D-32). // // The image and News sitemaps, IndexNow and the HTML sitemap (D-31, D-34): // ImagesEnabled lists the sitemap-images files, NewsEnabled sitemap-news.xml // with NewsPublication (empty: app.name) and NewsSources (type: and // category:: values; empty: every article type). IndexNowEnabled // notifies api.indexnow.org with IndexNowKey after each rebuild, and the // last answer is IndexNowLastAt, IndexNowLastCount and IndexNowLastStatus. // HTMLEnabled serves the HTML sitemap API, HTMLPaths is the site's HTML // sitemap address per locale and HTMLPageSize the links per page. type Settings struct { ID uint `gorm:"column:id;primaryKey"` SiteURL *string `gorm:"column:site_url"` FileSize int `gorm:"column:file_size"` DefaultLocale *string `gorm:"column:default_locale"` ExtraURLs lagoon.Jsonable[[]ExtraURL] `gorm:"column:extra_urls"` RebuildPendingUntil *time.Time `gorm:"column:rebuild_pending_until"` NumberedSize *int `gorm:"column:numbered_size"` BuiltAt *time.Time `gorm:"column:built_at"` RebuildRequestedAt *time.Time `gorm:"column:rebuild_requested_at"` ImagesEnabled bool `gorm:"column:images_enabled"` NewsEnabled bool `gorm:"column:news_enabled"` NewsPublication *string `gorm:"column:news_publication"` NewsSources lagoon.Jsonable[[]string] `gorm:"column:news_sources"` IndexNowEnabled bool `gorm:"column:indexnow_enabled"` IndexNowKey *string `gorm:"column:indexnow_key"` IndexNowLastAt *time.Time `gorm:"column:indexnow_last_at"` IndexNowLastCount *int `gorm:"column:indexnow_last_count"` IndexNowLastStatus *int `gorm:"column:indexnow_last_status"` HTMLEnabled bool `gorm:"column:html_enabled"` HTMLPaths lagoon.Jsonable[map[string]string] `gorm:"column:html_paths"` HTMLPageSize int `gorm:"column:html_page_size"` } // DefaultFileSize is the number of addresses per sitemap file when the // setting is not stored (D-30); MaxFileSize is the protocol's limit. const ( DefaultFileSize = 1000 MaxFileSize = 50000 ) // ExtraURL is one additional address of the settings: URL is a path (or an // absolute http(s) URL on the site), LastMod its optional last change as // the admin's date picker sends it. type ExtraURL struct { URL string `json:"url"` LastMod string `json:"lastmod"` } // EffectiveFileSize is the stored file size, or DefaultFileSize when it is // outside 1..MaxFileSize. func (s Settings) EffectiveFileSize() int { if s.FileSize < 1 || s.FileSize > MaxFileSize { return DefaultFileSize } return s.FileSize } // Extra returns the additional addresses. func (s Settings) Extra() []ExtraURL { return s.ExtraURLs.Get() } func (Settings) TableName() string { return "golem15_sitemap_settings" } // Fillable lists the columns the settings form edits. func (Settings) Fillable() []string { return []string{"site_url", "file_size", "extra_urls", "images_enabled", "news_enabled", "news_publication", "news_sources", "indexnow_enabled"} } // Rules is empty: the settings controller runs Validate before a save. func (Settings) Rules() map[string]string { return map[string]string{} } // The phrase keys of the other settings refusals. const ( errFileSize = "golem15.sitemap::lang.errors.file_size" errURLScheme = "golem15.sitemap::lang.errors.url_scheme" errLastmod = "golem15.sitemap::lang.errors.lastmod" errNewsName = "golem15.sitemap::lang.errors.news_publication" errNewsSrc = "golem15.sitemap::lang.errors.news_source" ) // MaxNewsPublication is the longest publication name. const MaxNewsPublication = 255 // Validate answers every problem Problems finds as a validation error on // its field; the settings controller calls it before a save, so the save // answers 422 and the row is left as it was. func (s Settings) Validate() error { problems := s.Problems() if len(problems) == 0 { return nil } details := make(map[string]any, len(problems)) for field, keys := range problems { details[field] = keys } return &cabana.ValidationError{Details: details} } // Problems lists the settings' refusals by field (UI-SPEC C26): a site // address that AllowedFrontendURL refuses, a file size outside 1..MaxFileSize and an additional // address that is not a path or an http(s) URL (extra_urls..url) or // whose lastmod is not a date (extra_urls..lastmod). func (s Settings) Problems() map[string][]string { out := map[string][]string{} if !AllowedFrontendURL(Text(s.SiteURL)) { out["site_url"] = []string{errFrontendURL} } if s.FileSize < 1 || s.FileSize > MaxFileSize { out["file_size"] = []string{errFileSize} } for i, e := range s.Extra() { if !AllowedAddress(strings.TrimSpace(e.URL)) { out["extra_urls."+strconv.Itoa(i)+".url"] = []string{errURLScheme} } if mod := strings.TrimSpace(e.LastMod); mod != "" && !ValidDate(mod) { out["extra_urls."+strconv.Itoa(i)+".lastmod"] = []string{errLastmod} } } if name := Text(s.NewsPublication); utf8.RuneCountInString(name) > MaxNewsPublication || strings.ContainsFunc(name, unicode.IsControl) { out["news_publication"] = []string{errNewsName} } for _, v := range s.NewsSources.Get() { if !ValidNewsSource(strings.TrimSpace(v)) { out["news_sources"] = []string{errNewsSrc} break } } return out } // The prefixes of the news_sources values (P33-15): type: and // category::. const ( NewsSourceType = "type:" NewsSourceCategory = "category:" ) // ValidNewsSource reports whether v is a news_sources value: // type: or category::, each part 1 to 128 letters, // digits, '-', '_' or '.'. func ValidNewsSource(v string) bool { switch { case strings.HasPrefix(v, NewsSourceType): return validSourcePart(strings.TrimPrefix(v, NewsSourceType)) case strings.HasPrefix(v, NewsSourceCategory): rest := strings.TrimPrefix(v, NewsSourceCategory) i := strings.LastIndex(rest, ":") return i > 0 && validSourcePart(rest[:i]) && validSourcePart(rest[i+1:]) } return false } func validSourcePart(s string) bool { if s == "" || len(s) > 128 { return false } for _, r := range s { if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.') { return false } } return true } // AllowedFrontendURL reports whether u may be the site address: the empty // string, or an absolute http or https URL with a host and without // whitespace, control characters, a double quote, <, > or &. The value // starts every and hreflang link of the sitemaps, so a value outside // this form (a javascript: URL, or text that closes the attribute it is // written into) would reach their readers (T-17-15, T-33-12). func AllowedFrontendURL(u string) bool { if u == "" { return true } if !utf8.ValidString(u) { return false } if strings.ContainsFunc(u, func(r rune) bool { return unicode.IsControl(r) || unicode.IsSpace(r) || r == '"' || r == '<' || r == '>' || r == '&' }) { return false } parsed, err := url.Parse(u) if err != nil { return false } if parsed.Scheme != "http" && parsed.Scheme != "https" { return false } return parsed.Opaque == "" && parsed.Hostname() != "" } // DefaultSettings is the settings read when no row is stored. func DefaultSettings() Settings { return Settings{ID: 1, FileSize: DefaultFileSize, ImagesEnabled: true, HTMLEnabled: true, HTMLPageSize: DefaultHTMLPageSize} } // The HTML sitemap's links per page: DefaultHTMLPageSize when not stored, // MinHTMLPageSize to MaxHTMLPageSize when set (UI-SPEC C26). const ( DefaultHTMLPageSize = 150 MinHTMLPageSize = 50 MaxHTMLPageSize = 500 ) // DefaultHTMLPaths are the HTML sitemap's addresses when none is stored. func DefaultHTMLPaths() map[string]string { return map[string]string{"pl": "/mapa-strony", "en": "/sitemap"} } // EffectiveHTMLPageSize is the stored page size, or DefaultHTMLPageSize // when it is outside MinHTMLPageSize..MaxHTMLPageSize. func (s Settings) EffectiveHTMLPageSize() int { if s.HTMLPageSize < MinHTMLPageSize || s.HTMLPageSize > MaxHTMLPageSize { return DefaultHTMLPageSize } return s.HTMLPageSize } // EffectiveHTMLPaths are the stored HTML sitemap addresses without blank // ones, or DefaultHTMLPaths when none is stored. func (s Settings) EffectiveHTMLPaths() map[string]string { out := map[string]string{} for locale, path := range s.HTMLPaths.Get() { if p := strings.TrimSpace(path); p != "" && strings.TrimSpace(locale) != "" { out[locale] = p } } if len(out) == 0 { return DefaultHTMLPaths() } return out } // Sources returns the News sources without blank values. func (s Settings) Sources() []string { var out []string for _, v := range s.NewsSources.Get() { if v = strings.TrimSpace(v); v != "" { out = append(out, v) } } return out } func init() { Register(Settings{}) } // dateLayouts are the lastmod forms an additional address may carry. var dateLayouts = []string{time.RFC3339Nano, "2006-01-02 15:04:05", "2006-01-02T15:04:05", "2006-01-02"} // ParseDate reads a lastmod as the settings form or an import gives it // (YYYY-MM-DD, "YYYY-MM-DD hh:mm:ss" or RFC 3339; UTC unless it names a // zone); nil when empty or unreadable. func ParseDate(s string) *time.Time { s = strings.TrimSpace(s) if s == "" { return nil } for _, layout := range dateLayouts { if t, err := time.Parse(layout, s); err == nil { t = t.UTC() return &t } } return nil } // ValidDate reports whether s is a lastmod ParseDate reads. func ValidDate(s string) bool { return ParseDate(s) != nil }