sm-seo-plugin/models/settings.go

package models

import (
	"fmt"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode"

	"git.golem15.com/golem15/summercms/modules/lagoon"
)

// The query parameter kinds of the D-07 list.
const (
	ParamContent  = "content"
	ParamFilter   = "filter"
	ParamTracking = "tracking"
)

// The address options of D-20b.
const (
	HostKeep  = "keep"
	HostWWW   = "www"
	HostBare  = "bare"
	SlashKeep = "keep"
	SlashAdd  = "add"
	SlashDrop = "remove"
)

// Default templates (D-03).
const (
	DefaultTitleTemplate = "%title% | %site%"
	DefaultHomeTemplate  = "%site%"
)

// The phrase keys of the settings refusals.
const (
	errVerifyFilenameInvalid = "golem15.seo::lang.settings.verify_filename_invalid"
	errVerifyFilenameTaken   = "golem15.seo::lang.settings.verify_filename_taken"
	errTemplateUnknown       = "golem15.seo::lang.settings.template_unknown"
	errDomainInvalid         = "golem15.seo::lang.settings.domain_invalid"
	errValueInvalid          = "golem15.seo::lang.api.value_invalid"
	errSameAsInvalid         = "golem15.seo::lang.settings.same_as_invalid"
	errURLInvalid            = "golem15.seo::lang.settings.url_invalid"
	errEmailInvalid          = "golem15.seo::lang.settings.email_invalid"
	errLocalPerson           = "golem15.seo::lang.settings.local_person"
	errLocalAddress          = "golem15.seo::lang.settings.local_address_required"
	errTimeInvalid           = "golem15.seo::lang.settings.time_invalid"
	errHoursOrder            = "golem15.seo::lang.settings.hours_order"
	errCountryInvalid        = "golem15.seo::lang.settings.country_invalid"
	errCoordinateInvalid     = "golem15.seo::lang.settings.coordinate_invalid"
	errPageSegmentInvalid    = "golem15.seo::lang.settings.page_segment_invalid"
)

// PagePlaceholders are the placeholders a page suffix may use (D-41).
var PagePlaceholders = []string{"%page%", "%pages%"}

// SuffixProblems refuses a page suffix with a placeholder other than
// %page% and %pages%.
func SuffixProblems(suffix string) []Problem {
	for _, found := range placeholderPattern.FindAllString(suffix, -1) {
		if !oneOfList(found, PagePlaceholders) {
			return []Problem{{Key: errTemplateUnknown, Params: map[string]string{"placeholder": found}}}
		}
	}
	return nil
}

// PaginationProblems lists the refusals of the Pagination tab: the form,
// a segment outside [a-z0-9-] and unknown suffix placeholders.
func (s Settings) PaginationProblems() map[string][]Problem {
	out := map[string][]Problem{}
	if s.PageForm != "" && !oneOfList(s.PageForm, []string{PageFormQuery, PageFormPath}) {
		out["page_form"] = []Problem{{Key: errValueInvalid}}
	}
	if seg := strings.TrimSpace(s.PageSegment); seg != "" && !pageSegment.MatchString(seg) {
		out["page_segment"] = []Problem{{Key: errPageSegmentInvalid}}
	}
	for field, v := range map[string]string{"page_title_suffix": s.PageTitleSuffix, "page_description_suffix": s.PageDescriptionSuffix} {
		if p := SuffixProblems(v); p != nil {
			out[field] = p
		}
	}
	return out
}

// pageSegment is a valid page path segment.
var pageSegment = regexp.MustCompile(`^[a-z0-9-]+$`)

// LocalTypes are the LocalBusiness types the settings offer (UI-SPEC C20).
var LocalTypes = []string{
	"LocalBusiness", "ProfessionalService", "Store", "FoodEstablishment",
	"HealthAndBeautyBusiness", "MedicalBusiness", "LegalService",
	"HomeAndConstructionBusiness", "AutomotiveBusiness", "FinancialService",
}

// Days are the day codes of the opening hours, Monday first.
var Days = []string{"mon", "tue", "wed", "thu", "fri", "sat", "sun"}

// QueryParam is one row of the D-07 query-parameter list: a name (a
// trailing * matches a prefix), its kind, and an optional path pattern the
// row is limited to.
type QueryParam struct {
	Name string `json:"name"`
	Kind string `json:"kind"`
	Path string `json:"path"`
}

// VerifyMeta is one extra verification meta tag (D-05).
type VerifyMeta struct {
	Name    string `json:"name"`
	Content string `json:"content"`
}

// VerifyFile is one verification file served from the site root (D-05).
type VerifyFile struct {
	Filename string `json:"filename"`
	Body     string `json:"body"`
}

// The publisher types (D-45) and the page address forms (D-40).
const (
	PublisherOrganization = "organization"
	PublisherPerson       = "person"
	PageFormQuery         = "query"
	PageFormPath          = "path"
	DefaultLocalType      = "LocalBusiness"
)

// OpeningHours is one opening-hours row of a local business: a day (mon
// … sun) and its opening and closing time (24-hour HH:MM).
type OpeningHours struct {
	Day    string `json:"day"`
	Opens  string `json:"opens"`
	Closes string `json:"closes"`
}

// DefaultQueryParams is the seeded D-07 list.
func DefaultQueryParams() []QueryParam {
	return []QueryParam{
		{Name: "page", Kind: ParamContent},
		{Name: "utm_*", Kind: ParamTracking},
		{Name: "fbclid", Kind: ParamTracking},
		{Name: "gclid", Kind: ParamTracking},
		{Name: "msclkid", Kind: ParamTracking},
		{Name: "sort", Kind: ParamFilter},
	}
}

// Settings is the SEO settings singleton (row id 1) on golem15_seo_settings
// (D-03, D-05, D-07, D-20b, D-21, D-23). site_name and default_description
// are translatable; the other locales live with the translate plugin.
type Settings struct {
	ID                 uint                          `gorm:"column:id;primaryKey"`
	SiteName           string                        `gorm:"column:site_name"`
	TwitterSite        string                        `gorm:"column:twitter_site"`
	TitleTemplate      string                        `gorm:"column:title_template"`
	HomeTemplate       string                        `gorm:"column:home_template"`
	DefaultDescription string                        `gorm:"column:default_description"`
	DefaultOGImage     string                        `gorm:"column:default_og_image"`
	QueryParams        lagoon.Jsonable[[]QueryParam] `gorm:"column:query_params"`
	NoindexPaths       string                        `gorm:"column:noindex_paths"`
	SearchPaths        string                        `gorm:"column:search_paths"`
	URLHost            string                        `gorm:"column:url_host"`
	URLSlash           string                        `gorm:"column:url_slash"`
	URLLowercase       bool                          `gorm:"column:url_lowercase"`
	VerifyGoogle       string                        `gorm:"column:verify_google"`
	VerifyBing         string                        `gorm:"column:verify_bing"`
	VerifyYandex       string                        `gorm:"column:verify_yandex"`
	VerifyOther        lagoon.Jsonable[[]VerifyMeta] `gorm:"column:verify_other"`
	VerifyFiles        lagoon.Jsonable[[]VerifyFile] `gorm:"column:verify_files"`
	CSVDelimiter       string                        `gorm:"column:csv_delimiter"`
	// LinksNewTab opens links to other sites in a new tab (Links tab, on
	// by default); NofollowDomains and SponsoredDomains list the domains
	// whose links always get nofollow or sponsored, one per line,
	// subdomains included (D-83).
	LinksNewTab      bool   `gorm:"column:links_new_tab"`
	NofollowDomains  string `gorm:"column:nofollow_domains"`
	SponsoredDomains string `gorm:"column:sponsored_domains"`
	// BreadcrumbHome is the home crumb's name (translatable; empty: the
	// per-locale default copy).
	BreadcrumbHome string `gorm:"column:breadcrumb_home"`
	// PageForm is the paginated address form, query (?page=N) or path
	// (/<segment>/N); PageSegment the path segment, translatable per
	// locale; the suffixes are added on page 2 and later (D-40, D-41).
	PageForm              string `gorm:"column:page_form"`
	PageSegment           string `gorm:"column:page_segment"`
	PageTitleSuffix       string `gorm:"column:page_title_suffix"`
	PageDescriptionSuffix string `gorm:"column:page_description_suffix"`
	// The publisher of the site (D-45): organization or person, its name
	// (translatable; empty: the site name), logo or photo, website, email,
	// phone and sameAs profiles (one https URL per line).
	PublisherType   string `gorm:"column:publisher_type"`
	PublisherName   string `gorm:"column:publisher_name"`
	PublisherLogo   string `gorm:"column:publisher_logo"`
	PublisherURL    string `gorm:"column:publisher_url"`
	PublisherEmail  string `gorm:"column:publisher_email"`
	PublisherPhone  string `gorm:"column:publisher_phone"`
	PublisherSameAs string `gorm:"column:publisher_same_as"`
	// The local business data of an organisation publisher (D-45).
	LocalEnabled    bool                            `gorm:"column:local_enabled"`
	LocalType       string                          `gorm:"column:local_type"`
	LocalPriceRange string                          `gorm:"column:local_price_range"`
	LocalStreet     string                          `gorm:"column:local_street"`
	LocalPostalCode string                          `gorm:"column:local_postal_code"`
	LocalCity       string                          `gorm:"column:local_city"`
	LocalRegion     string                          `gorm:"column:local_region"`
	LocalCountry    string                          `gorm:"column:local_country"`
	LocalLatitude   string                          `gorm:"column:local_latitude"`
	LocalLongitude  string                          `gorm:"column:local_longitude"`
	LocalHours      lagoon.Jsonable[[]OpeningHours] `gorm:"column:local_hours"`
	// The Site audit tab (D-14, UI-SPEC C20): the automatic crawl (off,
	// daily or weekly at night), pages per crawl (10 to 50 000), the pause
	// between requests in milliseconds (100 to 5000), the crawls kept (1 to
	// 50) and the generic link texts, one per line.
	CrawlSchedule  string `gorm:"column:crawl_schedule"`
	CrawlMaxPages  int    `gorm:"column:crawl_max_pages"`
	CrawlDelay     int    `gorm:"column:crawl_delay"`
	CrawlKeep      int    `gorm:"column:crawl_keep"`
	GenericAnchors string `gorm:"column:generic_anchors"`
	// RoutesReportedAt is the time of the newest route report (D-33),
	// written by the report only.
	RoutesReportedAt *time.Time `gorm:"column:routes_reported_at"`
	CreatedAt        *time.Time `gorm:"column:created_at"`
	UpdatedAt        *time.Time `gorm:"column:updated_at"`
}

func (Settings) TableName() string { return "golem15_seo_settings" }

// MorphName names the settings row for translations.
func (Settings) MorphName() string { return `Golem15\SEO\Models\Settings` }

// Fillable lists the form's columns.
func (Settings) Fillable() []string {
	return []string{
		"site_name", "twitter_site", "title_template", "home_template",
		"default_description", "default_og_image", "query_params",
		"noindex_paths", "search_paths", "url_host", "url_slash",
		"url_lowercase", "verify_google", "verify_bing", "verify_yandex",
		"verify_other", "verify_files", "links_new_tab", "nofollow_domains",
		"sponsored_domains", "breadcrumb_home", "publisher_type",
		"publisher_name", "publisher_logo", "publisher_url", "publisher_email",
		"publisher_phone", "publisher_same_as", "local_enabled", "local_type",
		"local_price_range", "local_street", "local_postal_code", "local_city",
		"local_region", "local_country", "local_latitude", "local_longitude",
		"local_hours", "page_form", "page_segment", "page_title_suffix",
		"page_description_suffix", "crawl_schedule", "crawl_max_pages",
		"crawl_delay", "crawl_keep", "generic_anchors",
	}
}

// Rules is empty: the settings controller checks Problems before a save.
func (Settings) Rules() map[string]string { return map[string]string{} }

// DefaultSettings is what the resolver reads when the row is missing.
func DefaultSettings() Settings {
	return Settings{
		ID:             1,
		TitleTemplate:  DefaultTitleTemplate,
		HomeTemplate:   DefaultHomeTemplate,
		QueryParams:    lagoon.Jsonable[[]QueryParam]{Data: DefaultQueryParams(), Valid: true},
		URLHost:        HostKeep,
		URLSlash:       SlashKeep,
		LinksNewTab:    true,
		PageForm:       PageFormQuery,
		PublisherType:  PublisherOrganization,
		LocalType:      DefaultLocalType,
		CrawlSchedule:  CrawlOff,
		CrawlMaxPages:  DefaultCrawlMaxPages,
		CrawlDelay:     DefaultCrawlDelay,
		CrawlKeep:      DefaultCrawlKeep,
		GenericAnchors: DefaultGenericAnchors,
	}
}

// The automatic crawl settings (crawl_schedule).
const (
	CrawlOff    = "off"
	CrawlDaily  = "daily"
	CrawlWeekly = "weekly"
)

// The Site audit defaults and ranges (UI-SPEC C20).
const (
	DefaultCrawlMaxPages = 5000
	DefaultCrawlDelay    = 250
	DefaultCrawlKeep     = 10
	MinCrawlMaxPages     = 10
	MaxCrawlMaxPages     = 50_000
	MinCrawlDelay        = 100
	MaxCrawlDelay        = 5000
	MinCrawlKeep         = 1
	MaxCrawlKeep         = 50
)

// DefaultGenericAnchors is the seed of the generic link texts: the en and
// pl lists of UI-SPEC C27, one per line.
const DefaultGenericAnchors = "click here\nhere\nmore\nread more\nlearn more\nthis link\nlink\nkliknij tutaj\ntutaj\ntu\nwięcej\nczytaj więcej\nczytaj dalej\nten link"

// AuditProblems refuses Site audit values outside their ranges
// (lagoon::validation.between.numeric, UI-SPEC C20) and an unknown
// schedule. The settings screen and the API check it on save; Problems
// leaves it out, so a partial Settings value checks its templates only.
func (s Settings) AuditProblems() map[string][]Problem {
	out := map[string][]Problem{}
	between := func(field string, v, min, max int) {
		if v < min || v > max {
			out[field] = append(out[field], Problem{Key: "lagoon::validation.between.numeric", Params: map[string]string{
				"attribute": strings.ReplaceAll(field, "_", " "), "min": strconv.Itoa(min), "max": strconv.Itoa(max),
			}})
		}
	}
	between("crawl_max_pages", s.CrawlMaxPages, MinCrawlMaxPages, MaxCrawlMaxPages)
	between("crawl_delay", s.CrawlDelay, MinCrawlDelay, MaxCrawlDelay)
	between("crawl_keep", s.CrawlKeep, MinCrawlKeep, MaxCrawlKeep)
	switch s.CrawlSchedule {
	case CrawlOff, CrawlDaily, CrawlWeekly:
	default:
		out["crawl_schedule"] = append(out["crawl_schedule"], Problem{Key: "golem15.seo::lang.api.value_invalid"})
	}
	return out
}

// Hours returns the local business's opening hours.
func (s Settings) Hours() []OpeningHours { return s.LocalHours.Get() }

// Params returns the query-parameter list.
func (s Settings) Params() []QueryParam { return s.QueryParams.Get() }

// OtherMeta returns the extra verification meta tags.
func (s Settings) OtherMeta() []VerifyMeta { return s.VerifyOther.Get() }

// Files returns the verification files.
func (s Settings) Files() []VerifyFile { return s.VerifyFiles.Get() }

// verifyFilename is the allowed form of a verification file name.
var verifyFilename = regexp.MustCompile(`^[A-Za-z0-9._-]+\.(html|txt|xml)$`)

// placeholderPattern finds %name% placeholders in a template.
var placeholderPattern = regexp.MustCompile(`%[A-Za-z0-9_]*%`)

// KnownPlaceholders are the placeholders a template may use.
var KnownPlaceholders = []string{"%title%", "%description%", "%site%"}

// UnknownPlaceholder returns the first placeholder of tpl that is not
// known, "" when every one is.
func UnknownPlaceholder(tpl string) string {
	for _, found := range placeholderPattern.FindAllString(tpl, -1) {
		known := false
		for _, k := range KnownPlaceholders {
			if found == k {
				known = true
				break
			}
		}
		if !known {
			return found
		}
	}
	return ""
}

// Problem is one field refusal: a phrase key and its :name parameters.
// The admin controllers translate it in the request locale.
type Problem struct {
	Key    string
	Params map[string]string
}

// TemplateProblems refuses a template with an unknown placeholder.
func TemplateProblems(tpl string) []Problem {
	if p := UnknownPlaceholder(tpl); p != "" {
		return []Problem{{Key: errTemplateUnknown, Params: map[string]string{"placeholder": p}}}
	}
	return nil
}

// Problems lists the settings refusals per field: templates with an unknown
// placeholder, and verification file names that are malformed or used
// twice (verify_files.<i>.filename).
func (s Settings) Problems() map[string][]Problem {
	out := map[string][]Problem{}
	for field, tpl := range map[string]string{"title_template": s.TitleTemplate, "home_template": s.HomeTemplate} {
		if p := TemplateProblems(tpl); p != nil {
			out[field] = p
		}
	}
	seen := map[string]bool{}
	for i, f := range s.Files() {
		name := strings.TrimSpace(f.Filename)
		key := fmt.Sprintf("verify_files.%d.filename", i)
		switch {
		case !verifyFilename.MatchString(name):
			out[key] = []Problem{{Key: errVerifyFilenameInvalid}}
		case seen[strings.ToLower(name)]:
			out[key] = []Problem{{Key: errVerifyFilenameTaken}}
		}
		seen[strings.ToLower(name)] = true
	}
	for field, list := range map[string]string{"nofollow_domains": s.NofollowDomains, "sponsored_domains": s.SponsoredDomains} {
		if p := DomainProblems(list); p != nil {
			out[field] = p
		}
	}
	for field, list := range s.PaginationProblems() {
		out[field] = append(out[field], list...)
	}
	for field, list := range s.PublisherProblems() {
		out[field] = append(out[field], list...)
	}
	return out
}

// timePattern is a 24-hour HH:MM time.
var timePattern = regexp.MustCompile(`^([01][0-9]|2[0-3]):[0-5][0-9]$`)

// countryPattern is a two-letter country code.
var countryPattern = regexp.MustCompile(`^[A-Za-z]{2}$`)

// HTTPSURL reports whether s is an absolute https URL with a host.
func HTTPSURL(s string) bool {
	u, err := url.Parse(strings.TrimSpace(s))
	return err == nil && u.Scheme == "https" && u.Host != "" && !strings.ContainsAny(s, " \t\n<>\"")
}

// oneOfList reports whether v is one of list.
func oneOfList(v string, list []string) bool {
	for _, item := range list {
		if v == item {
			return true
		}
	}
	return false
}

// PublisherProblems lists the refusals of the Publisher and Local
// business tabs (UI-SPEC C20, D-45, T-33-25): the publisher type, https
// sameAs lines (with the line number), the website, the email, a person
// as a local business, the business's address, country, coordinates and
// opening hours (local_hours.<i>.opens / .closes).
func (s Settings) PublisherProblems() map[string][]Problem {
	out := map[string][]Problem{}
	add := func(field string, p Problem) { out[field] = append(out[field], p) }
	if s.PublisherType != "" && !oneOfList(s.PublisherType, []string{PublisherOrganization, PublisherPerson}) {
		add("publisher_type", Problem{Key: errValueInvalid})
	}
	n := 0
	for _, line := range strings.Split(strings.ReplaceAll(s.PublisherSameAs, "\r\n", "\n"), "\n") {
		n++
		if v := strings.TrimSpace(line); v != "" && !HTTPSURL(v) {
			add("publisher_same_as", Problem{Key: errSameAsInvalid, Params: map[string]string{"line": strconv.Itoa(n)}})
			break
		}
	}
	if v := strings.TrimSpace(s.PublisherURL); v != "" && !HTTPSURL(v) {
		add("publisher_url", Problem{Key: errURLInvalid})
	}
	if v := strings.TrimSpace(s.PublisherEmail); v != "" {
		local, domain, ok := strings.Cut(v, "@")
		if !ok || local == "" || !strings.Contains(domain, ".") || strings.ContainsAny(v, " <>\"") {
			add("publisher_email", Problem{Key: errEmailInvalid})
		}
	}
	if !s.LocalEnabled {
		return out
	}
	if s.PublisherType == PublisherPerson {
		add("local_enabled", Problem{Key: errLocalPerson})
	}
	if s.LocalType != "" && !oneOfList(s.LocalType, LocalTypes) {
		add("local_type", Problem{Key: errValueInvalid})
	}
	if strings.TrimSpace(s.LocalStreet) == "" || strings.TrimSpace(s.LocalPostalCode) == "" || strings.TrimSpace(s.LocalCity) == "" {
		for field, v := range map[string]string{"local_street": s.LocalStreet, "local_postal_code": s.LocalPostalCode, "local_city": s.LocalCity} {
			if strings.TrimSpace(v) == "" {
				add(field, Problem{Key: errLocalAddress})
			}
		}
	}
	if v := strings.TrimSpace(s.LocalCountry); v != "" && !countryPattern.MatchString(v) {
		add("local_country", Problem{Key: errCountryInvalid})
	}
	coordinate := func(field, v string, limit float64) {
		if v = strings.TrimSpace(v); v == "" {
			return
		}
		f, err := strconv.ParseFloat(strings.ReplaceAll(v, ",", "."), 64)
		if err != nil || f < -limit || f > limit {
			add(field, Problem{Key: errCoordinateInvalid})
		}
	}
	coordinate("local_latitude", s.LocalLatitude, 90)
	coordinate("local_longitude", s.LocalLongitude, 180)
	for i, h := range s.Hours() {
		base := fmt.Sprintf("local_hours.%d.", i)
		if !oneOfList(h.Day, Days) {
			add(base+"day", Problem{Key: errValueInvalid})
		}
		okOpens, okCloses := timePattern.MatchString(h.Opens), timePattern.MatchString(h.Closes)
		if !okOpens {
			add(base+"opens", Problem{Key: errTimeInvalid})
		}
		if !okCloses {
			add(base+"closes", Problem{Key: errTimeInvalid})
		}
		if okOpens && okCloses && h.Closes <= h.Opens {
			add(base+"closes", Problem{Key: errHoursOrder})
		}
	}
	return out
}

// domainLabel is one label of a host name: letters (any script, written
// in lowercase), digits and inner hyphens, at most 63 bytes.
func domainLabel(label string) bool {
	if label == "" || len(label) > 63 || strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") {
		return false
	}
	for _, r := range label {
		if r != '-' && !unicode.IsDigit(r) && !(unicode.IsLetter(r) && !unicode.IsUpper(r)) {
			return false
		}
	}
	return true
}

// ValidDomain reports whether d is a host name with at least two labels,
// without a scheme, port, path or trailing dot.
func ValidDomain(d string) bool {
	if d == "" || len(d) > 253 {
		return false
	}
	labels := strings.Split(d, ".")
	if len(labels) < 2 {
		return false
	}
	for _, l := range labels {
		if !domainLabel(l) {
			return false
		}
	}
	return true
}

// DomainLines splits a domain list into its trimmed, lowercased,
// non-empty lines, in order and each once.
func DomainLines(list string) []string {
	out := []string{}
	seen := map[string]bool{}
	for _, line := range strings.Split(strings.ReplaceAll(list, "\r\n", "\n"), "\n") {
		d := strings.ToLower(strings.TrimSpace(line))
		if d == "" || seen[d] {
			continue
		}
		seen[d] = true
		out = append(out, d)
	}
	return out
}

// NormalizeDomains rewrites a domain list as DomainLines, one per line.
func NormalizeDomains(list string) string {
	return strings.Join(DomainLines(list), "\n")
}

// DomainProblems refuses the first line of list that is not a domain
// (settings.domain_invalid with its line number and text).
func DomainProblems(list string) []Problem {
	n := 0
	for _, line := range strings.Split(strings.ReplaceAll(list, "\r\n", "\n"), "\n") {
		n++
		d := strings.ToLower(strings.TrimSpace(line))
		if d == "" || ValidDomain(d) {
			continue
		}
		return []Problem{{Key: errDomainInvalid, Params: map[string]string{"line": strconv.Itoa(n), "value": strings.TrimSpace(line)}}}
	}
	return nil
}

func init() {
	Register(Settings{})
}