package models import ( "fmt" "net/url" "regexp" "strconv" "strings" "time" "unicode" "git.golem15.com/golem15/summercms/modules/lagoon" ) // The query parameter kinds of the D-07 list. const ( ParamContent = "content" ParamFilter = "filter" ParamTracking = "tracking" ) // The address options of D-20b. const ( HostKeep = "keep" HostWWW = "www" HostBare = "bare" SlashKeep = "keep" SlashAdd = "add" SlashDrop = "remove" ) // Default templates (D-03). const ( DefaultTitleTemplate = "%title% | %site%" DefaultHomeTemplate = "%site%" ) // The phrase keys of the settings refusals. const ( errVerifyFilenameInvalid = "golem15.seo::lang.settings.verify_filename_invalid" errVerifyFilenameTaken = "golem15.seo::lang.settings.verify_filename_taken" errTemplateUnknown = "golem15.seo::lang.settings.template_unknown" errDomainInvalid = "golem15.seo::lang.settings.domain_invalid" errValueInvalid = "golem15.seo::lang.api.value_invalid" errSameAsInvalid = "golem15.seo::lang.settings.same_as_invalid" errURLInvalid = "golem15.seo::lang.settings.url_invalid" errEmailInvalid = "golem15.seo::lang.settings.email_invalid" errLocalPerson = "golem15.seo::lang.settings.local_person" errLocalAddress = "golem15.seo::lang.settings.local_address_required" errTimeInvalid = "golem15.seo::lang.settings.time_invalid" errHoursOrder = "golem15.seo::lang.settings.hours_order" errCountryInvalid = "golem15.seo::lang.settings.country_invalid" errCoordinateInvalid = "golem15.seo::lang.settings.coordinate_invalid" errPageSegmentInvalid = "golem15.seo::lang.settings.page_segment_invalid" ) // PagePlaceholders are the placeholders a page suffix may use (D-41). var PagePlaceholders = []string{"%page%", "%pages%"} // SuffixProblems refuses a page suffix with a placeholder other than // %page% and %pages%. func SuffixProblems(suffix string) []Problem { for _, found := range placeholderPattern.FindAllString(suffix, -1) { if !oneOfList(found, PagePlaceholders) { return []Problem{{Key: errTemplateUnknown, Params: map[string]string{"placeholder": found}}} } } return nil } // PaginationProblems lists the refusals of the Pagination tab: the form, // a segment outside [a-z0-9-] and unknown suffix placeholders. func (s Settings) PaginationProblems() map[string][]Problem { out := map[string][]Problem{} if s.PageForm != "" && !oneOfList(s.PageForm, []string{PageFormQuery, PageFormPath}) { out["page_form"] = []Problem{{Key: errValueInvalid}} } if seg := strings.TrimSpace(s.PageSegment); seg != "" && !pageSegment.MatchString(seg) { out["page_segment"] = []Problem{{Key: errPageSegmentInvalid}} } for field, v := range map[string]string{"page_title_suffix": s.PageTitleSuffix, "page_description_suffix": s.PageDescriptionSuffix} { if p := SuffixProblems(v); p != nil { out[field] = p } } return out } // pageSegment is a valid page path segment. var pageSegment = regexp.MustCompile(`^[a-z0-9-]+$`) // LocalTypes are the LocalBusiness types the settings offer (UI-SPEC C20). var LocalTypes = []string{ "LocalBusiness", "ProfessionalService", "Store", "FoodEstablishment", "HealthAndBeautyBusiness", "MedicalBusiness", "LegalService", "HomeAndConstructionBusiness", "AutomotiveBusiness", "FinancialService", } // Days are the day codes of the opening hours, Monday first. var Days = []string{"mon", "tue", "wed", "thu", "fri", "sat", "sun"} // QueryParam is one row of the D-07 query-parameter list: a name (a // trailing * matches a prefix), its kind, and an optional path pattern the // row is limited to. type QueryParam struct { Name string `json:"name"` Kind string `json:"kind"` Path string `json:"path"` } // VerifyMeta is one extra verification meta tag (D-05). type VerifyMeta struct { Name string `json:"name"` Content string `json:"content"` } // VerifyFile is one verification file served from the site root (D-05). type VerifyFile struct { Filename string `json:"filename"` Body string `json:"body"` } // The publisher types (D-45) and the page address forms (D-40). const ( PublisherOrganization = "organization" PublisherPerson = "person" PageFormQuery = "query" PageFormPath = "path" DefaultLocalType = "LocalBusiness" ) // OpeningHours is one opening-hours row of a local business: a day (mon // … sun) and its opening and closing time (24-hour HH:MM). type OpeningHours struct { Day string `json:"day"` Opens string `json:"opens"` Closes string `json:"closes"` } // DefaultQueryParams is the seeded D-07 list. func DefaultQueryParams() []QueryParam { return []QueryParam{ {Name: "page", Kind: ParamContent}, {Name: "utm_*", Kind: ParamTracking}, {Name: "fbclid", Kind: ParamTracking}, {Name: "gclid", Kind: ParamTracking}, {Name: "msclkid", Kind: ParamTracking}, {Name: "sort", Kind: ParamFilter}, } } // Settings is the SEO settings singleton (row id 1) on golem15_seo_settings // (D-03, D-05, D-07, D-20b, D-21, D-23). site_name and default_description // are translatable; the other locales live with the translate plugin. type Settings struct { ID uint `gorm:"column:id;primaryKey"` SiteName string `gorm:"column:site_name"` TwitterSite string `gorm:"column:twitter_site"` TitleTemplate string `gorm:"column:title_template"` HomeTemplate string `gorm:"column:home_template"` DefaultDescription string `gorm:"column:default_description"` DefaultOGImage string `gorm:"column:default_og_image"` QueryParams lagoon.Jsonable[[]QueryParam] `gorm:"column:query_params"` NoindexPaths string `gorm:"column:noindex_paths"` SearchPaths string `gorm:"column:search_paths"` URLHost string `gorm:"column:url_host"` URLSlash string `gorm:"column:url_slash"` URLLowercase bool `gorm:"column:url_lowercase"` VerifyGoogle string `gorm:"column:verify_google"` VerifyBing string `gorm:"column:verify_bing"` VerifyYandex string `gorm:"column:verify_yandex"` VerifyOther lagoon.Jsonable[[]VerifyMeta] `gorm:"column:verify_other"` VerifyFiles lagoon.Jsonable[[]VerifyFile] `gorm:"column:verify_files"` CSVDelimiter string `gorm:"column:csv_delimiter"` // LinksNewTab opens links to other sites in a new tab (Links tab, on // by default); NofollowDomains and SponsoredDomains list the domains // whose links always get nofollow or sponsored, one per line, // subdomains included (D-83). LinksNewTab bool `gorm:"column:links_new_tab"` NofollowDomains string `gorm:"column:nofollow_domains"` SponsoredDomains string `gorm:"column:sponsored_domains"` // AutolinkMax is the most phrases of SEO › Auto-links linked in one // text (Links tab, 0 to 20, 3 by default); 0 turns auto-linking off // (D-82). AutolinkMax int `gorm:"column:autolink_max"` // BreadcrumbHome is the home crumb's name (translatable; empty: the // per-locale default copy). BreadcrumbHome string `gorm:"column:breadcrumb_home"` // PageForm is the paginated address form, query (?page=N) or path // (//N); PageSegment the path segment, translatable per // locale; the suffixes are added on page 2 and later (D-40, D-41). PageForm string `gorm:"column:page_form"` PageSegment string `gorm:"column:page_segment"` PageTitleSuffix string `gorm:"column:page_title_suffix"` PageDescriptionSuffix string `gorm:"column:page_description_suffix"` // The publisher of the site (D-45): organization or person, its name // (translatable; empty: the site name), logo or photo, website, email, // phone and sameAs profiles (one https URL per line). PublisherType string `gorm:"column:publisher_type"` PublisherName string `gorm:"column:publisher_name"` PublisherLogo string `gorm:"column:publisher_logo"` PublisherURL string `gorm:"column:publisher_url"` PublisherEmail string `gorm:"column:publisher_email"` PublisherPhone string `gorm:"column:publisher_phone"` PublisherSameAs string `gorm:"column:publisher_same_as"` // The local business data of an organisation publisher (D-45). LocalEnabled bool `gorm:"column:local_enabled"` LocalType string `gorm:"column:local_type"` LocalPriceRange string `gorm:"column:local_price_range"` LocalStreet string `gorm:"column:local_street"` LocalPostalCode string `gorm:"column:local_postal_code"` LocalCity string `gorm:"column:local_city"` LocalRegion string `gorm:"column:local_region"` LocalCountry string `gorm:"column:local_country"` LocalLatitude string `gorm:"column:local_latitude"` LocalLongitude string `gorm:"column:local_longitude"` LocalHours lagoon.Jsonable[[]OpeningHours] `gorm:"column:local_hours"` // The Site audit tab (D-14, UI-SPEC C20): the automatic crawl (off, // daily or weekly at night), pages per crawl (10 to 50 000), the pause // between requests in milliseconds (100 to 5000), the crawls kept (1 to // 50) and the generic link texts, one per line. CrawlSchedule string `gorm:"column:crawl_schedule"` CrawlMaxPages int `gorm:"column:crawl_max_pages"` CrawlDelay int `gorm:"column:crawl_delay"` CrawlKeep int `gorm:"column:crawl_keep"` GenericAnchors string `gorm:"column:generic_anchors"` // RoutesReportedAt is the time of the newest route report (D-33), // written by the report only. RoutesReportedAt *time.Time `gorm:"column:routes_reported_at"` CreatedAt *time.Time `gorm:"column:created_at"` UpdatedAt *time.Time `gorm:"column:updated_at"` } func (Settings) TableName() string { return "golem15_seo_settings" } // MorphName names the settings row for translations. func (Settings) MorphName() string { return `Golem15\SEO\Models\Settings` } // Fillable lists the form's columns. func (Settings) Fillable() []string { return []string{ "site_name", "twitter_site", "title_template", "home_template", "default_description", "default_og_image", "query_params", "noindex_paths", "search_paths", "url_host", "url_slash", "url_lowercase", "verify_google", "verify_bing", "verify_yandex", "verify_other", "verify_files", "links_new_tab", "nofollow_domains", "sponsored_domains", "breadcrumb_home", "publisher_type", "publisher_name", "publisher_logo", "publisher_url", "publisher_email", "publisher_phone", "publisher_same_as", "local_enabled", "local_type", "local_price_range", "local_street", "local_postal_code", "local_city", "local_region", "local_country", "local_latitude", "local_longitude", "local_hours", "page_form", "page_segment", "page_title_suffix", "page_description_suffix", "crawl_schedule", "crawl_max_pages", "crawl_delay", "crawl_keep", "generic_anchors", "autolink_max", } } // Rules is empty: the settings controller checks Problems before a save. func (Settings) Rules() map[string]string { return map[string]string{} } // DefaultSettings is what the resolver reads when the row is missing. func DefaultSettings() Settings { return Settings{ ID: 1, TitleTemplate: DefaultTitleTemplate, HomeTemplate: DefaultHomeTemplate, QueryParams: lagoon.Jsonable[[]QueryParam]{Data: DefaultQueryParams(), Valid: true}, URLHost: HostKeep, URLSlash: SlashKeep, LinksNewTab: true, AutolinkMax: DefaultAutolinkMax, PageForm: PageFormQuery, PublisherType: PublisherOrganization, LocalType: DefaultLocalType, CrawlSchedule: CrawlOff, CrawlMaxPages: DefaultCrawlMaxPages, CrawlDelay: DefaultCrawlDelay, CrawlKeep: DefaultCrawlKeep, GenericAnchors: DefaultGenericAnchors, } } // The automatic crawl settings (crawl_schedule). const ( CrawlOff = "off" CrawlDaily = "daily" CrawlWeekly = "weekly" ) // The Site audit defaults and ranges (UI-SPEC C20). const ( DefaultCrawlMaxPages = 5000 DefaultCrawlDelay = 250 DefaultCrawlKeep = 10 MinCrawlMaxPages = 10 MaxCrawlMaxPages = 50_000 MinCrawlDelay = 100 MaxCrawlDelay = 5000 MinCrawlKeep = 1 MaxCrawlKeep = 50 ) // DefaultGenericAnchors is the seed of the generic link texts: the en and // pl lists of UI-SPEC C27, one per line. const DefaultGenericAnchors = "click here\nhere\nmore\nread more\nlearn more\nthis link\nlink\nkliknij tutaj\ntutaj\ntu\nwięcej\nczytaj więcej\nczytaj dalej\nten link" // AuditProblems refuses Site audit values and the Links tab's // autolink_max outside their ranges (lagoon::validation.between.numeric, // UI-SPEC C20) and an unknown schedule. The settings screen and the API check it on save; Problems // leaves it out, so a partial Settings value checks its templates only. func (s Settings) AuditProblems() map[string][]Problem { out := map[string][]Problem{} between := func(field string, v, min, max int) { if v < min || v > max { out[field] = append(out[field], Problem{Key: "lagoon::validation.between.numeric", Params: map[string]string{ "attribute": strings.ReplaceAll(field, "_", " "), "min": strconv.Itoa(min), "max": strconv.Itoa(max), }}) } } between("crawl_max_pages", s.CrawlMaxPages, MinCrawlMaxPages, MaxCrawlMaxPages) between("crawl_delay", s.CrawlDelay, MinCrawlDelay, MaxCrawlDelay) between("crawl_keep", s.CrawlKeep, MinCrawlKeep, MaxCrawlKeep) between("autolink_max", s.AutolinkMax, MinAutolinkMax, MaxAutolinkMax) switch s.CrawlSchedule { case CrawlOff, CrawlDaily, CrawlWeekly: default: out["crawl_schedule"] = append(out["crawl_schedule"], Problem{Key: "golem15.seo::lang.api.value_invalid"}) } return out } // Hours returns the local business's opening hours. func (s Settings) Hours() []OpeningHours { return s.LocalHours.Get() } // Params returns the query-parameter list. func (s Settings) Params() []QueryParam { return s.QueryParams.Get() } // OtherMeta returns the extra verification meta tags. func (s Settings) OtherMeta() []VerifyMeta { return s.VerifyOther.Get() } // Files returns the verification files. func (s Settings) Files() []VerifyFile { return s.VerifyFiles.Get() } // verifyFilename is the allowed form of a verification file name. var verifyFilename = regexp.MustCompile(`^[A-Za-z0-9._-]+\.(html|txt|xml)$`) // placeholderPattern finds %name% placeholders in a template. var placeholderPattern = regexp.MustCompile(`%[A-Za-z0-9_]*%`) // KnownPlaceholders are the placeholders a template may use. var KnownPlaceholders = []string{"%title%", "%description%", "%site%"} // UnknownPlaceholder returns the first placeholder of tpl that is not // known, "" when every one is. func UnknownPlaceholder(tpl string) string { for _, found := range placeholderPattern.FindAllString(tpl, -1) { known := false for _, k := range KnownPlaceholders { if found == k { known = true break } } if !known { return found } } return "" } // Problem is one field refusal: a phrase key and its :name parameters. // The admin controllers translate it in the request locale. type Problem struct { Key string Params map[string]string } // TemplateProblems refuses a template with an unknown placeholder. func TemplateProblems(tpl string) []Problem { if p := UnknownPlaceholder(tpl); p != "" { return []Problem{{Key: errTemplateUnknown, Params: map[string]string{"placeholder": p}}} } return nil } // Problems lists the settings refusals per field: templates with an unknown // placeholder, and verification file names that are malformed or used // twice (verify_files..filename). func (s Settings) Problems() map[string][]Problem { out := map[string][]Problem{} for field, tpl := range map[string]string{"title_template": s.TitleTemplate, "home_template": s.HomeTemplate} { if p := TemplateProblems(tpl); p != nil { out[field] = p } } seen := map[string]bool{} for i, f := range s.Files() { name := strings.TrimSpace(f.Filename) key := fmt.Sprintf("verify_files.%d.filename", i) switch { case !verifyFilename.MatchString(name): out[key] = []Problem{{Key: errVerifyFilenameInvalid}} case seen[strings.ToLower(name)]: out[key] = []Problem{{Key: errVerifyFilenameTaken}} } seen[strings.ToLower(name)] = true } for field, list := range map[string]string{"nofollow_domains": s.NofollowDomains, "sponsored_domains": s.SponsoredDomains} { if p := DomainProblems(list); p != nil { out[field] = p } } for field, list := range s.PaginationProblems() { out[field] = append(out[field], list...) } for field, list := range s.PublisherProblems() { out[field] = append(out[field], list...) } return out } // timePattern is a 24-hour HH:MM time. var timePattern = regexp.MustCompile(`^([01][0-9]|2[0-3]):[0-5][0-9]$`) // countryPattern is a two-letter country code. var countryPattern = regexp.MustCompile(`^[A-Za-z]{2}$`) // HTTPSURL reports whether s is an absolute https URL with a host. func HTTPSURL(s string) bool { u, err := url.Parse(strings.TrimSpace(s)) return err == nil && u.Scheme == "https" && u.Host != "" && !strings.ContainsAny(s, " \t\n<>\"") } // oneOfList reports whether v is one of list. func oneOfList(v string, list []string) bool { for _, item := range list { if v == item { return true } } return false } // PublisherProblems lists the refusals of the Publisher and Local // business tabs (UI-SPEC C20, D-45, T-33-25): the publisher type, https // sameAs lines (with the line number), the website, the email, a person // as a local business, the business's address, country, coordinates and // opening hours (local_hours..opens / .closes). func (s Settings) PublisherProblems() map[string][]Problem { out := map[string][]Problem{} add := func(field string, p Problem) { out[field] = append(out[field], p) } if s.PublisherType != "" && !oneOfList(s.PublisherType, []string{PublisherOrganization, PublisherPerson}) { add("publisher_type", Problem{Key: errValueInvalid}) } n := 0 for _, line := range strings.Split(strings.ReplaceAll(s.PublisherSameAs, "\r\n", "\n"), "\n") { n++ if v := strings.TrimSpace(line); v != "" && !HTTPSURL(v) { add("publisher_same_as", Problem{Key: errSameAsInvalid, Params: map[string]string{"line": strconv.Itoa(n)}}) break } } if v := strings.TrimSpace(s.PublisherURL); v != "" && !HTTPSURL(v) { add("publisher_url", Problem{Key: errURLInvalid}) } if v := strings.TrimSpace(s.PublisherEmail); v != "" { local, domain, ok := strings.Cut(v, "@") if !ok || local == "" || !strings.Contains(domain, ".") || strings.ContainsAny(v, " <>\"") { add("publisher_email", Problem{Key: errEmailInvalid}) } } if !s.LocalEnabled { return out } if s.PublisherType == PublisherPerson { add("local_enabled", Problem{Key: errLocalPerson}) } if s.LocalType != "" && !oneOfList(s.LocalType, LocalTypes) { add("local_type", Problem{Key: errValueInvalid}) } if strings.TrimSpace(s.LocalStreet) == "" || strings.TrimSpace(s.LocalPostalCode) == "" || strings.TrimSpace(s.LocalCity) == "" { for field, v := range map[string]string{"local_street": s.LocalStreet, "local_postal_code": s.LocalPostalCode, "local_city": s.LocalCity} { if strings.TrimSpace(v) == "" { add(field, Problem{Key: errLocalAddress}) } } } if v := strings.TrimSpace(s.LocalCountry); v != "" && !countryPattern.MatchString(v) { add("local_country", Problem{Key: errCountryInvalid}) } coordinate := func(field, v string, limit float64) { if v = strings.TrimSpace(v); v == "" { return } f, err := strconv.ParseFloat(strings.ReplaceAll(v, ",", "."), 64) if err != nil || f < -limit || f > limit { add(field, Problem{Key: errCoordinateInvalid}) } } coordinate("local_latitude", s.LocalLatitude, 90) coordinate("local_longitude", s.LocalLongitude, 180) for i, h := range s.Hours() { base := fmt.Sprintf("local_hours.%d.", i) if !oneOfList(h.Day, Days) { add(base+"day", Problem{Key: errValueInvalid}) } okOpens, okCloses := timePattern.MatchString(h.Opens), timePattern.MatchString(h.Closes) if !okOpens { add(base+"opens", Problem{Key: errTimeInvalid}) } if !okCloses { add(base+"closes", Problem{Key: errTimeInvalid}) } if okOpens && okCloses && h.Closes <= h.Opens { add(base+"closes", Problem{Key: errHoursOrder}) } } return out } // domainLabel is one label of a host name: letters (any script, written // in lowercase), digits and inner hyphens, at most 63 bytes. func domainLabel(label string) bool { if label == "" || len(label) > 63 || strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") { return false } for _, r := range label { if r != '-' && !unicode.IsDigit(r) && !(unicode.IsLetter(r) && !unicode.IsUpper(r)) { return false } } return true } // ValidDomain reports whether d is a host name with at least two labels, // without a scheme, port, path or trailing dot. func ValidDomain(d string) bool { if d == "" || len(d) > 253 { return false } labels := strings.Split(d, ".") if len(labels) < 2 { return false } for _, l := range labels { if !domainLabel(l) { return false } } return true } // DomainLines splits a domain list into its trimmed, lowercased, // non-empty lines, in order and each once. func DomainLines(list string) []string { out := []string{} seen := map[string]bool{} for _, line := range strings.Split(strings.ReplaceAll(list, "\r\n", "\n"), "\n") { d := strings.ToLower(strings.TrimSpace(line)) if d == "" || seen[d] { continue } seen[d] = true out = append(out, d) } return out } // NormalizeDomains rewrites a domain list as DomainLines, one per line. func NormalizeDomains(list string) string { return strings.Join(DomainLines(list), "\n") } // DomainProblems refuses the first line of list that is not a domain // (settings.domain_invalid with its line number and text). func DomainProblems(list string) []Problem { n := 0 for _, line := range strings.Split(strings.ReplaceAll(list, "\r\n", "\n"), "\n") { n++ d := strings.ToLower(strings.TrimSpace(line)) if d == "" || ValidDomain(d) { continue } return []Problem{{Key: errDomainInvalid, Params: map[string]string{"line": strconv.Itoa(n), "value": strings.TrimSpace(line)}}} } return nil } func init() { Register(Settings{}) }