package classes import ( "bufio" "encoding/xml" "io" "strings" "time" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/signpost" "gorm.io/gorm" ) // NSNews is the namespace of the Google News sitemap. const NSNews = "http://www.google.com/schemas/sitemap-news/0.9" // NewsFile is the name of the News sitemap file. const NewsFile = "sitemap-news.xml" // The News sitemap's limits (RESEARCH Pitfall 9): articles first published // in the last NewsWindow, at most MaxNewsEntries. const ( NewsWindow = 48 * time.Hour MaxNewsEntries = 1000 ) // NewsEntry is one of the News sitemap: the article's absolute // address, the publication name and language, its first publication and // its title. type NewsEntry struct { Loc string Publication string Language string Published time.Time Title string } // The prefixes of the news_sources values (P33-15). const ( NewsSourceType = models.NewsSourceType NewsSourceCategory = models.NewsSourceCategory ) // newsSources are the parsed news_sources values: the article types and // the parents listed. type newsSources struct { types map[string]bool parents map[models.Ref]bool } // parseNewsSources reads the news_sources values; a value of another form // is ignored (the settings refuse it on save). func parseNewsSources(values []string) newsSources { out := newsSources{types: map[string]bool{}, parents: map[models.Ref]bool{}} for _, v := range values { switch { case strings.HasPrefix(v, NewsSourceType): if key := strings.TrimPrefix(v, NewsSourceType); key != "" { out.types[key] = true } case strings.HasPrefix(v, NewsSourceCategory): rest := strings.TrimPrefix(v, NewsSourceCategory) if i := strings.LastIndex(rest, ":"); i > 0 && i < len(rest)-1 { out.parents[models.Ref{Type: rest[:i], ID: rest[i+1:]}] = true } } } return out } // empty reports whether no source is listed: every article type is. func (s newsSources) empty() bool { return len(s.types) == 0 && len(s.parents) == 0 } // matches reports whether an article of typeKey with parents is listed: // its type or one of its parents is (P33-15). func (s newsSources) matches(typeKey string, parents []models.Ref) bool { if s.empty() || s.types[typeKey] { return true } for _, p := range parents { if s.parents[p] { return true } } return false } // NewsLanguage is the News language code of a locale: the lowercase // primary subtag (pl-PL: pl), except Chinese, which Google names zh-cn or // zh-tw. func NewsLanguage(locale string) string { l := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(locale), "_", "-")) if l == "zh-tw" || l == "zh-hant" || strings.HasPrefix(l, "zh-hant-") { return "zh-tw" } if l == "zh" || strings.HasPrefix(l, "zh-") { return "zh-cn" } if i := strings.IndexByte(l, '-'); i > 0 { return l[:i] } return l } // NewsPublication is the publication name of the News sitemap: the // news_publication setting, else the config key app.name. func NewsPublication(app *backpack.App, s models.Settings) string { if name := strings.TrimSpace(models.Text(s.NewsPublication)); name != "" { return name } if app != nil && app.Config != nil { return strings.TrimSpace(app.Config.String("app.name")) } return "" } // NewsEntries lists the News sitemap's articles (D-31, RESEARCH Pitfall 9): // listed addresses of content types of kind article first published within // NewsWindow before now, of the news_sources (every article type when none // is listed), newest first, at most MaxNewsEntries. An address without a // locale takes the default_locale setting, else app.locale. func NewsEntries(db *gorm.DB, app *backpack.App, s models.Settings, now time.Time) ([]NewsEntry, error) { sources := parseNewsSources(s.Sources()) q := db.Model(&models.URL{}). Where("kind = ? AND excluded_reason = '' AND published_at IS NOT NULL AND published_at >= ? AND published_at <= ?", pact.ContentKindArticle, now.Add(-NewsWindow), now). Order("published_at DESC, id DESC") if sources.empty() { q = q.Limit(MaxNewsEntries) } var rows []models.URL if err := q.Find(&rows).Error; err != nil { return nil, err } site := SiteURL(app, s) publication := NewsPublication(app, s) fallback := models.Text(s.DefaultLocale) if fallback == "" && app != nil && app.Config != nil { fallback = app.Config.String("app.locale") } out := make([]NewsEntry, 0, min(len(rows), MaxNewsEntries)) for _, r := range rows { if len(out) == MaxNewsEntries { break } if !sources.matches(r.TypeKey, r.ParentRefs.Get()) { continue } loc := Absolute(site, r.URL) if loc == "" || r.PublishedAt == nil { continue } locale := r.Locale if locale == "" { locale = fallback } title := strings.TrimSpace(r.Title) if title == "" { title = loc } out = append(out, NewsEntry{Loc: loc, Publication: publication, Language: NewsLanguage(locale), Published: r.PublishedAt.UTC(), Title: title}) } return out, nil } // WriteNewsURLSet writes the News sitemap: a per article with its // and news:news (news:publication with news:name and news:language, // news:publication_date and news:title). func WriteNewsURLSet(w io.Writer, entries []NewsEntry) error { bw := bufio.NewWriter(w) bw.WriteString(xml.Header) bw.WriteString(``) for _, e := range entries { bw.WriteString("\n") escape(bw, e.Loc) bw.WriteString("") escape(bw, e.Publication) bw.WriteString("") escape(bw, e.Language) bw.WriteString("") bw.WriteString(lastmod(e.Published)) bw.WriteString("") escape(bw, e.Title) bw.WriteString("") } bw.WriteString("\n\n") return bw.Flush() } // NewsSource is one value the news_sources setting may take, with what it // lists: Type is the content type key, ID the category's record ID ("" for // a whole type) and Title the category's title. type NewsSource struct { Value string `json:"value"` Type string `json:"type"` ID string `json:"id,omitempty"` Title string `json:"title,omitempty"` } // maxNewsSourceCategories caps the categories NewsSources lists. const maxNewsSourceCategories = 500 // NewsSources lists the values news_sources may take (P33-15): type: // for every content type of kind article, then category:: for // every parent the listed articles name (their categories), with its title // from the slot table (the default locale's first), at most 500. func NewsSources(db *gorm.DB, reg *signpost.Registry, defaultLocale string) ([]NewsSource, error) { out := []NewsSource{} if reg != nil { for _, typ := range reg.ContentTypes() { if typ.Content != nil && typ.Content.Kind == pact.ContentKindArticle { out = append(out, NewsSource{Value: NewsSourceType + typ.Key, Type: typ.Key}) } } } var rows []struct { Type string `gorm:"column:type"` ID string `gorm:"column:id"` Title string `gorm:"column:title"` } if err := db.Raw(`SELECT DISTINCT ON (p.type, p.id) p.type, p.id, COALESCE(c.title, '') AS title FROM (SELECT DISTINCT r->>'type' AS type, r->>'id' AS id FROM golem15_sitemap_urls u, jsonb_array_elements(u.parent_refs) r WHERE u.kind = ? AND u.excluded_reason = '') p LEFT JOIN golem15_sitemap_urls c ON c.type_key = p.type AND c.record_id = p.id ORDER BY p.type, p.id, (c.locale = ?) DESC, c.id LIMIT ?`, pact.ContentKindArticle, defaultLocale, maxNewsSourceCategories).Scan(&rows).Error; err != nil { return nil, err } for _, r := range rows { v := NewsSourceCategory + r.Type + ":" + r.ID if models.ValidNewsSource(v) { out = append(out, NewsSource{Value: v, Type: r.Type, ID: r.ID, Title: r.Title}) } } return out, nil }