feat: Google News sitemap
- sitemap-news.xml lists the articles (content kind article) first published in the last two days, newest first, at most 1000, with the publication name (news_publication, else app.name), the language of the locale, the publication date and the title - news_sources narrows it to type:<key> and category:<key>:<id> values; the Images and News tab lists the values to use in a partial - the index lists the file while news_enabled is on; otherwise it is 404
This commit is contained in:
Jakub Zych
239
classes/news.go
Normal file
239
classes/news.go
Normal file
@@ -0,0 +1,239 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"git.golem15.com/golem15/summercms/modules/signpost"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// NSNews is the namespace of the Google News sitemap.
|
||||
const NSNews = "http://www.google.com/schemas/sitemap-news/0.9"
|
||||
|
||||
// NewsFile is the name of the News sitemap file.
|
||||
const NewsFile = "sitemap-news.xml"
|
||||
|
||||
// The News sitemap's limits (RESEARCH Pitfall 9): articles first published
|
||||
// in the last NewsWindow, at most MaxNewsEntries.
|
||||
const (
|
||||
NewsWindow = 48 * time.Hour
|
||||
MaxNewsEntries = 1000
|
||||
)
|
||||
|
||||
// NewsEntry is one <url> of the News sitemap: the article's absolute
|
||||
// address, the publication name and language, its first publication and
|
||||
// its title.
|
||||
type NewsEntry struct {
|
||||
Loc string
|
||||
Publication string
|
||||
Language string
|
||||
Published time.Time
|
||||
Title string
|
||||
}
|
||||
|
||||
// The prefixes of the news_sources values (P33-15).
|
||||
const (
|
||||
NewsSourceType = models.NewsSourceType
|
||||
NewsSourceCategory = models.NewsSourceCategory
|
||||
)
|
||||
|
||||
// newsSources are the parsed news_sources values: the article types and
|
||||
// the parents listed.
|
||||
type newsSources struct {
|
||||
types map[string]bool
|
||||
parents map[models.Ref]bool
|
||||
}
|
||||
|
||||
// parseNewsSources reads the news_sources values; a value of another form
|
||||
// is ignored (the settings refuse it on save).
|
||||
func parseNewsSources(values []string) newsSources {
|
||||
out := newsSources{types: map[string]bool{}, parents: map[models.Ref]bool{}}
|
||||
for _, v := range values {
|
||||
switch {
|
||||
case strings.HasPrefix(v, NewsSourceType):
|
||||
if key := strings.TrimPrefix(v, NewsSourceType); key != "" {
|
||||
out.types[key] = true
|
||||
}
|
||||
case strings.HasPrefix(v, NewsSourceCategory):
|
||||
rest := strings.TrimPrefix(v, NewsSourceCategory)
|
||||
if i := strings.LastIndex(rest, ":"); i > 0 && i < len(rest)-1 {
|
||||
out.parents[models.Ref{Type: rest[:i], ID: rest[i+1:]}] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// empty reports whether no source is listed: every article type is.
|
||||
func (s newsSources) empty() bool { return len(s.types) == 0 && len(s.parents) == 0 }
|
||||
|
||||
// matches reports whether an article of typeKey with parents is listed:
|
||||
// its type or one of its parents is (P33-15).
|
||||
func (s newsSources) matches(typeKey string, parents []models.Ref) bool {
|
||||
if s.empty() || s.types[typeKey] {
|
||||
return true
|
||||
}
|
||||
for _, p := range parents {
|
||||
if s.parents[p] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// NewsLanguage is the News language code of a locale: the lowercase
|
||||
// primary subtag (pl-PL: pl), except Chinese, which Google names zh-cn or
|
||||
// zh-tw.
|
||||
func NewsLanguage(locale string) string {
|
||||
l := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(locale), "_", "-"))
|
||||
if l == "zh-tw" || l == "zh-hant" || strings.HasPrefix(l, "zh-hant-") {
|
||||
return "zh-tw"
|
||||
}
|
||||
if l == "zh" || strings.HasPrefix(l, "zh-") {
|
||||
return "zh-cn"
|
||||
}
|
||||
if i := strings.IndexByte(l, '-'); i > 0 {
|
||||
return l[:i]
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// NewsPublication is the publication name of the News sitemap: the
|
||||
// news_publication setting, else the config key app.name.
|
||||
func NewsPublication(app *backpack.App, s models.Settings) string {
|
||||
if name := strings.TrimSpace(models.Text(s.NewsPublication)); name != "" {
|
||||
return name
|
||||
}
|
||||
if app != nil && app.Config != nil {
|
||||
return strings.TrimSpace(app.Config.String("app.name"))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// NewsEntries lists the News sitemap's articles (D-31, RESEARCH Pitfall 9):
|
||||
// listed addresses of content types of kind article first published within
|
||||
// NewsWindow before now, of the news_sources (every article type when none
|
||||
// is listed), newest first, at most MaxNewsEntries. An address without a
|
||||
// locale takes the default_locale setting, else app.locale.
|
||||
func NewsEntries(db *gorm.DB, app *backpack.App, s models.Settings, now time.Time) ([]NewsEntry, error) {
|
||||
sources := parseNewsSources(s.Sources())
|
||||
q := db.Model(&models.URL{}).
|
||||
Where("kind = ? AND excluded_reason = '' AND published_at IS NOT NULL AND published_at >= ? AND published_at <= ?", pact.ContentKindArticle, now.Add(-NewsWindow), now).
|
||||
Order("published_at DESC, id DESC")
|
||||
if sources.empty() {
|
||||
q = q.Limit(MaxNewsEntries)
|
||||
}
|
||||
var rows []models.URL
|
||||
if err := q.Find(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
site := SiteURL(app, s)
|
||||
publication := NewsPublication(app, s)
|
||||
fallback := models.Text(s.DefaultLocale)
|
||||
if fallback == "" && app != nil && app.Config != nil {
|
||||
fallback = app.Config.String("app.locale")
|
||||
}
|
||||
out := make([]NewsEntry, 0, min(len(rows), MaxNewsEntries))
|
||||
for _, r := range rows {
|
||||
if len(out) == MaxNewsEntries {
|
||||
break
|
||||
}
|
||||
if !sources.matches(r.TypeKey, r.ParentRefs.Get()) {
|
||||
continue
|
||||
}
|
||||
loc := Absolute(site, r.URL)
|
||||
if loc == "" || r.PublishedAt == nil {
|
||||
continue
|
||||
}
|
||||
locale := r.Locale
|
||||
if locale == "" {
|
||||
locale = fallback
|
||||
}
|
||||
title := strings.TrimSpace(r.Title)
|
||||
if title == "" {
|
||||
title = loc
|
||||
}
|
||||
out = append(out, NewsEntry{Loc: loc, Publication: publication, Language: NewsLanguage(locale), Published: r.PublishedAt.UTC(), Title: title})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// WriteNewsURLSet writes the News sitemap: a <url> per article with its
|
||||
// <loc> and news:news (news:publication with news:name and news:language,
|
||||
// news:publication_date and news:title).
|
||||
func WriteNewsURLSet(w io.Writer, entries []NewsEntry) error {
|
||||
bw := bufio.NewWriter(w)
|
||||
bw.WriteString(xml.Header)
|
||||
bw.WriteString(`<urlset xmlns="` + NSSitemap + `" xmlns:news="` + NSNews + `">`)
|
||||
for _, e := range entries {
|
||||
bw.WriteString("\n<url><loc>")
|
||||
escape(bw, e.Loc)
|
||||
bw.WriteString("</loc><news:news><news:publication><news:name>")
|
||||
escape(bw, e.Publication)
|
||||
bw.WriteString("</news:name><news:language>")
|
||||
escape(bw, e.Language)
|
||||
bw.WriteString("</news:language></news:publication><news:publication_date>")
|
||||
bw.WriteString(lastmod(e.Published))
|
||||
bw.WriteString("</news:publication_date><news:title>")
|
||||
escape(bw, e.Title)
|
||||
bw.WriteString("</news:title></news:news></url>")
|
||||
}
|
||||
bw.WriteString("\n</urlset>\n")
|
||||
return bw.Flush()
|
||||
}
|
||||
|
||||
// NewsSource is one value the news_sources setting may take, with what it
|
||||
// lists: Type is the content type key, ID the category's record ID ("" for
|
||||
// a whole type) and Title the category's title.
|
||||
type NewsSource struct {
|
||||
Value string `json:"value"`
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
}
|
||||
|
||||
// maxNewsSourceCategories caps the categories NewsSources lists.
|
||||
const maxNewsSourceCategories = 500
|
||||
|
||||
// NewsSources lists the values news_sources may take (P33-15): type:<key>
|
||||
// for every content type of kind article, then category:<key>:<id> for
|
||||
// every parent the listed articles name (their categories), with its title
|
||||
// from the slot table (the default locale's first), at most 500.
|
||||
func NewsSources(db *gorm.DB, reg *signpost.Registry, defaultLocale string) ([]NewsSource, error) {
|
||||
out := []NewsSource{}
|
||||
if reg != nil {
|
||||
for _, typ := range reg.ContentTypes() {
|
||||
if typ.Content != nil && typ.Content.Kind == pact.ContentKindArticle {
|
||||
out = append(out, NewsSource{Value: NewsSourceType + typ.Key, Type: typ.Key})
|
||||
}
|
||||
}
|
||||
}
|
||||
var rows []struct {
|
||||
Type string `gorm:"column:type"`
|
||||
ID string `gorm:"column:id"`
|
||||
Title string `gorm:"column:title"`
|
||||
}
|
||||
if err := db.Raw(`SELECT DISTINCT ON (p.type, p.id) p.type, p.id, COALESCE(c.title, '') AS title
|
||||
FROM (SELECT DISTINCT r->>'type' AS type, r->>'id' AS id
|
||||
FROM golem15_sitemap_urls u, jsonb_array_elements(u.parent_refs) r
|
||||
WHERE u.kind = ? AND u.excluded_reason = '') p
|
||||
LEFT JOIN golem15_sitemap_urls c ON c.type_key = p.type AND c.record_id = p.id
|
||||
ORDER BY p.type, p.id, (c.locale = ?) DESC, c.id
|
||||
LIMIT ?`, pact.ContentKindArticle, defaultLocale, maxNewsSourceCategories).Scan(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, r := range rows {
|
||||
v := NewsSourceCategory + r.Type + ":" + r.ID
|
||||
if models.ValidNewsSource(v) {
|
||||
out = append(out, NewsSource{Value: v, Type: r.Type, ID: r.ID, Title: r.Title})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user