diff --git a/classes/news.go b/classes/news.go new file mode 100644 index 0000000..9ac1084 --- /dev/null +++ b/classes/news.go @@ -0,0 +1,239 @@ +package classes + +import ( + "bufio" + "encoding/xml" + "io" + "strings" + "time" + + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/pact" + "git.golem15.com/golem15/summercms/modules/signpost" + "gorm.io/gorm" +) + +// NSNews is the namespace of the Google News sitemap. +const NSNews = "http://www.google.com/schemas/sitemap-news/0.9" + +// NewsFile is the name of the News sitemap file. +const NewsFile = "sitemap-news.xml" + +// The News sitemap's limits (RESEARCH Pitfall 9): articles first published +// in the last NewsWindow, at most MaxNewsEntries. +const ( + NewsWindow = 48 * time.Hour + MaxNewsEntries = 1000 +) + +// NewsEntry is one of the News sitemap: the article's absolute +// address, the publication name and language, its first publication and +// its title. +type NewsEntry struct { + Loc string + Publication string + Language string + Published time.Time + Title string +} + +// The prefixes of the news_sources values (P33-15). +const ( + NewsSourceType = models.NewsSourceType + NewsSourceCategory = models.NewsSourceCategory +) + +// newsSources are the parsed news_sources values: the article types and +// the parents listed. +type newsSources struct { + types map[string]bool + parents map[models.Ref]bool +} + +// parseNewsSources reads the news_sources values; a value of another form +// is ignored (the settings refuse it on save). +func parseNewsSources(values []string) newsSources { + out := newsSources{types: map[string]bool{}, parents: map[models.Ref]bool{}} + for _, v := range values { + switch { + case strings.HasPrefix(v, NewsSourceType): + if key := strings.TrimPrefix(v, NewsSourceType); key != "" { + out.types[key] = true + } + case strings.HasPrefix(v, NewsSourceCategory): + rest := strings.TrimPrefix(v, NewsSourceCategory) + if i := strings.LastIndex(rest, ":"); i > 0 && i < len(rest)-1 { + out.parents[models.Ref{Type: rest[:i], ID: rest[i+1:]}] = true + } + } + } + return out +} + +// empty reports whether no source is listed: every article type is. +func (s newsSources) empty() bool { return len(s.types) == 0 && len(s.parents) == 0 } + +// matches reports whether an article of typeKey with parents is listed: +// its type or one of its parents is (P33-15). +func (s newsSources) matches(typeKey string, parents []models.Ref) bool { + if s.empty() || s.types[typeKey] { + return true + } + for _, p := range parents { + if s.parents[p] { + return true + } + } + return false +} + +// NewsLanguage is the News language code of a locale: the lowercase +// primary subtag (pl-PL: pl), except Chinese, which Google names zh-cn or +// zh-tw. +func NewsLanguage(locale string) string { + l := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(locale), "_", "-")) + if l == "zh-tw" || l == "zh-hant" || strings.HasPrefix(l, "zh-hant-") { + return "zh-tw" + } + if l == "zh" || strings.HasPrefix(l, "zh-") { + return "zh-cn" + } + if i := strings.IndexByte(l, '-'); i > 0 { + return l[:i] + } + return l +} + +// NewsPublication is the publication name of the News sitemap: the +// news_publication setting, else the config key app.name. +func NewsPublication(app *backpack.App, s models.Settings) string { + if name := strings.TrimSpace(models.Text(s.NewsPublication)); name != "" { + return name + } + if app != nil && app.Config != nil { + return strings.TrimSpace(app.Config.String("app.name")) + } + return "" +} + +// NewsEntries lists the News sitemap's articles (D-31, RESEARCH Pitfall 9): +// listed addresses of content types of kind article first published within +// NewsWindow before now, of the news_sources (every article type when none +// is listed), newest first, at most MaxNewsEntries. An address without a +// locale takes the default_locale setting, else app.locale. +func NewsEntries(db *gorm.DB, app *backpack.App, s models.Settings, now time.Time) ([]NewsEntry, error) { + sources := parseNewsSources(s.Sources()) + q := db.Model(&models.URL{}). + Where("kind = ? AND excluded_reason = '' AND published_at IS NOT NULL AND published_at >= ? AND published_at <= ?", pact.ContentKindArticle, now.Add(-NewsWindow), now). + Order("published_at DESC, id DESC") + if sources.empty() { + q = q.Limit(MaxNewsEntries) + } + var rows []models.URL + if err := q.Find(&rows).Error; err != nil { + return nil, err + } + site := SiteURL(app, s) + publication := NewsPublication(app, s) + fallback := models.Text(s.DefaultLocale) + if fallback == "" && app != nil && app.Config != nil { + fallback = app.Config.String("app.locale") + } + out := make([]NewsEntry, 0, min(len(rows), MaxNewsEntries)) + for _, r := range rows { + if len(out) == MaxNewsEntries { + break + } + if !sources.matches(r.TypeKey, r.ParentRefs.Get()) { + continue + } + loc := Absolute(site, r.URL) + if loc == "" || r.PublishedAt == nil { + continue + } + locale := r.Locale + if locale == "" { + locale = fallback + } + title := strings.TrimSpace(r.Title) + if title == "" { + title = loc + } + out = append(out, NewsEntry{Loc: loc, Publication: publication, Language: NewsLanguage(locale), Published: r.PublishedAt.UTC(), Title: title}) + } + return out, nil +} + +// WriteNewsURLSet writes the News sitemap: a per article with its +// and news:news (news:publication with news:name and news:language, +// news:publication_date and news:title). +func WriteNewsURLSet(w io.Writer, entries []NewsEntry) error { + bw := bufio.NewWriter(w) + bw.WriteString(xml.Header) + bw.WriteString(``) + for _, e := range entries { + bw.WriteString("\n") + escape(bw, e.Loc) + bw.WriteString("") + escape(bw, e.Publication) + bw.WriteString("") + escape(bw, e.Language) + bw.WriteString("") + bw.WriteString(lastmod(e.Published)) + bw.WriteString("") + escape(bw, e.Title) + bw.WriteString("") + } + bw.WriteString("\n\n") + return bw.Flush() +} + +// NewsSource is one value the news_sources setting may take, with what it +// lists: Type is the content type key, ID the category's record ID ("" for +// a whole type) and Title the category's title. +type NewsSource struct { + Value string `json:"value"` + Type string `json:"type"` + ID string `json:"id,omitempty"` + Title string `json:"title,omitempty"` +} + +// maxNewsSourceCategories caps the categories NewsSources lists. +const maxNewsSourceCategories = 500 + +// NewsSources lists the values news_sources may take (P33-15): type: +// for every content type of kind article, then category:: for +// every parent the listed articles name (their categories), with its title +// from the slot table (the default locale's first), at most 500. +func NewsSources(db *gorm.DB, reg *signpost.Registry, defaultLocale string) ([]NewsSource, error) { + out := []NewsSource{} + if reg != nil { + for _, typ := range reg.ContentTypes() { + if typ.Content != nil && typ.Content.Kind == pact.ContentKindArticle { + out = append(out, NewsSource{Value: NewsSourceType + typ.Key, Type: typ.Key}) + } + } + } + var rows []struct { + Type string `gorm:"column:type"` + ID string `gorm:"column:id"` + Title string `gorm:"column:title"` + } + if err := db.Raw(`SELECT DISTINCT ON (p.type, p.id) p.type, p.id, COALESCE(c.title, '') AS title +FROM (SELECT DISTINCT r->>'type' AS type, r->>'id' AS id + FROM golem15_sitemap_urls u, jsonb_array_elements(u.parent_refs) r + WHERE u.kind = ? AND u.excluded_reason = '') p +LEFT JOIN golem15_sitemap_urls c ON c.type_key = p.type AND c.record_id = p.id +ORDER BY p.type, p.id, (c.locale = ?) DESC, c.id +LIMIT ?`, pact.ContentKindArticle, defaultLocale, maxNewsSourceCategories).Scan(&rows).Error; err != nil { + return nil, err + } + for _, r := range rows { + v := NewsSourceCategory + r.Type + ":" + r.ID + if models.ValidNewsSource(v) { + out = append(out, NewsSource{Value: v, Type: r.Type, ID: r.ID, Title: r.Title}) + } + } + return out, nil +} diff --git a/controllers/settings.go b/controllers/settings.go index 6575482..f2b8601 100644 --- a/controllers/settings.go +++ b/controllers/settings.go @@ -12,6 +12,7 @@ import ( "git.golem15.com/golem15/summercms/modules/cabana" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/phrasebook" + "git.golem15.com/golem15/summercms/modules/signpost" "gorm.io/gorm" ) @@ -129,9 +130,76 @@ type reasonRow struct { // and the index note; files.empty before the first build and // files.building while a rebuild asked for from the admin runs. func (c settingsAdminController) PartialData(ctx context.Context, name string, _ any) (any, error) { - if name != "sitemap_files" { - return nil, nil + switch name { + case "sitemap_files": + return c.filesData(ctx) + case "news_sources": + return c.newsSourcesData(ctx) } + return nil, nil +} + +// sourceRow is one row of the news_sources partial: the value to enter and +// what it lists. +type sourceRow struct { + Value string + Label string +} + +// newsSourcesView is the view model of the news_sources partial. +type newsSourcesView struct { + Rows []sourceRow +} + +// newsSourcesData lists the values the news_sources taglist takes +// (classes.NewsSources): each article type, then the categories its +// listed articles name, labelled ": ". +func (c settingsAdminController) newsSourcesData(ctx context.Context) (any, error) { + app := appOf(c.app) + db, err := database(ctx, c.app) + if err != nil { + return nil, err + } + settings, err := classes.LoadSettings(db) + if err != nil { + return nil, err + } + var reg *signpost.Registry + if app != nil { + reg, _ = app.Lookup[*signpost.Registry]() + } + sources, err := classes.NewsSources(db, reg, models.Text(settings.DefaultLocale)) + if err != nil { + return nil, err + } + tr := translator(c.app) + label := func(key string) string { + if reg != nil { + if typ, ok := reg.ContentType(key); ok && typ.Label != "" { + return Phrase(ctx, tr, typ.Label, nil) + } + } + return key + } + view := newsSourcesView{} + for _, src := range sources { + row := sourceRow{Value: src.Value} + if src.ID == "" { + row.Label = Phrase(ctx, tr, "golem15.sitemap::lang.news.source_type", map[string]string{"type": label(src.Type)}) + } else { + title := src.Title + if title == "" { + title = src.ID + } + row.Label = label(src.Type) + ": " + title + } + view.Rows = append(view.Rows, row) + } + return view, nil +} + +// filesData is the view of the S1 files partial. +func (c settingsAdminController) filesData(ctx context.Context) (any, error) { app := appOf(c.app) db, err := database(ctx, c.app) if err != nil { diff --git a/controllers/settings/_news_sources.htm b/controllers/settings/_news_sources.htm new file mode 100644 index 0000000..e1bd021 --- /dev/null +++ b/controllers/settings/_news_sources.htm @@ -0,0 +1,19 @@ +{{- if .Data.Rows -}} +<section> +<h4>{{ trans "golem15.sitemap::lang.news.sources" }}</h4> +<div class="summer-table-wrap"> +<table class="summer-table"> +<thead><tr><th>{{ trans "golem15.sitemap::lang.news.source_value" }}</th><th>{{ trans "golem15.sitemap::lang.news.source_label" }}</th></tr></thead> +<tbody> +{{- range .Data.Rows -}} +<tr><td><code>{{ .Value }}</code></td><td>{{ .Label }}</td></tr> +{{- end -}} +</tbody> +</table> +</div> +</section> +{{- else -}} +<div class="summer-callout" role="status"> +<p class="summer-callout__text">{{ trans "golem15.sitemap::lang.news.none" }}</p> +</div> +{{- end -}} diff --git a/lang/en/lang.yaml b/lang/en/lang.yaml index a048444..0e346ce 100644 --- a/lang/en/lang.yaml +++ b/lang/en/lang.yaml @@ -12,6 +12,8 @@ errors: html_path: Start the address with / and use a path only, e.g. /mapa-strony. page_size: Enter a whole number from 50 to 500. lastmod: 'Enter a date as YYYY-MM-DD, e.g. 2026-10-28, or leave it empty.' + news_publication: Enter at most 255 characters, without line breaks. + news_source: 'Use the values listed under the field: type:<type key> or category:<type key>:<id>.' settings: label: Sitemaps description: XML sitemaps per content type, images, News, IndexNow and the HTML sitemap. @@ -69,6 +71,12 @@ files: index_note: Search engines read the index at :url, and robots.txt points to it. The files are rebuilt after content changes and every night; saving this form rebuilds them now. empty: No files yet. They are built within a few minutes after the first save. building: Rebuilding now. Reload the page in a minute. +news: + sources: Values for Types and categories + source_value: Value + source_label: Articles + source_type: 'Every article of :type' + none: No content type publishes articles yet. indexnow: key: 'Key: :key, served at :url' no_key: A key is generated when you turn IndexNow on and save. diff --git a/lang/pl/lang.yaml b/lang/pl/lang.yaml index a97c42a..2555810 100644 --- a/lang/pl/lang.yaml +++ b/lang/pl/lang.yaml @@ -12,6 +12,8 @@ errors: html_path: Zacznij adres od / i użyj samej ścieżki, np. /mapa-strony. page_size: Wpisz liczbę całkowitą od 50 do 500. lastmod: 'Wpisz datę w formacie RRRR-MM-DD, np. 2026-10-28, albo zostaw puste.' + news_publication: Wpisz najwyżej 255 znaków, bez podziału wiersza. + news_source: 'Użyj wartości z listy pod polem: type:<klucz typu> albo category:<klucz typu>:<id>.' settings: label: Mapy witryny description: Mapy XML według typów treści, obrazy, News, IndexNow i mapa HTML. @@ -69,6 +71,12 @@ files: index_note: Wyszukiwarki czytają indeks pod :url, a robots.txt go wskazuje. Pliki są odbudowywane po zmianach treści i co noc; zapisanie tego formularza odbudowuje je od razu. empty: Nie ma jeszcze plików. Powstaną w ciągu kilku minut po pierwszym zapisie. building: Trwa odbudowa. Odśwież stronę za minutę. +news: + sources: Wartości pola Typy i kategorie + source_value: Wartość + source_label: Artykuły + source_type: 'Wszystkie artykuły: :type' + none: Żaden typ treści nie publikuje jeszcze artykułów. indexnow: key: 'Klucz: :key, udostępniany pod :url' no_key: Klucz zostanie wygenerowany po włączeniu IndexNow i zapisie. diff --git a/models/settings.go b/models/settings.go index 1e013b8..5cfb453 100644 --- a/models/settings.go +++ b/models/settings.go @@ -92,7 +92,7 @@ func (Settings) TableName() string { return "golem15_sitemap_settings" } // Fillable lists the columns the settings form edits. func (Settings) Fillable() []string { - return []string{"site_url", "file_size", "extra_urls"} + return []string{"site_url", "file_size", "extra_urls", "images_enabled", "news_enabled", "news_publication", "news_sources"} } // Rules is empty: the settings controller runs Validate before a save. @@ -103,8 +103,13 @@ const ( errFileSize = "golem15.sitemap::lang.errors.file_size" errURLScheme = "golem15.sitemap::lang.errors.url_scheme" errLastmod = "golem15.sitemap::lang.errors.lastmod" + errNewsName = "golem15.sitemap::lang.errors.news_publication" + errNewsSrc = "golem15.sitemap::lang.errors.news_source" ) +// MaxNewsPublication is the longest publication name. +const MaxNewsPublication = 255 + // Validate answers every problem Problems finds as a validation error on // its field; the settings controller calls it before a save, so the save // answers 422 and the row is left as it was. @@ -140,9 +145,52 @@ func (s Settings) Problems() map[string][]string { out["extra_urls."+strconv.Itoa(i)+".lastmod"] = []string{errLastmod} } } + if name := Text(s.NewsPublication); utf8.RuneCountInString(name) > MaxNewsPublication || strings.ContainsFunc(name, unicode.IsControl) { + out["news_publication"] = []string{errNewsName} + } + for _, v := range s.NewsSources.Get() { + if !ValidNewsSource(strings.TrimSpace(v)) { + out["news_sources"] = []string{errNewsSrc} + break + } + } return out } +// The prefixes of the news_sources values (P33-15): type:<type key> and +// category:<type key>:<id>. +const ( + NewsSourceType = "type:" + NewsSourceCategory = "category:" +) + +// ValidNewsSource reports whether v is a news_sources value: +// type:<type key> or category:<type key>:<id>, each part 1 to 128 letters, +// digits, '-', '_' or '.'. +func ValidNewsSource(v string) bool { + switch { + case strings.HasPrefix(v, NewsSourceType): + return validSourcePart(strings.TrimPrefix(v, NewsSourceType)) + case strings.HasPrefix(v, NewsSourceCategory): + rest := strings.TrimPrefix(v, NewsSourceCategory) + i := strings.LastIndex(rest, ":") + return i > 0 && validSourcePart(rest[:i]) && validSourcePart(rest[i+1:]) + } + return false +} + +func validSourcePart(s string) bool { + if s == "" || len(s) > 128 { + return false + } + for _, r := range s { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.') { + return false + } + } + return true +} + // AllowedFrontendURL reports whether u may be the site address: the empty // string, or an absolute http or https URL with a host and without // whitespace, control characters, a double quote, <, > or &. The value diff --git a/models/settings/fields.yaml b/models/settings/fields.yaml index 4d8c761..1af62f1 100644 --- a/models/settings/fields.yaml +++ b/models/settings/fields.yaml @@ -48,3 +48,54 @@ fields: comment: golem15.sitemap::lang.settings.extra_url_lastmod_comment type: text span: right + + images_enabled: + label: golem15.sitemap::lang.settings.images_enabled + comment: golem15.sitemap::lang.settings.images_enabled_comment + type: switch + default: true + span: full + tab: golem15.sitemap::lang.settings.tab_media + + news_enabled: + label: golem15.sitemap::lang.settings.news_enabled + comment: golem15.sitemap::lang.settings.news_enabled_comment + type: switch + default: false + span: full + tab: golem15.sitemap::lang.settings.tab_media + + news_publication: + label: golem15.sitemap::lang.settings.news_publication + comment: golem15.sitemap::lang.settings.news_publication_comment + type: text + span: left + tab: golem15.sitemap::lang.settings.tab_media + trigger: + action: hide + field: news_enabled + condition: unchecked + + # Taglist options are a fixed list, so the article types and their + # categories are listed by the news_sources partial below instead. + news_sources: + label: golem15.sitemap::lang.settings.news_sources + comment: golem15.sitemap::lang.settings.news_sources_comment + type: taglist + mode: array + span: right + tab: golem15.sitemap::lang.settings.tab_media + trigger: + action: hide + field: news_enabled + condition: unchecked + + _news_sources: + type: partial + path: news_sources + span: full + tab: golem15.sitemap::lang.settings.tab_media + trigger: + action: hide + field: news_enabled + condition: unchecked diff --git a/routes.go b/routes.go index 5a8bae0..e5c047b 100644 --- a/routes.go +++ b/routes.go @@ -35,7 +35,7 @@ const FilesPath = "/_sitemap/files/" func (p *Plugin) Routes(r pact.Router) error { r.Get("/sitemap.xml", p.sitemapIndex, "throttle:"+BucketSitemap) r.Get("/_sitemap/files/{file}", p.sitemapFile, "throttle:"+BucketSitemap) - r.Where("file", `sitemap-[a-z0-9-]+-[0-9]+\.xml`) + r.Where("file", `sitemap-(news|[a-z0-9-]+-[0-9]+)\.xml`) return nil } @@ -111,10 +111,22 @@ GROUP BY image_file_no ORDER BY image_file_no`).Scan(&images).Error; err != nil } rows = append(rows, images...) } - files := make([]classes.IndexFile, 0, len(rows)) + files := make([]classes.IndexFile, 0, len(rows)+1) for _, row := range rows { files = append(files, classes.IndexFile{Loc: site + "/" + classes.FileName(row.Grp, row.FileNo), LastMod: row.LastMod}) } + if settings.NewsEnabled { + news, err := classes.NewsEntries(db, p.app, settings, time.Now().UTC()) + if err != nil { + pluginLog.Error("golem15.sitemap: news index", "error", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + if len(news) > 0 { + newest := news[0].Published + files = append(files, classes.IndexFile{Loc: site + "/" + classes.NewsFile, LastMod: &newest}) + } + } var buf bytes.Buffer if err := classes.WriteIndex(&buf, files); err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) @@ -127,6 +139,10 @@ GROUP BY image_file_no ORDER BY image_file_no`).Scan(&images).Error; err != nil // first seen. An unknown file, or one without a listed address, is a 404. // sitemap-images-<n>.xml is an image sitemap file (imageFile). func (p *Plugin) sitemapFile(w http.ResponseWriter, r *http.Request) { + if r.PathValue("file") == classes.NewsFile { + p.newsFile(w, r) + return + } grp, n, ok := classes.ParseFileName(r.PathValue("file")) if !ok { http.NotFound(w, r) @@ -223,6 +239,39 @@ func (p *Plugin) imageFile(w http.ResponseWriter, r *http.Request, db *gorm.DB, writeXML(w, buf.Bytes(), true) } +// newsFile serves sitemap-news.xml (D-31): the articles of the last two +// days (classes.NewsEntries). It is a 404 while the News sitemap is off. +func (p *Plugin) newsFile(w http.ResponseWriter, r *http.Request) { + db, err := database(p.app) + if err != nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + db = db.WithContext(r.Context()) + settings, err := classes.LoadSettings(db) + if err != nil { + pluginLog.Error("golem15.sitemap: load settings", "error", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + if !settings.NewsEnabled || classes.SiteURL(p.app, settings) == "" { + http.NotFound(w, r) + return + } + entries, err := classes.NewsEntries(db, p.app, settings, time.Now().UTC()) + if err != nil { + pluginLog.Error("golem15.sitemap: news file", "error", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + var buf bytes.Buffer + if err := classes.WriteNewsURLSet(&buf, entries); err != nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + writeXML(w, buf.Bytes(), true) +} + // writeXML answers 200 with an XML body. A file carries X-Robots-Tag // noindex, so search engines read it without indexing it as a page. func writeXML(w http.ResponseWriter, body []byte, noindex bool) { diff --git a/sitemap_extras_test.go b/sitemap_extras_test.go index e05b6f0..a84a391 100644 --- a/sitemap_extras_test.go +++ b/sitemap_extras_test.go @@ -181,3 +181,112 @@ func TestImageSitemap(t *testing.T) { t.Errorf("image file with the image sitemap off: %d", rec.Code) } } + +// newsLocs lists the <loc> values of a News sitemap. +func newsLocs(body string) []string { return locs(body) } + +// TestNewsSitemap pins D-31's News sitemap (RESEARCH Pitfall 9): with +// news_enabled the index lists sitemap-news.xml with the articles first +// published in the last two days (1 and 47 hours ago, not 49), newest +// first, never a record of another kind; each with the publication name +// (news_publication, else app.name), the language of its locale, the +// publication date and the title; news_sources narrows it to the listed +// types and categories; more than 1000 candidates list the newest 1000; +// the settings form refuses a malformed source and lists the values to +// use; news_enabled off takes the file out of the index and answers 404. +func TestNewsSitemap(t *testing.T) { + e := bootJournal(t) + now := time.Now().UTC().Truncate(time.Second) + fresh := post(1, "fresh", "pl", now.Add(-time.Hour)) + fresh.Locales[0].Title = `Grzyby & "huby"` + fresh.Parents = []pact.ContentRef{{Type: categoryType, ID: "3"}} + contentStore.put(postType, fresh) + contentStore.put(postType, post(2, "older", "en", now.Add(-47*time.Hour))) + contentStore.put(postType, post(3, "stale", "en", now.Add(-49*time.Hour))) + cat := now.Add(-time.Hour) + contentStore.put(categoryType, fakeRecord{ID: 3, Published: true, LastMod: cat, PublishedAt: &cat, Locales: []pact.ContentLocale{{Locale: "pl", URL: "/kategoria/lecznicze", Title: "Lecznicze"}}}) + e.rebuild(t) + + // Off by default: not in the index, the file is a 404. + if index := e.get(t, "/sitemap.xml").Body.String(); strings.Contains(index, "sitemap-news.xml") { + t.Fatalf("index with the News sitemap off:\n%s", index) + } + if rec := e.get(t, "/_sitemap/files/sitemap-news.xml"); rec.Code != http.StatusNotFound { + t.Fatalf("News file while off: %d", rec.Code) + } + + // The settings form: the Images and News tab, a malformed source is + // refused, the partial lists the values. + token := e.admin(t, permSitemap) + const base = "/golem15/sitemap/settings" + schema := e.call(t, http.MethodGet, base+"/schema/form", token, nil).Body.String() + for _, want := range []string{`"images_enabled"`, `"news_enabled"`, `"news_publication"`, `"news_sources"`, "Images and News"} { + if !strings.Contains(schema, want) { + t.Errorf("form schema lacks %s", want) + } + } + for _, bad := range []any{[]string{"posts"}, []string{"category:acme-category"}, []string{"type:a b"}} { + if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"news_sources": bad}); rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), `"news_sources"`) { + t.Errorf("news_sources %v: %d %s", bad, rec.Code, rec.Body.String()) + } + } + rec := e.call(t, http.MethodGet, base+"/partials/news_sources", token, nil) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "type:acme-post") || !strings.Contains(rec.Body.String(), "category:acme-category:3") || !strings.Contains(rec.Body.String(), "Categories: Lecznicze") { + t.Fatalf("news_sources partial: %d %s", rec.Code, rec.Body.String()) + } + if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"news_enabled": true, "news_sources": []string{}}); rec.Code != http.StatusOK { + t.Fatalf("enable News: %d %s", rec.Code, rec.Body.String()) + } + + index := e.get(t, "/sitemap.xml").Body.String() + if !strings.Contains(index, "<loc>https://api.example.test/sitemap-news.xml</loc>") { + t.Fatalf("index lacks the News file:\n%s", index) + } + rec = e.get(t, "/_sitemap/files/sitemap-news.xml") + body := rec.Body.String() + if rec.Code != http.StatusOK || rec.Header().Get("X-Robots-Tag") != "noindex" || !strings.Contains(body, `xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"`) { + t.Fatalf("News file: %d %v\n%s", rec.Code, rec.Header(), body) + } + if got := newsLocs(body); strings.Join(got, " ") != "https://api.example.test/blog/fresh https://api.example.test/blog/older" { + t.Fatalf("News articles = %v\n%s", got, body) + } + wantFresh := "<url><loc>https://api.example.test/blog/fresh</loc><news:news><news:publication><news:name>sitemap-smoke</news:name><news:language>pl</news:language></news:publication><news:publication_date>" + + now.Add(-time.Hour).Format(time.RFC3339) + "</news:publication_date><news:title>Grzyby & "huby"</news:title></news:news></url>" + if !strings.Contains(body, wantFresh) { + t.Fatalf("News entry of the fresh post:\n%s\nwant %s", body, wantFresh) + } + + // The publication name setting and the sources. + e.exec(t, `UPDATE golem15_sitemap_settings SET news_publication = 'Gazeta', news_sources = '["category:acme-category:3"]' WHERE id = 1`) + body = e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String() + if got := newsLocs(body); strings.Join(got, " ") != "https://api.example.test/blog/fresh" || !strings.Contains(body, "<news:name>Gazeta</news:name>") { + t.Fatalf("News with a category source = %v\n%s", got, body) + } + e.exec(t, `UPDATE golem15_sitemap_settings SET news_sources = '["type:acme-post"]' WHERE id = 1`) + if got := newsLocs(e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String()); len(got) != 2 { + t.Fatalf("News with a type source = %v", got) + } + e.exec(t, `UPDATE golem15_sitemap_settings SET news_sources = '["type:acme-category"]' WHERE id = 1`) + if got := newsLocs(e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String()); len(got) != 0 { + t.Fatalf("a collection type in the News sitemap = %v", got) + } + + // More than 1000 candidates: the newest 1000. + e.exec(t, `UPDATE golem15_sitemap_settings SET news_sources = '[]' WHERE id = 1`) + for i := 0; i < 1005; i++ { + contentStore.put(postType, post(100+i, fmt.Sprintf("bulk-%d", i), "en", now.Add(-2*time.Hour-time.Duration(i)*time.Minute))) + } + e.rebuild(t) + got := newsLocs(e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String()) + if len(got) != 1000 || got[0] != "https://api.example.test/blog/fresh" || got[999] != "https://api.example.test/blog/bulk-998" { + t.Fatalf("News with 1007 candidates lists %d, first %v last %v", len(got), got[:1], got[len(got)-1:]) + } + + e.exec(t, `UPDATE golem15_sitemap_settings SET news_enabled = FALSE WHERE id = 1`) + if index := e.get(t, "/sitemap.xml").Body.String(); strings.Contains(index, "sitemap-news.xml") { + t.Fatalf("index after turning News off:\n%s", index) + } + if rec := e.get(t, "/_sitemap/files/sitemap-news.xml"); rec.Code != http.StatusNotFound { + t.Fatalf("News file after turning it off: %d", rec.Code) + } +}