feat: hreflang alternates and the sitemap filter contract
- every listed address of a record with more than one listed locale carries xhtml:link alternates for each locale plus x-default on the default locale (D-55 to D-57); no alternate for a missing translation - each page of records and the additional addresses go to the published contract.Filter in one batch: Exclude leaves the files with its reason, Group moves the address, LastMod seeds a new row (D-95, D-24) - classes.Listing implements contract.Listing and is published in Register
This commit is contained in:
Jakub Zych
154
classes/listing.go
Normal file
154
classes/listing.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/contract"
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/signpost"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Listing implements contract.Listing over the slot table and the
|
||||
// settings (D-95): other plugins read the site address, the index address,
|
||||
// the file groups and each group's listed addresses without importing the
|
||||
// sitemap plugin.
|
||||
type Listing struct {
|
||||
App *backpack.App
|
||||
}
|
||||
|
||||
var _ contract.Listing = Listing{}
|
||||
|
||||
// The page sizes of Entries.
|
||||
const (
|
||||
defaultEntriesLimit = 100
|
||||
maxEntriesLimit = 1000
|
||||
)
|
||||
|
||||
func (l Listing) db(ctx context.Context) (*gorm.DB, error) {
|
||||
if l.App == nil {
|
||||
return nil, ErrNoDatabase
|
||||
}
|
||||
db, ok := l.App.Lookup[*gorm.DB]()
|
||||
if !ok || db == nil {
|
||||
return nil, ErrNoDatabase
|
||||
}
|
||||
return db.WithContext(ctx), nil
|
||||
}
|
||||
|
||||
// Settings answers the site address, the index address, the default
|
||||
// locale and the groups with listed addresses (by key), with their titles:
|
||||
// a content type's saved HTML sitemap headings, else the titles of the
|
||||
// published Filter's group of that key.
|
||||
func (l Listing) Settings(ctx context.Context) (contract.ListingSettings, error) {
|
||||
var out contract.ListingSettings
|
||||
db, err := l.db(ctx)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
settings, err := LoadSettings(db)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.SiteURL = SiteURL(l.App, settings)
|
||||
if out.SiteURL != "" {
|
||||
out.IndexURL = out.SiteURL + "/sitemap.xml"
|
||||
}
|
||||
out.DefaultLocale = models.Text(settings.DefaultLocale)
|
||||
var counts []struct {
|
||||
Grp string `gorm:"column:grp"`
|
||||
N int `gorm:"column:n"`
|
||||
}
|
||||
if err := db.Raw(`SELECT grp, COUNT(*) AS n FROM golem15_sitemap_urls WHERE excluded_reason = '' GROUP BY grp ORDER BY grp`).Scan(&counts).Error; err != nil {
|
||||
return out, err
|
||||
}
|
||||
titles := map[string]map[string]string{}
|
||||
if filter, ok := l.App.Lookup[contract.Filter](); ok && filter != nil {
|
||||
groups, err := filter.Groups(ctx)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, g := range groups {
|
||||
if len(g.Titles) > 0 {
|
||||
titles[g.Key] = g.Titles
|
||||
}
|
||||
}
|
||||
}
|
||||
if reg, ok := l.App.Lookup[*signpost.Registry](); ok && reg != nil {
|
||||
rows, err := LoadTypeSettings(db)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, typ := range reg.ContentTypes() {
|
||||
cfg := EffectiveType(typ, rows)
|
||||
if len(cfg.Titles) > 0 {
|
||||
titles[cfg.Slug] = cfg.Titles
|
||||
}
|
||||
}
|
||||
}
|
||||
out.Groups = make([]contract.GroupInfo, 0, len(counts))
|
||||
for _, c := range counts {
|
||||
out.Groups = append(out.Groups, contract.GroupInfo{Key: c.Grp, Slug: c.Grp, Titles: titles[c.Grp], Count: c.N})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Entries pages a group's listed addresses in the order they were stored
|
||||
// (row id); Next is the last row id of a page that has a successor. A
|
||||
// Locale keeps the addresses of that locale and those without one.
|
||||
func (l Listing) Entries(ctx context.Context, q contract.EntriesQuery) (contract.EntriesPage, error) {
|
||||
page := contract.EntriesPage{Entries: []contract.Entry{}}
|
||||
db, err := l.db(ctx)
|
||||
if err != nil {
|
||||
return page, err
|
||||
}
|
||||
settings, err := LoadSettings(db)
|
||||
if err != nil {
|
||||
return page, err
|
||||
}
|
||||
site := SiteURL(l.App, settings)
|
||||
limit := q.Limit
|
||||
if limit <= 0 {
|
||||
limit = defaultEntriesLimit
|
||||
}
|
||||
limit = min(limit, maxEntriesLimit)
|
||||
scope := func() *gorm.DB {
|
||||
s := db.Model(&models.URL{}).Where("grp = ? AND excluded_reason = ''", q.Group)
|
||||
if loc := strings.TrimSpace(q.Locale); loc != "" {
|
||||
s = s.Where("locale = ? OR locale = ''", loc)
|
||||
}
|
||||
return s
|
||||
}
|
||||
var total int64
|
||||
if err := scope().Count(&total).Error; err != nil {
|
||||
return page, err
|
||||
}
|
||||
page.Total = int(total)
|
||||
q2 := scope()
|
||||
if after, err := strconv.ParseUint(strings.TrimSpace(q.After), 10, 64); err == nil && after > 0 {
|
||||
q2 = q2.Where("id > ?", after)
|
||||
}
|
||||
var rows []models.URL
|
||||
if err := q2.Order("id ASC").Limit(limit + 1).Find(&rows).Error; err != nil {
|
||||
return page, err
|
||||
}
|
||||
if len(rows) > limit {
|
||||
rows = rows[:limit]
|
||||
page.Next = strconv.FormatUint(rows[len(rows)-1].ID, 10)
|
||||
}
|
||||
for _, r := range rows {
|
||||
page.Entries = append(page.Entries, contract.Entry{
|
||||
Group: r.Grp,
|
||||
TypeKey: r.TypeKey,
|
||||
RecordID: r.RecordID,
|
||||
Locale: r.Locale,
|
||||
URL: Absolute(site, r.URL),
|
||||
Title: r.Title,
|
||||
LastMod: r.LastMod,
|
||||
})
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
@@ -130,7 +130,8 @@ func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
|
||||
return res, err
|
||||
}
|
||||
reg, _ := app.Lookup[*signpost.Registry]()
|
||||
found, err := collect(ctx, reg, settings, types)
|
||||
filter, _ := app.Lookup[contract.Filter]()
|
||||
found, err := collect(ctx, reg, filter, settings, types)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
@@ -152,31 +153,43 @@ func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
|
||||
// collect walks the content types page by page and adds the additional
|
||||
// addresses. The records of a type switched off in the settings are kept
|
||||
// in the table with the reason contract.ReasonType, so the settings screen
|
||||
// counts them.
|
||||
func collect(ctx context.Context, reg *signpost.Registry, settings models.Settings, types map[string]models.TypeSetting) (collected, error) {
|
||||
// counts them. Every other page of addresses (and the additional
|
||||
// addresses) is one batch for the published contract.Filter, when there is
|
||||
// one (D-95): an excluded address keeps its row with the Filter's reason, a
|
||||
// Decision.Group moves it to that file group, and Decision.LastMod is the
|
||||
// first lastmod of a new row. The hreflang alternates of a record are its
|
||||
// listed addresses (D-55 to D-57).
|
||||
func collect(ctx context.Context, reg *signpost.Registry, filter contract.Filter, settings models.Settings, types map[string]models.TypeSetting) (collected, error) {
|
||||
out := collected{unpublished: map[slotKey]bool{}}
|
||||
seen := map[slotKey]bool{}
|
||||
add := func(w wanted) {
|
||||
if seen[w.key] {
|
||||
return
|
||||
defaultLocale := models.Text(settings.DefaultLocale)
|
||||
// batch decides one batch and adds its rows; rows of one record are
|
||||
// adjacent, so the alternates are computed per run of a record.
|
||||
batch := func(rows []wanted) error {
|
||||
var fresh []wanted
|
||||
for _, w := range rows {
|
||||
if !seen[w.key] {
|
||||
seen[w.key] = true
|
||||
fresh = append(fresh, w)
|
||||
}
|
||||
}
|
||||
seen[w.key] = true
|
||||
out.rows = append(out.rows, w)
|
||||
if err := decide(ctx, filter, fresh); err != nil {
|
||||
return err
|
||||
}
|
||||
setAlternates(fresh, defaultLocale)
|
||||
out.rows = append(out.rows, fresh...)
|
||||
return nil
|
||||
}
|
||||
unprefixed := models.Text(settings.DefaultLocale)
|
||||
if reg != nil {
|
||||
for _, typ := range reg.ContentTypes() {
|
||||
cfg := EffectiveType(typ, types)
|
||||
reason := ""
|
||||
if !cfg.Include {
|
||||
reason = contract.ReasonType
|
||||
}
|
||||
after := ""
|
||||
for {
|
||||
page, err := reg.Records(ctx, typ.Key, after, recordsPage, unprefixed)
|
||||
page, err := reg.Records(ctx, typ.Key, after, recordsPage, defaultLocale)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("golem15.sitemap: records of %s: %w", typ.Key, err)
|
||||
}
|
||||
var rows []wanted
|
||||
for _, rec := range page.Records {
|
||||
if !rec.Published {
|
||||
out.unpublished[slotKey{typ.Key, rec.ID, ""}] = true
|
||||
@@ -186,16 +199,22 @@ func collect(ctx context.Context, reg *signpost.Registry, settings models.Settin
|
||||
if strings.TrimSpace(l.URL) == "" {
|
||||
continue
|
||||
}
|
||||
add(wanted{
|
||||
w := wanted{
|
||||
key: slotKey{typ.Key, rec.ID, l.Locale},
|
||||
grp: cfg.Slug,
|
||||
url: l.URL,
|
||||
title: l.Title,
|
||||
firstMod: rec.LastMod,
|
||||
reason: reason,
|
||||
})
|
||||
}
|
||||
if !cfg.Include {
|
||||
w.reason = contract.ReasonType
|
||||
}
|
||||
rows = append(rows, w)
|
||||
}
|
||||
}
|
||||
if err := batch(rows); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if page.Next == "" || page.Next == after {
|
||||
break
|
||||
}
|
||||
@@ -203,13 +222,14 @@ func collect(ctx context.Context, reg *signpost.Registry, settings models.Settin
|
||||
}
|
||||
}
|
||||
}
|
||||
var extras []wanted
|
||||
for _, e := range settings.Extra() {
|
||||
u := strings.TrimSpace(e.URL)
|
||||
if !models.AllowedAddress(u) {
|
||||
continue
|
||||
}
|
||||
mod := models.ParseDate(e.LastMod)
|
||||
add(wanted{
|
||||
extras = append(extras, wanted{
|
||||
key: slotKey{models.ExtraTypeKey, ExtraRecordID(u), ""},
|
||||
grp: models.ExtraGroup,
|
||||
url: u,
|
||||
@@ -218,9 +238,101 @@ func collect(ctx context.Context, reg *signpost.Registry, settings models.Settin
|
||||
firstMod: mod,
|
||||
})
|
||||
}
|
||||
if err := batch(extras); err != nil {
|
||||
return out, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// decide asks filter about the rows not already left out (a type switched
|
||||
// off is decided by the settings) and applies its decisions. Without a
|
||||
// Filter every row stays listed.
|
||||
func decide(ctx context.Context, filter contract.Filter, rows []wanted) error {
|
||||
if filter == nil {
|
||||
return nil
|
||||
}
|
||||
var asked []int
|
||||
var candidates []contract.Candidate
|
||||
for i, w := range rows {
|
||||
if w.reason != "" {
|
||||
continue
|
||||
}
|
||||
asked = append(asked, i)
|
||||
candidates = append(candidates, contract.Candidate{TypeKey: w.key.typeKey, RecordID: w.key.recordID, Locale: w.key.locale, URL: w.url})
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return nil
|
||||
}
|
||||
decisions, err := filter.Decide(ctx, candidates)
|
||||
if err != nil {
|
||||
return fmt.Errorf("golem15.sitemap: filter: %w", err)
|
||||
}
|
||||
if len(decisions) != len(candidates) {
|
||||
return fmt.Errorf("golem15.sitemap: filter answered %d decisions for %d candidates", len(decisions), len(candidates))
|
||||
}
|
||||
for j, d := range decisions {
|
||||
w := &rows[asked[j]]
|
||||
if d.Exclude {
|
||||
w.reason = strings.TrimSpace(d.Reason)
|
||||
if w.reason == "" {
|
||||
w.reason = contract.ReasonNoindex
|
||||
}
|
||||
continue
|
||||
}
|
||||
if g := strings.TrimSpace(d.Group); g != "" {
|
||||
if ValidSlug(g) && !ReservedSlugs[g] {
|
||||
w.grp = g
|
||||
} else {
|
||||
pluginLog.Warn("golem15.sitemap: filter group ignored", "group", g, "type", w.key.typeKey, "record", w.key.recordID)
|
||||
}
|
||||
}
|
||||
if d.LastMod != nil {
|
||||
w.firstMod = d.LastMod
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// setAlternates gives the listed addresses of every record with more than
|
||||
// one listed locale its hreflang alternates: each listed locale with its
|
||||
// address, itself included, then x-default on the default locale's address
|
||||
// (the record's first listed locale when the default has none). A locale
|
||||
// without an address or left out has no alternate; a record with one
|
||||
// listed locale gets none (D-55 to D-57, RESEARCH Pitfall 3).
|
||||
func setAlternates(rows []wanted, defaultLocale string) {
|
||||
type record struct{ typeKey, id string }
|
||||
listed := map[record][]int{}
|
||||
var order []record
|
||||
for i, w := range rows {
|
||||
if w.reason != "" || w.extra || w.key.locale == "" || w.key.locale == "x-default" {
|
||||
continue
|
||||
}
|
||||
r := record{w.key.typeKey, w.key.recordID}
|
||||
if _, ok := listed[r]; !ok {
|
||||
order = append(order, r)
|
||||
}
|
||||
listed[r] = append(listed[r], i)
|
||||
}
|
||||
for _, r := range order {
|
||||
idx := listed[r]
|
||||
if len(idx) < 2 {
|
||||
continue
|
||||
}
|
||||
alts := make([]models.Alternate, 0, len(idx)+1)
|
||||
xDefault := rows[idx[0]].url
|
||||
for _, i := range idx {
|
||||
alts = append(alts, models.Alternate{Hreflang: rows[i].key.locale, Href: rows[i].url})
|
||||
if rows[i].key.locale == defaultLocale {
|
||||
xDefault = rows[i].url
|
||||
}
|
||||
}
|
||||
alts = append(alts, models.Alternate{Hreflang: "x-default", Href: xDefault})
|
||||
for _, i := range idx {
|
||||
rows[i].alternates = alts
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// apply writes the found addresses into the slot table and numbers the
|
||||
// files, inside the rebuild's transaction.
|
||||
func apply(tx *gorm.DB, found collected, settings models.Settings, site string) (RebuildResult, error) {
|
||||
|
||||
Reference in New Issue
Block a user