diff --git a/admin_navigation.go b/admin_navigation.go index bbbb8c5..25db6fa 100644 --- a/admin_navigation.go +++ b/admin_navigation.go @@ -38,6 +38,13 @@ func (*Plugin) Navigation() []pact.NavigationItem { Permissions: []string{controllers.PermissionManage}, Controller: controllers.RulesControllerID, }, + { + Code: "pages", + Label: "golem15.seo::lang.nav.pages", + Icon: "files", + Permissions: []string{controllers.PermissionManage}, + Controller: controllers.PagesControllerID, + }, { Code: "robots", Label: "golem15.seo::lang.nav.robots", diff --git a/classes/registry.go b/classes/registry.go new file mode 100644 index 0000000..16ddbf1 --- /dev/null +++ b/classes/registry.go @@ -0,0 +1,733 @@ +package classes + +import ( + "context" + "errors" + "net/url" + "sort" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "git.golem15.com/golem15/sm-seo-plugin/models" + "git.golem15.com/golem15/summercms/modules/cabana" + "git.golem15.com/golem15/summercms/modules/lagoon" + "git.golem15.com/golem15/summercms/modules/pact" + "gorm.io/gorm" +) + +// The page registry (D-33): the site's own frontend pages (home, contact, +// offer pages) as a signpost content type of the SEO plugin, so each page +// gets the SEO tab, resolves ok and is listed in the sitemaps. +const ( + // PageTypeKey is the registry's menu item type. + PageTypeKey = "seo-page" + // PagesController is the Pages screen's admin controller. + PagesController = "golem15.seo.pages" + // PagePattern is the registry type's pattern. A registry page stores + // its own address per locale, so the pattern only satisfies signpost. + PagePattern = "/:page" + // PageLabel is the registry type's label (a content type's label names + // its tab in the SEO settings, so it is not the plain "Page" another + // content type may use). + PageLabel = "golem15.seo::lang.pages.type" +) + +// PageModel is the registry page's morph type, the model its SEO entries +// and record events name. +var PageModel = models.Page{}.MorphName() + +// The route report's limits (P33-20). +const ( + MaxReportedRoutes = 2000 + MaxRouteBytes = 2048 +) + +// PageOrigin is the origin of a page created through the management API: +// mcp through MCP, else manual. +func PageOrigin(ctx context.Context) string { + if ViaOf(ctx) == models.ViaMCP { + return models.PageOriginMCP + } + return models.PageOriginManual +} + +// The phrase keys of the registry refusals (UI-SPEC C21). +const ( + errPageURLRequired = "golem15.seo::lang.pages.url_required" + errPageURLInvalid = "golem15.seo::lang.pages.url_invalid" + errPageURLTaken = "golem15.seo::lang.pages.url_taken" + errPageURLOwned = "golem15.seo::lang.pages.url_owned" + errPageParentLoop = "golem15.seo::lang.pages.parent_loop" + errValueInvalid = "golem15.seo::lang.api.value_invalid" +) + +// pageGroups are the sitemap file groups of the page kinds (P33-18). +var pageGroups = map[string]string{ + models.PageKindPage: "pages", + models.PageKindOffer: "offers", + models.PageKindService: "services", + models.PageKindLanding: "landing-pages", +} + +// PageKindLabels are the phrase keys of the page kinds. +var PageKindLabels = map[string]string{ + models.PageKindPage: "golem15.seo::lang.pages.kind_page", + models.PageKindOffer: "golem15.seo::lang.pages.kind_offer", + models.PageKindService: "golem15.seo::lang.pages.kind_service", + models.PageKindLanding: "golem15.seo::lang.pages.kind_landing", +} + +// PageGroup is the sitemap file group of a page kind; an unknown kind is +// listed with the pages. +func PageGroup(kind string) string { + if g, ok := pageGroups[kind]; ok { + return g + } + return pageGroups[models.PageKindPage] +} + +// LoadPage returns registry page id; ErrNoRecord when there is none. +func LoadPage(db *gorm.DB, id uint) (models.Page, error) { + var rows []models.Page + if err := db.Where("id = ?", id).Limit(1).Find(&rows).Error; err != nil { + return models.Page{}, err + } + if len(rows) == 0 { + return models.Page{}, ErrNoRecord + } + return rows[0], nil +} + +// pageByID reads a registry page by its id as text, nil when none. +func pageByID(db *gorm.DB, id string) (*models.Page, error) { + n, err := strconv.ParseUint(strings.TrimSpace(id), 10, 64) + if err != nil || n == 0 { + return nil, nil + } + p, err := LoadPage(db, uint(n)) + if errors.Is(err, ErrNoRecord) { + return nil, nil + } + if err != nil { + return nil, err + } + return &p, nil +} + +// PageAt returns the registry page whose address in any locale is path, +// with that locale; nil when none. +func (s *Service) PageAt(ctx context.Context, db *gorm.DB, path string) (*models.Page, string, error) { + var rows []models.Page + if err := db.Where("url = ?", path).Limit(1).Find(&rows).Error; err != nil { + return nil, "", err + } + if len(rows) > 0 { + return &rows[0], s.DefaultLocale(ctx, db), nil + } + var hit []struct { + ID uint `gorm:"column:id"` + Locale string `gorm:"column:locale"` + } + if err := db.Raw(`SELECT p.id, e.key AS locale FROM golem15_seo_pages p, jsonb_each_text(p.urls) e WHERE e.value = ? ORDER BY p.id LIMIT 1`, path).Scan(&hit).Error; err != nil { + return nil, "", err + } + if len(hit) == 0 { + return nil, "", nil + } + p, err := LoadPage(db, hit[0].ID) + if err != nil { + return nil, "", err + } + if def := s.DefaultLocale(ctx, db); hit[0].Locale == def { + return nil, "", nil + } + return &p, hit[0].Locale, nil +} + +// PageRecord describes a registry page as a signpost content record: its +// address and name per locale (the default locale first, then the other +// enabled locales with an address), its content date as lastmod and its +// parent page. +func (s *Service) PageRecord(ctx context.Context, db *gorm.DB, p models.Page) pact.ContentRecord { + def := s.DefaultLocale(ctx, db) + urls := p.LocaleURLs(def) + rec := pact.ContentRecord{ID: p.IDText(), Published: true, LastMod: p.UpdatedAt} + if p.ContentUpdatedAt != nil && !p.ContentUpdatedAt.IsZero() { + day := p.ContentUpdatedAt.Time(time.UTC) + rec.LastMod = &day + } + for _, locale := range s.pageLocales(ctx, db, urls) { + rec.Locales = append(rec.Locales, pact.ContentLocale{Locale: locale, URL: urls[locale], Title: p.TitleIn(locale)}) + } + if p.ParentID != nil && *p.ParentID != 0 { + rec.Parents = []pact.ContentRef{{Type: PageTypeKey, ID: strconv.FormatUint(uint64(*p.ParentID), 10)}} + } + return rec +} + +// pageLocales orders the locales of urls: the site's locales first (the +// default one leading), then any other stored locale by code. +func (s *Service) pageLocales(ctx context.Context, db *gorm.DB, urls map[string]string) []string { + var out []string + seen := map[string]bool{} + for _, l := range s.Locales(ctx, db) { + if urls[l] != "" && !seen[l] { + out = append(out, l) + seen[l] = true + } + } + var rest []string + for l := range urls { + if !seen[l] { + rest = append(rest, l) + } + } + sort.Strings(rest) + return append(out, rest...) +} + +// PageMenuItemType is the registry's menu item type seo-page: a menu item +// may point at a registry page, and its content descriptor lists, describes +// and matches the pages for signpost's consumers (the SEO tab, the +// resolver, the sitemaps). service is asked on every call. +func PageMenuItemType(service func() *Service) pact.MenuItemType { + svc := func() (*Service, error) { + if service == nil { + return nil, ErrNotBooted + } + s := service() + if s == nil { + return nil, ErrNotBooted + } + return s, nil + } + return pact.MenuItemType{ + Key: PageTypeKey, + Label: PageLabel, + DefaultPattern: PagePattern, + References: func(ctx context.Context) ([]pact.MenuItemReference, error) { + s, err := svc() + if err != nil { + return nil, err + } + return s.pageReferences(ctx) + }, + Resolve: func(ctx context.Context, in pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { + s, err := svc() + if err != nil { + return nil, err + } + return s.resolvePageItem(ctx, in.Reference) + }, + Content: &pact.ContentDescriptor{ + Controller: PagesController, + Model: PageModel, + Kind: pact.ContentKindPage, + Records: func(ctx context.Context, in pact.ContentList) (pact.ContentPage, error) { + s, err := svc() + if err != nil { + return pact.ContentPage{}, err + } + return s.pageRecords(ctx, in) + }, + Describe: func(ctx context.Context, in pact.ContentDescribe) (*pact.ContentRecord, error) { + s, err := svc() + if err != nil { + return nil, err + } + db, err := s.DB(ctx) + if err != nil { + return nil, err + } + p, err := pageByID(db, in.ID) + if err != nil || p == nil { + return nil, err + } + rec := s.PageRecord(ctx, db, *p) + return &rec, nil + }, + Match: func(ctx context.Context, in pact.ContentMatch) (*pact.ContentMatchResult, error) { + s, err := svc() + if err != nil { + return nil, err + } + db, err := s.DB(ctx) + if err != nil { + return nil, err + } + p, locale, err := s.PageAt(ctx, db, in.Path) + if err != nil || p == nil { + return nil, err + } + return &pact.ContentMatchResult{ID: p.IDText(), Locale: locale, Exact: true}, nil + }, + }, + } +} + +// pageRecords is one page of registry records by id. +func (s *Service) pageRecords(ctx context.Context, in pact.ContentList) (pact.ContentPage, error) { + db, err := s.DB(ctx) + if err != nil { + return pact.ContentPage{}, err + } + limit := in.Limit + if limit <= 0 || limit > 1000 { + limit = 1000 + } + after, _ := strconv.ParseUint(strings.TrimSpace(in.After), 10, 64) + var rows []models.Page + if err := db.Where("id > ? AND url <> ''", after).Order("id ASC").Limit(limit).Find(&rows).Error; err != nil { + return pact.ContentPage{}, err + } + out := pact.ContentPage{} + for _, p := range rows { + out.Records = append(out.Records, s.PageRecord(ctx, db, p)) + } + if len(rows) == limit { + out.Next = rows[len(rows)-1].IDText() + } + return out, nil +} + +// pageReferences lists the registry pages a menu item can point at, as a +// tree by parent, each level by address. +func (s *Service) pageReferences(ctx context.Context) ([]pact.MenuItemReference, error) { + db, err := s.DB(ctx) + if err != nil { + return nil, err + } + var rows []models.Page + if err := db.Where("url <> ''").Order("url ASC").Limit(5000).Find(&rows).Error; err != nil { + return nil, err + } + children := map[uint][]models.Page{} + known := map[uint]bool{} + for _, p := range rows { + known[p.ID] = true + } + for _, p := range rows { + parent := p.Parent() + if !known[parent] { + parent = 0 + } + children[parent] = append(children[parent], p) + } + visited := map[uint]bool{} + var build func(parent uint, depth int) []pact.MenuItemReference + build = func(parent uint, depth int) []pact.MenuItemReference { + var out []pact.MenuItemReference + for _, p := range children[parent] { + if visited[p.ID] || depth > 50 { + continue + } + visited[p.ID] = true + label := p.Title + if strings.TrimSpace(label) == "" { + label = p.URL + } + out = append(out, pact.MenuItemReference{Value: p.IDText(), Label: label, Items: build(p.ID, depth+1)}) + } + return out + } + return build(0, 0), nil +} + +// resolvePageItem resolves a menu item pointing at registry page id: its +// default-locale address and name, and one alternate per locale. +func (s *Service) resolvePageItem(ctx context.Context, id string) (*pact.ResolvedMenuItem, error) { + db, err := s.DB(ctx) + if err != nil { + return nil, err + } + p, err := pageByID(db, id) + if err != nil || p == nil { + return nil, err + } + rec := s.PageRecord(ctx, db, *p) + item := &pact.ResolvedMenuItem{Title: p.Title, URL: p.URL, MTime: rec.LastMod} + for _, l := range rec.Locales { + item.Alternates = append(item.Alternates, pact.MenuItemAlternate{Locale: l.Locale, URL: l.URL, Title: l.Title}) + } + return item, nil +} + +// PageWrite is one change of a registry page through the API or MCP: a +// nil field is left as it is. URL and Title map locales to text; a locale +// sent empty removes that locale's address or name, and the "" key stands +// for the default locale. ParentSet with a nil ParentID moves the page to +// the top level. +type PageWrite struct { + URL map[string]string + Title map[string]string + Kind *string + ParentSet bool + ParentID *uint + ContentUpdatedAt *string + Note *string +} + +// ApplyPageTexts merges submitted addresses and names into p: def is the +// default locale, whose values go to the url and title columns. Every +// address is trimmed and normalised with the site's address rules, so it +// is stored in the form the resolver matches. +func (s *Service) ApplyPageTexts(ctx context.Context, db *gorm.DB, p *models.Page, urls, titles map[string]string) error { + def := s.DefaultLocale(ctx, db) + settings, err := LoadSettings(db) + if err != nil { + return err + } + opts := NormalizeOptionsOf(settings) + if urls != nil { + others := map[string]string{} + for l, v := range p.URLs.Get() { + others[l] = v + } + for locale, path := range urls { + if locale == "" { + locale = def + } + path = normalPagePath(path, opts) + if locale == def { + p.URL = path + continue + } + if path == "" { + delete(others, locale) + } else { + others[locale] = path + } + } + p.URLs = lagoon.Jsonable[map[string]string]{Data: others, Valid: true} + } + if titles != nil { + others := map[string]string{} + for l, v := range p.Titles.Get() { + others[l] = v + } + for locale, title := range titles { + title = strings.TrimSpace(title) + if locale == "" { + locale = def + } + if locale == def { + p.Title = title + continue + } + if title == "" { + delete(others, locale) + } else { + others[locale] = title + } + } + p.Titles = lagoon.Jsonable[map[string]string]{Data: others, Valid: true} + } + if !p.URLs.Valid { + p.URLs = lagoon.Jsonable[map[string]string]{Data: map[string]string{}, Valid: true} + } + if !p.Titles.Valid { + p.Titles = lagoon.Jsonable[map[string]string]{Data: map[string]string{}, Valid: true} + } + return nil +} + +// normalPagePath trims an address and, when it is a valid path, applies +// the site's address rules (lowercase, trailing slash) to it. +func normalPagePath(path string, opts NormalizeOptions) string { + path = strings.TrimSpace(path) + if !ValidPagePath(path) { + return path + } + norm, _ := Normalize(&url.URL{Path: path}, opts) + if norm.Path == "" { + return path + } + return norm.Path +} + +// ValidPagePath reports whether path is a registry address: a path +// starting with a single "/", without a domain, query or fragment, of at +// most MaxRouteBytes bytes of valid UTF-8 without control characters. +func ValidPagePath(path string) bool { + if path == "" || len(path) > MaxRouteBytes || !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") { + return false + } + if !utf8.ValidString(path) || strings.ContainsAny(path, "?#\\ ") { + return false + } + for _, r := range path { + if unicode.IsControl(r) { + return false + } + } + u, err := url.Parse(path) + return err == nil && !u.IsAbs() && u.Host == "" +} + +// ApplyPageFields sets the scalar fields of w on p; an invalid kind or +// date is a problem. +func ApplyPageFields(p *models.Page, w PageWrite) map[string][]models.Problem { + problems := map[string][]models.Problem{} + if w.Kind != nil { + p.Kind = strings.TrimSpace(*w.Kind) + } + if w.ParentSet { + p.ParentID = nil + if w.ParentID != nil && *w.ParentID != 0 { + ref := models.PageRef(*w.ParentID) + p.ParentID = &ref + } + } + if w.ContentUpdatedAt != nil { + v := strings.TrimSpace(*w.ContentUpdatedAt) + if v == "" { + p.ContentUpdatedAt = nil + } else if d, err := lagoon.ParseDate(v); err == nil { + p.ContentUpdatedAt = &d + } else { + problems["content_updated_at"] = append(problems["content_updated_at"], models.Problem{Key: errValueInvalid}) + } + } + if w.Note != nil { + p.Note = strings.TrimSpace(*w.Note) + } + return problems +} + +// CheckPage lists the refusals of a registry page before a save (UI-SPEC +// C21), keyed by field (url, kind, parent_id): a missing default-locale +// address, an invalid address, an address another registry page uses, an +// address another content type owns (naming the type and the record), an +// unknown kind, and a parent that is unknown or would make a loop. +func (s *Service) CheckPage(ctx context.Context, db *gorm.DB, p models.Page) (map[string][]models.Problem, error) { + problems := map[string][]models.Problem{} + add := func(field string, pr models.Problem) { + for _, have := range problems[field] { + if have.Key == pr.Key { + return + } + } + problems[field] = append(problems[field], pr) + } + if !models.KnownPageKind(p.Kind) { + add("kind", models.Problem{Key: errValueInvalid}) + } + def := s.DefaultLocale(ctx, db) + if strings.TrimSpace(p.URL) == "" { + add("url", models.Problem{Key: errPageURLRequired}) + } + urls := p.LocaleURLs(def) + var paths []string + for _, locale := range s.pageLocales(ctx, db, urls) { + path := urls[locale] + if !ValidPagePath(path) { + add("url", models.Problem{Key: errPageURLInvalid}) + continue + } + paths = append(paths, path) + } + if len(paths) > 0 { + var taken int64 + q := db.Model(&models.Page{}).Where(`(url IN ? OR EXISTS (SELECT 1 FROM jsonb_each_text(golem15_seo_pages.urls) e WHERE e.value IN ?))`, paths, paths) + if p.ID != 0 { + q = q.Where("id <> ?", p.ID) + } + if err := q.Count(&taken).Error; err != nil { + return nil, err + } + if taken > 0 { + add("url", models.Problem{Key: errPageURLTaken}) + } + for _, path := range paths { + owner, err := s.ownerOutsideRegistry(ctx, path) + if err != nil { + return nil, err + } + if owner != nil { + add("url", *owner) + break + } + } + } + if parent := p.Parent(); parent != 0 { + loop, known, err := parentLoop(db, p.ID, parent) + if err != nil { + return nil, err + } + switch { + case !known: + add("parent_id", models.Problem{Key: errValueInvalid}) + case loop: + add("parent_id", models.Problem{Key: errPageParentLoop}) + } + } + return problems, nil +} + +// parentLoop walks up from parent: loop is true when it reaches id (or +// goes round), known false when parent does not exist. +func parentLoop(db *gorm.DB, id, parent uint) (loop, known bool, err error) { + seen := map[uint]bool{} + cur := parent + for depth := 0; cur != 0; depth++ { + if (id != 0 && cur == id) || seen[cur] || depth > 100 { + return true, true, nil + } + seen[cur] = true + p, err := LoadPage(db, cur) + if errors.Is(err, ErrNoRecord) { + return false, depth > 0, nil + } + if err != nil { + return false, false, err + } + if p.Parent() == 0 { + return false, true, nil + } + cur = p.Parent() + } + return false, true, nil +} + +// ownerOutsideRegistry returns the url_owned refusal when a content type +// other than the registry owns path exactly: the type's label and the +// record's title as params. nil when none does. +func (s *Service) ownerOutsideRegistry(ctx context.Context, path string) (*models.Problem, error) { + reg := s.Registry() + if reg == nil { + return nil, nil + } + unprefixed := s.UnprefixedLocale() + for _, typ := range reg.ContentTypes() { + if typ.Key == PageTypeKey || typ.Content == nil || typ.Content.Match == nil { + continue + } + got, err := typ.Content.Match(ctx, pact.ContentMatch{Path: trimSlash(path), Pattern: reg.Pattern(typ.Key, ""), UnprefixedLocale: unprefixed}) + if err != nil { + return nil, err + } + if got == nil || !got.Exact { + continue + } + title := path + if rec, err := reg.Describe(ctx, typ.Key, got.ID, unprefixed); err == nil && rec != nil { + if l := contentLocale(rec, got.Locale); strings.TrimSpace(l.Title) != "" { + title = l.Title + } + } + label := typ.Label + if strings.TrimSpace(label) == "" { + label = typ.Key + } + return &models.Problem{Key: errPageURLOwned, Params: map[string]string{"type": s.Phrase(label, nil), "title": title}}, nil + } + return nil, nil +} + +// PageValues are a registry page's values for history. +func PageValues(p models.Page) map[string]any { + parent := any(nil) + if p.Parent() != 0 { + parent = p.Parent() + } + date := "" + if p.ContentUpdatedAt != nil { + date = p.ContentUpdatedAt.String() + } + return map[string]any{ + "url": p.URL, "urls": plainValue(p.URLs.Get()), "title": p.Title, "titles": plainValue(p.Titles.Get()), + "kind": p.Kind, "origin": p.Origin, "parent_id": parent, "content_updated_at": date, "note": p.Note, + } +} + +// pageWhat is a registry page's history description. +func (s *Service) pageWhat(p models.Page) string { + return s.Phrase("golem15.seo::lang.history.what_page", map[string]string{"url": p.URL}) +} + +// RecordPage records a registry page's change in history. +func (s *Service) RecordPage(ctx context.Context, tx *gorm.DB, before map[string]any, p models.Page, deleted bool) error { + c := Change{Kind: models.SubjectPage, SubjectID: p.IDText(), What: s.pageWhat(p), Before: before} + if !deleted { + c.After = PageValues(p) + } + return s.Record(ctx, tx, c) +} + +// SavePage checks and writes a registry page inside tx, records it in +// history, and fires the record event after commit, so the SEO tab's +// address tracking and the sitemap follow as they do for an admin save. +func (s *Service) SavePage(ctx context.Context, tx *gorm.DB, p *models.Page, before map[string]any) error { + problems, err := s.CheckPage(ctx, tx, *p) + if err != nil { + return err + } + if len(problems) > 0 { + return &ProblemsError{Problems: problems} + } + now := time.Now().UTC() + created := p.ID == 0 + if created { + p.CreatedAt = &now + } + p.UpdatedAt = &now + if err := tx.Save(p).Error; err != nil { + return err + } + if err := s.RecordPage(ctx, tx, before, *p, false); err != nil { + return err + } + cabana.EmitRecordSaved(ctx, s.app, tx, pact.RecordSaved{Controller: PagesController, Model: PageModel, ID: p.IDText(), Created: created}) + return nil +} + +// DeletePage deletes a registry page inside tx, records it in history and +// fires the record event after commit: the SEO plugin then deletes the +// page's SEO entries and the sitemap drops its addresses. +func (s *Service) DeletePage(ctx context.Context, tx *gorm.DB, p models.Page) error { + if err := tx.Delete(&models.Page{}, p.ID).Error; err != nil { + return err + } + if err := s.RecordPage(ctx, tx, PageValues(p), p, true); err != nil { + return err + } + cabana.EmitRecordDeleted(ctx, s.app, tx, pact.RecordDeleted{Controller: PagesController, Model: PageModel, ID: p.IDText()}) + return nil +} + +// PageMissing reports whether p is a reported page that the newest route +// report (at reported) did not name. +func PageMissing(p models.Page, reported *time.Time) bool { + if p.Origin != models.PageOriginReported || reported == nil { + return false + } + return p.ReportedAt == nil || p.ReportedAt.Before(*reported) +} + +// pageKinds reads the kind of each registry page id (as text). +func pageKinds(db *gorm.DB, ids []string) (map[string]string, error) { + var nums []uint64 + for _, id := range ids { + if n, err := strconv.ParseUint(id, 10, 64); err == nil { + nums = append(nums, n) + } + } + out := map[string]string{} + if len(nums) == 0 { + return out, nil + } + var rows []struct { + ID uint64 `gorm:"column:id"` + Kind string `gorm:"column:kind"` + } + if err := db.Raw(`SELECT id, kind FROM golem15_seo_pages WHERE id IN ?`, nums).Scan(&rows).Error; err != nil { + return nil, err + } + for _, r := range rows { + out[strconv.FormatUint(r.ID, 10)] = r.Kind + } + return out, nil +} diff --git a/classes/sitemap_filter.go b/classes/sitemap_filter.go index 39b311e..b4b671a 100644 --- a/classes/sitemap_filter.go +++ b/classes/sitemap_filter.go @@ -51,19 +51,60 @@ func (f SitemapFilter) Decide(ctx context.Context, batch []contract.Candidate) ( if err != nil { return nil, err } + var pageIDs []string + for _, c := range batch { + if c.TypeKey == PageTypeKey { + pageIDs = append(pageIDs, c.RecordID) + } + } + kinds, err := pageKinds(b.db, pageIDs) + if err != nil { + return nil, err + } for i, c := range batch { d, err := b.decide(ctx, c) if err != nil { return nil, err } + if c.TypeKey == PageTypeKey && !d.Exclude { + d.Group = PageGroup(kinds[c.RecordID]) + } out[i] = d } return out, nil } -// Groups lists the file groups the decisions name. -func (f SitemapFilter) Groups(context.Context) ([]contract.Group, error) { - return nil, nil +// Groups lists the file groups the decisions name: the registry's page +// kinds (pages, offers, services, landing-pages), titled in every locale +// of the site with the kind's name (P33-18). +func (f SitemapFilter) Groups(ctx context.Context) ([]contract.Group, error) { + var svc *Service + if f.Service != nil { + svc = f.Service() + } + if svc == nil { + return nil, nil + } + db, err := svc.DB(ctx) + if err != nil { + return nil, err + } + locales := svc.Locales(ctx, db) + tr := svc.translator() + out := make([]contract.Group, 0, len(models.PageKinds)) + for _, kind := range models.PageKinds { + titles := map[string]string{} + key := PageKindLabels[kind] + for _, locale := range locales { + if tr == nil { + titles[locale] = key + continue + } + titles[locale] = tr.GetIn(locale, key, nil) + } + out = append(out, contract.Group{Key: PageGroup(kind), Titles: titles}) + } + return out, nil } // newFilterBatch reads the settings and the redirects once for a batch. diff --git a/controllers/admin_registry.go b/controllers/admin_registry.go index 718c043..132d921 100644 --- a/controllers/admin_registry.go +++ b/controllers/admin_registry.go @@ -32,6 +32,7 @@ func AdminControllers(app func() *backpack.App) []pact.AdminController { redirectsAdminController{app: app}, notFoundAdminController{app: app}, rulesAdminController{app: app}, + pagesAdminController{app: app}, robotsAdminController{app: app}, historyAdminController{app: app}, transferAdminController{app: app}, diff --git a/controllers/api/registry.go b/controllers/api/registry.go new file mode 100644 index 0000000..38fcb02 --- /dev/null +++ b/controllers/api/registry.go @@ -0,0 +1,298 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "strconv" + "strings" + + "git.golem15.com/golem15/sm-seo-plugin/classes" + "git.golem15.com/golem15/sm-seo-plugin/models" + "gorm.io/gorm" +) + +// ListPagesInput is the query of GET /_seo/api/v1/pages. +type ListPagesInput struct { + Page int `json:"page,omitempty" query:"page" jsonschema:"the page number, from 1"` + PerPage int `json:"per_page,omitempty" query:"per_page" jsonschema:"rows per page, at most 200 (50 by default)"` + Search string `json:"search,omitempty" query:"search" jsonschema:"text the address, another locale's address or the name contains"` + ID uint `json:"id,omitempty" query:"id" jsonschema:"one page by id"` + Kind string `json:"kind,omitempty" query:"kind" jsonschema:"page, offer, service or landing"` + Origin string `json:"origin,omitempty" query:"origin" jsonschema:"manual, reported or mcp"` + Missing string `json:"missing,omitempty" query:"missing" jsonschema:"only: reported pages the site's newest route report did not include; hide: all others"` +} + +// PageIDInput names one registry page. +type PageIDInput struct { + ID uint `json:"id" path:"id" jsonschema:"the registry page id"` +} + +// CreatePageInput is the body of POST /_seo/api/v1/pages. +type CreatePageInput struct { + URL map[string]string `json:"url" jsonschema:"the page's path per locale code, e.g. {\"pl\": \"/kontakt\", \"en\": \"/contact\"}; the default locale is required, a locale left out or empty has no version of the page; a plain string is the default locale's path"` + Title map[string]string `json:"title,omitempty" jsonschema:"the page's name per locale code (default SEO title, HTML sitemap, breadcrumbs); a plain string is the default locale's"` + Kind *string `json:"kind,omitempty" jsonschema:"page (the default), offer, service or landing: decides the sitemap file and the default structured data"` + ParentID *uint `json:"parent_id,omitempty" jsonschema:"the parent registry page's id (breadcrumbs); null or 0 for the top level"` + ContentUpdatedAt *string `json:"content_updated_at,omitempty" jsonschema:"the date the page's content last changed, YYYY-MM-DD (sitemap lastmod); today by default"` + Note *string `json:"note,omitempty" jsonschema:"an internal note"` +} + +// UpdatePageInput is PUT /_seo/api/v1/pages/{id}: only the fields sent are +// changed; in url and title only the locales sent. +type UpdatePageInput struct { + ID uint `json:"id" path:"id" jsonschema:"the registry page id"` + URL map[string]string `json:"url,omitempty" jsonschema:"paths per locale code to change; an empty path removes that locale's version (the default locale's cannot be removed)"` + Title map[string]string `json:"title,omitempty" jsonschema:"names per locale code to change"` + Kind *string `json:"kind,omitempty" jsonschema:"page, offer, service or landing"` + ParentID *uint `json:"parent_id,omitempty" jsonschema:"the parent registry page's id; null or 0 for the top level"` + ContentUpdatedAt *string `json:"content_updated_at,omitempty" jsonschema:"YYYY-MM-DD; empty clears it"` + Note *string `json:"note,omitempty" jsonschema:"an internal note"` +} + +// pageBody is a decoded create or update body; a translatable value is a +// locale map or a plain string (the default locale's). +type pageBody struct { + write classes.PageWrite +} + +func (b *pageBody) UnmarshalJSON(raw []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return err + } + texts := func(name string) (map[string]string, error) { + v, ok := fields[name] + if !ok || bytes.Equal(bytes.TrimSpace(v), []byte("null")) { + return nil, nil + } + var s string + if json.Unmarshal(v, &s) == nil { + return map[string]string{"": s}, nil + } + var m map[string]string + if err := json.Unmarshal(v, &m); err != nil { + return nil, err + } + if m == nil { + m = map[string]string{} + } + return m, nil + } + var err error + if b.write.URL, err = texts("url"); err != nil { + return err + } + if b.write.Title, err = texts("title"); err != nil { + return err + } + str := func(name string) (*string, error) { + v, ok := fields[name] + if !ok || bytes.Equal(bytes.TrimSpace(v), []byte("null")) { + return nil, nil + } + var s string + if err := json.Unmarshal(v, &s); err != nil { + return nil, err + } + return &s, nil + } + if b.write.Kind, err = str("kind"); err != nil { + return err + } + if b.write.ContentUpdatedAt, err = str("content_updated_at"); err != nil { + return err + } + if b.write.Note, err = str("note"); err != nil { + return err + } + if v, ok := fields["parent_id"]; ok { + b.write.ParentSet = true + if !bytes.Equal(bytes.TrimSpace(v), []byte("null")) { + var n uint + if err := json.Unmarshal(v, &n); err != nil { + return err + } + if n != 0 { + b.write.ParentID = &n + } + } + } + return nil +} + +// PageJSON is a registry page in an answer: its default-locale address and +// name, every locale's address and name, and whether the newest route +// report missed it. +func PageJSON(p models.Page, def string, reported *bool) map[string]any { + urls := p.LocaleURLs(def) + titles := map[string]string{def: p.Title} + for l, t := range p.Titles.Get() { + if l != def && t != "" { + titles[l] = t + } + } + var parent any + if p.Parent() != 0 { + parent = p.Parent() + } + var date any + if p.ContentUpdatedAt != nil && !p.ContentUpdatedAt.IsZero() { + date = p.ContentUpdatedAt.String() + } + out := map[string]any{ + "id": p.ID, "url": p.URL, "urls": urls, "title": p.Title, "titles": titles, + "kind": p.Kind, "origin": p.Origin, "parent_id": parent, "content_updated_at": date, + "reported_at": p.ReportedAt, "note": p.Note, "created_at": p.CreatedAt, "updated_at": p.UpdatedAt, + } + if reported != nil { + out["missing"] = *reported + } + return out +} + +// pageAnswer is a page in an answer with its missing state. +func pageAnswer(ctx context.Context, svc *classes.Service, db *gorm.DB, p models.Page) (map[string]any, error) { + settings, err := classes.LoadSettings(db) + if err != nil { + return nil, err + } + missing := classes.PageMissing(p, settings.RoutesReportedAt) + return PageJSON(p, svc.DefaultLocale(ctx, db), &missing), nil +} + +// ListPages serves GET /_seo/api/v1/pages: the registry by address, +// filtered by search text, id, kind, origin and the missing state. +func ListPages(service func() *classes.Service) http.HandlerFunc { + return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { + db, err := read(ctx, svc) + if err != nil { + return 0, nil, err + } + in := listParams(r) + query := r.URL.Query() + q := db.Model(&models.Page{}).Where("url <> ''").Order("url ASC, id ASC") + if in.Search != "" { + p := like(in.Search) + q = q.Where(`lower(url) LIKE ? OR lower(title) LIKE ? OR lower(urls::text) LIKE ? OR lower(titles::text) LIKE ?`, p, p, p, p) + } + if id, err := strconv.ParseUint(query.Get("id"), 10, 64); err == nil && id > 0 { + q = q.Where("id = ?", id) + } + for param, scope := range map[string]string{"kind": models.FilterPageKind, "origin": models.FilterPageOrigin, "missing": models.FilterPageMissing} { + if v := strings.TrimSpace(query.Get(param)); v != "" { + q = models.Page{}.FilterScope(scope, q, v) + } + } + var rows []models.Page + total, err := paginate(q, in, &rows) + if err != nil { + return 0, nil, err + } + settings, err := classes.LoadSettings(db) + if err != nil { + return 0, nil, err + } + def := svc.DefaultLocale(ctx, db) + out := make([]map[string]any, 0, len(rows)) + for _, row := range rows { + missing := classes.PageMissing(row, settings.RoutesReportedAt) + out = append(out, PageJSON(row, def, &missing)) + } + return http.StatusOK, listBody(out, in, total), nil + }) +} + +// savePage applies a body to p and saves it in tx. +func savePage(ctx context.Context, svc *classes.Service, tx *gorm.DB, p *models.Page, w classes.PageWrite, before map[string]any) error { + problems := classes.ApplyPageFields(p, w) + if len(problems) > 0 { + return &classes.ProblemsError{Problems: problems} + } + if err := svc.ApplyPageTexts(ctx, tx, p, w.URL, w.Title); err != nil { + return err + } + return svc.SavePage(ctx, tx, p, before) +} + +// CreatePage serves POST /_seo/api/v1/pages: 201 with the page, 422 +// validation_failed with the C21 refusals (url_required, url_invalid, +// url_taken, url_owned, parent_loop). The origin is mcp through MCP. +func CreatePage(service func() *classes.Service) http.HandlerFunc { + return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { + var body pageBody + if err := decode(r, maxBody, &body); err != nil { + return 0, nil, err + } + row := models.NewPage() + row.Origin = classes.PageOrigin(ctx) + var answer map[string]any + err := write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error { + if err := savePage(ctx, svc, tx, &row, body.write, nil); err != nil { + return err + } + var err error + answer, err = pageAnswer(ctx, svc, tx, row) + return err + }) + if err != nil { + return 0, nil, err + } + return http.StatusCreated, map[string]any{"data": answer}, nil + }) +} + +// UpdatePage serves PUT /_seo/api/v1/pages/{id}: the fields sent change +// and are checked like a create. +func UpdatePage(service func() *classes.Service) http.HandlerFunc { + return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { + id, err := pathID(r, "id") + if err != nil { + return 0, nil, err + } + var body pageBody + if err := decode(r, maxBody, &body); err != nil { + return 0, nil, err + } + var answer map[string]any + err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error { + row, err := classes.LoadPage(tx, id) + if err != nil { + return err + } + if err := savePage(ctx, svc, tx, &row, body.write, classes.PageValues(row)); err != nil { + return err + } + answer, err = pageAnswer(ctx, svc, tx, row) + return err + }) + if err != nil { + return 0, nil, err + } + return http.StatusOK, map[string]any{"data": answer}, nil + }) +} + +// DeletePage serves DELETE /_seo/api/v1/pages/{id}: the page and its SEO +// entries go (history keeps them); a page the site reports comes back with +// its next route report. +func DeletePage(service func() *classes.Service) http.HandlerFunc { + return manage(service, func(ctx context.Context, svc *classes.Service, r *http.Request) (int, any, error) { + id, err := pathID(r, "id") + if err != nil { + return 0, nil, err + } + err = write(ctx, svc, func(ctx context.Context, tx *gorm.DB) error { + row, err := classes.LoadPage(tx, id) + if err != nil { + return err + } + return svc.DeletePage(ctx, tx, row) + }) + if err != nil { + return 0, nil, err + } + return http.StatusOK, map[string]any{"deleted": id}, nil + }) +} diff --git a/controllers/pages.go b/controllers/pages.go new file mode 100644 index 0000000..dd23b9f --- /dev/null +++ b/controllers/pages.go @@ -0,0 +1,287 @@ +package controllers + +import ( + "context" + "time" + + "git.golem15.com/golem15/sm-seo-plugin/classes" + "git.golem15.com/golem15/sm-seo-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/lagoon" + "git.golem15.com/golem15/summercms/modules/pact" + "gorm.io/gorm" +) + +// PagesControllerID is the Pages screen, the page registry (UI-SPEC C21). +const PagesControllerID = classes.PagesController + +// pagesAdminController lists and edits the page registry (D-33). The +// address and the name per locale, their refusals and history come from +// the plugin's own form extension on this controller, which runs inside +// the save transaction once the row is written; the SEO tab is added +// because the registry is the content type seo-page. +type pagesAdminController struct { + app func() *backpack.App +} + +func (pagesAdminController) ID() string { return PagesControllerID } +func (pagesAdminController) ModelName() string { return `Golem15\SEO\Models\Page` } +func (pagesAdminController) ConfigDir() string { return "controllers/pages" } +func (pagesAdminController) RequiredPermissions() []string { + return []string{PermissionManage} +} + +// NewRecord is a page with the form's defaults: kind page, entered by +// hand, its content changed today. +func (pagesAdminController) NewRecord() any { + p := models.NewPage() + return &p +} + +// FormBeforeCreate dates a page without a content date today and refuses +// an unknown kind or a parent that is unknown or makes a loop. +func (c pagesAdminController) FormBeforeCreate(ctx context.Context, model any) error { + p, ok := model.(*models.Page) + if !ok || p == nil { + return nil + } + if p.ContentUpdatedAt == nil || p.ContentUpdatedAt.IsZero() { + today := lagoon.DateOf(time.Now().UTC()) + p.ContentUpdatedAt = &today + } + if p.Origin == "" { + p.Origin = models.PageOriginManual + } + return c.check(ctx, p) +} + +// FormBeforeUpdate checks an update's kind and parent. +func (c pagesAdminController) FormBeforeUpdate(ctx context.Context, model any) error { + p, ok := model.(*models.Page) + if !ok || p == nil { + return nil + } + return c.check(ctx, p) +} + +// check refuses the owner fields' problems (kind, parent_id); the address +// is checked by the form extension, which receives it. +func (c pagesAdminController) check(ctx context.Context, p *models.Page) error { + if p.ParentID != nil && *p.ParentID == 0 { + p.ParentID = nil + } + svc, err := classes.NewService(appOf(c.app)) + if err != nil { + return err + } + db, err := database(ctx, c.app) + if err != nil { + return err + } + problems, err := svc.CheckPage(ctx, db, *p) + if err != nil { + return err + } + owner := map[string][]models.Problem{} + for _, field := range []string{"kind", "parent_id"} { + if list := problems[field]; len(list) > 0 { + owner[field] = list + } + } + return problemsError(ctx, translator(c.app), owner) +} + +// FormAfterDelete records the deleted page in history inside the delete's +// transaction; the record event then deletes its SEO entries. +func (c pagesAdminController) FormAfterDelete(ctx context.Context, model any) error { + p, ok := model.(*models.Page) + if !ok || p == nil { + return nil + } + svc, err := classes.NewService(appOf(c.app)) + if err != nil { + return err + } + db, err := database(ctx, c.app) + if err != nil { + return err + } + return svc.RecordPage(ctx, db, classes.PageValues(*p), *p, true) +} + +// reportedAt is the settings' newest route report time, nil when none. +func reportedAt(db *gorm.DB) (*time.Time, error) { + var rows []models.Settings + if err := db.Select("id", "routes_reported_at").Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil { + return nil, err + } + if len(rows) == 0 { + return nil, nil + } + return rows[0].RoutesReportedAt, nil +} + +// ListRowStates marks a reported page that the newest route report did not +// name (UI-SPEC C21): negative, labelled pages.state_missing. The state is +// read from the table, since ListExtendRecords shows the origin as its +// label. +func (pagesAdminController) ListRowStates(ctx context.Context, db *gorm.DB, records []any) ([][]pact.RowState, error) { + var ids []uint + for _, record := range records { + if p, ok := record.(*models.Page); ok && p != nil { + ids = append(ids, p.ID) + } + } + missing := map[uint]bool{} + if len(ids) > 0 { + var rows []uint + if err := db.WithContext(ctx).Model(&models.Page{}).Where("id IN ?", ids).Where(models.MissingSQL).Pluck("id", &rows).Error; err != nil { + return nil, err + } + for _, id := range rows { + missing[id] = true + } + } + out := make([][]pact.RowState, len(records)) + for i, record := range records { + if p, ok := record.(*models.Page); ok && p != nil && missing[p.ID] { + out[i] = []pact.RowState{pact.RowStateNegative} + } + } + return out, nil +} + +// pageKindLabels and pageOriginLabels are the phrase keys of the kinds and +// origins. +var ( + pageKindLabels = classes.PageKindLabels + pageOriginLabels = map[string]string{ + models.PageOriginManual: "golem15.seo::lang.pages.origin_manual", + models.PageOriginReported: "golem15.seo::lang.pages.origin_reported", + models.PageOriginMCP: "golem15.seo::lang.pages.origin_mcp", + } +) + +// FilterOptions serves the choices of the kind, origin and missing +// filters. +func (pagesAdminController) FilterOptions(scope string) []pact.Option { + switch scope { + case models.FilterPageKind: + out := make([]pact.Option, 0, len(models.PageKinds)) + for _, k := range models.PageKinds { + out = append(out, pact.Option{Value: k, Label: pageKindLabels[k]}) + } + return out + case models.FilterPageOrigin: + out := make([]pact.Option, 0, len(models.PageOrigins)) + for _, o := range models.PageOrigins { + out = append(out, pact.Option{Value: o, Label: pageOriginLabels[o]}) + } + return out + case models.FilterPageMissing: + return []pact.Option{ + {Value: models.MissingOnly, Label: "golem15.seo::lang.pages.filter_missing_only"}, + {Value: models.MissingHide, Label: "golem15.seo::lang.pages.filter_missing_hide"}, + } + } + return nil +} + +// ListExtendRecords shows the kind and the origin of the list's rows as +// their labels in the request locale. The rows are list values only and +// never saved; the columns still sort by the stored values. +func (c pagesAdminController) ListExtendRecords(ctx context.Context, _ *gorm.DB, records []any) error { + tr := translator(c.app) + for _, record := range records { + p, ok := record.(*models.Page) + if !ok || p == nil { + continue + } + if key, ok := pageKindLabels[p.Kind]; ok { + p.Kind = Phrase(ctx, tr, key, nil) + } + if key, ok := pageOriginLabels[p.Origin]; ok { + p.Origin = Phrase(ctx, tr, key, nil) + } + } + return nil +} + +// DropdownOptions serves the parent dropdown: the registry pages by name +// (their address when they have none). +func (c pagesAdminController) DropdownOptions(field string) []pact.Option { + if field != "parent_id" { + return nil + } + db, err := database(context.Background(), c.app) + if err != nil { + return nil + } + var rows []models.Page + if err := db.Where("url <> ''").Order("title ASC, url ASC").Limit(1000).Find(&rows).Error; err != nil { + log.Warn("golem15.seo: parent options", "error", err) + return nil + } + out := make([]pact.Option, 0, len(rows)) + for _, p := range rows { + label := p.Title + if label == "" { + label = p.URL + } else { + label += " (" + p.URL + ")" + } + out = append(out, pact.Option{Value: p.IDText(), Label: label}) + } + return out +} + +// pageStatusView is the view model of P9 page_status: the reported or +// missing line, nothing for a page entered by hand. +type pageStatusView struct { + Text string + Warning bool +} + +// PartialData serves P9 page_status (UI-SPEC C21). +func (c pagesAdminController) PartialData(ctx context.Context, name string, record any) (any, error) { + if name != "page_status" { + return nil, nil + } + p, ok := record.(*models.Page) + if !ok || p == nil || p.Origin != models.PageOriginReported { + return pageStatusView{}, nil + } + db, err := database(ctx, c.app) + if err != nil { + return nil, err + } + reported, err := reportedAt(db) + if err != nil { + return nil, err + } + tr := translator(c.app) + stamp := func(t *time.Time) string { + if t == nil { + return "—" + } + return t.UTC().Format("2006-01-02 15:04") + } + if classes.PageMissing(*p, reported) { + return pageStatusView{Text: Phrase(ctx, tr, "golem15.seo::lang.pages.status_missing", map[string]string{"time": stamp(reported)}), Warning: true}, nil + } + return pageStatusView{Text: Phrase(ctx, tr, "golem15.seo::lang.pages.status_reported", map[string]string{"time": stamp(p.ReportedAt)})}, nil +} + +var ( + _ pact.AdminController = pagesAdminController{} + _ pact.AdminPermissioned = pagesAdminController{} + _ pact.AdminRecordSource = pagesAdminController{} + _ pact.FormBeforeCreate = pagesAdminController{} + _ pact.FormBeforeUpdate = pagesAdminController{} + _ pact.FormAfterDelete = pagesAdminController{} + _ pact.ListRowStates = pagesAdminController{} + _ pact.FilterOptions = pagesAdminController{} + _ pact.ListExtendRecords = pagesAdminController{} + _ pact.DropdownOptionsProvider = pagesAdminController{} + _ pact.AdminPartialData = pagesAdminController{} +) diff --git a/controllers/pages/_page_status.htm b/controllers/pages/_page_status.htm new file mode 100644 index 0000000..08a8680 --- /dev/null +++ b/controllers/pages/_page_status.htm @@ -0,0 +1,7 @@ +{{- with .Data -}} +{{- if .Text -}} +
{{ .Text }}
+