package classes import ( "context" "html" "strconv" "strings" "time" "unicode" "git.golem15.com/golem15/sm-journal-plugin/models" translatemodels "git.golem15.com/golem15/sm-translate-plugin/models" "git.golem15.com/golem15/summercms/modules/cabana" "git.golem15.com/golem15/summercms/modules/lagoon/attach" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/signpost" "gorm.io/gorm" ) // The admin controller IDs whose forms edit the described records. They // repeat the controllers package constants, which this package cannot // import (controllers imports classes). const ( postsControllerID = "golem15.journal.posts" categoriesControllerID = "golem15.journal.categories" tagsControllerID = "golem15.journal.tags" ) // contentDB is the database a content descriptor reads: the write // transaction in ctx when there is one (cabana.TxFromContext), so code // running inside a save sees what the save wrote, else the application's. func (m menuTypes) contentDB(ctx context.Context) (*gorm.DB, error) { if ctx == nil { ctx = context.Background() } if tx, ok := cabana.TxFromContext(ctx); ok { return tx.Session(&gorm.Session{NewDB: true, Context: ctx}), nil } return m.db(ctx) } // postContent is the D-93 descriptor of journal-post: posts are edited by // the posts controller and carry the Post morph name. func (m menuTypes) postContent() *pact.ContentDescriptor { return &pact.ContentDescriptor{ Controller: postsControllerID, Model: models.Post{}.MorphName(), Records: m.listPosts, Describe: m.describePost, Match: m.matchPost, } } // describePost describes one post, drafts included: Published tells // whether visitors see it. func (m menuTypes) describePost(ctx context.Context, in pact.ContentDescribe) (*pact.ContentRecord, error) { id, ok := menuReferenceID(in.ID) if !ok { return nil, nil } db, err := m.contentDB(ctx) if err != nil { return nil, err } var posts []models.Post if err := db.Where("id = ?", id).Limit(1).Find(&posts).Error; err != nil { return nil, err } if len(posts) == 0 { return nil, nil } records, err := describePosts(ctx, db, posts, in.Pattern, in.UnprefixedLocale) if err != nil { return nil, err } return &records[0], nil } // describePosts describes posts in the given order. Each locale's URL is // the alternate resolvePost builds; Title is the translated title, // Description the excerpt as plain text, Image the first featured image and // Words the body's word count, each translated with the default value as // fallback. func describePosts(ctx context.Context, db *gorm.DB, posts []models.Post, pattern, unprefixed string) ([]pact.ContentRecord, error) { in := pact.MenuItemResolve{Pattern: pattern, UnprefixedLocale: unprefixed} items, err := resolvePostItems(ctx, db, posts, in) if err != nil { return nil, err } locales, err := loadMenuLocales(ctx, db, unprefixed) if err != nil { return nil, err } ids := make([]uint, 0, len(posts)) for _, p := range posts { ids = append(ids, p.ID) } tr, err := loadMenuTranslations(ctx, db, models.Post{}.MorphName(), ids, locales) if err != nil { return nil, err } images, err := firstImages(ctx, db, models.Post{}.MorphName(), "featured_images", ids) if err != nil { return nil, err } out := make([]pact.ContentRecord, 0, len(posts)) for i := range posts { p := &posts[i] rec := pact.ContentRecord{ ID: strconv.FormatUint(uint64(p.ID), 10), Published: postPublished(p), LastMod: copyTime(p.UpdatedAt), } for _, alt := range items[i].Alternates { locale := alt.Locale if locale == xDefaultLocale { locale = "" } body := tr.value(locale, p.ID, "content_html", p.ContentHTML) if strings.TrimSpace(body) == "" { body = tr.value(locale, p.ID, "content", p.Content) } words := wordCount(plainText(body)) rec.Locales = append(rec.Locales, pact.ContentLocale{ Locale: alt.Locale, URL: alt.URL, Title: alt.Title, Description: strings.TrimSpace(plainText(tr.value(locale, p.ID, "excerpt", p.Excerpt))), Image: images[p.ID], Words: &words, }) } out = append(out, rec) } return out, nil } // postPublished is publishedPosts for one loaded post. func postPublished(p *models.Post) bool { return p.Published && p.PublishedAt != nil && p.PublishedAt.Before(time.Now()) } // firstImages maps each record id to the public URL of its first file in // field (sort order, then id). func firstImages(ctx context.Context, db *gorm.DB, morph, field string, ids []uint) (map[uint]string, error) { out := map[uint]string{} if len(ids) == 0 { return out, nil } keys := make([]string, 0, len(ids)) for _, id := range ids { keys = append(keys, strconv.FormatUint(uint64(id), 10)) } var files []attach.File err := db.WithContext(ctx). Where("attachment_type = ? AND field = ? AND attachment_id IN ?", morph, field, keys). Order("sort_order ASC, id ASC"). Find(&files).Error if err != nil { return nil, err } for i := range files { id, err := strconv.ParseUint(files[i].AttachmentID, 10, 32) if err != nil { continue } if _, seen := out[uint(id)]; !seen { out[uint(id)] = files[i].URL() } } return out, nil } func copyTime(t *time.Time) *time.Time { if t == nil { return nil } v := *t return &v } // plainText drops HTML tags and unescapes entities, with whitespace runs // collapsed to one space. func plainText(s string) string { var b strings.Builder inTag := false for _, r := range s { switch { case r == '<': inTag = true b.WriteRune(' ') case r == '>': inTag = false case !inTag: b.WriteRune(r) } } return strings.Join(strings.Fields(html.UnescapeString(b.String())), " ") } // wordCount counts the whitespace-separated words of s. func wordCount(s string) int { n := 0 in := false for _, r := range s { if unicode.IsSpace(r) { in = false continue } if !in { n++ in = true } } return n } // TagPagesKey is the host configuration key that says whether the site has // tag pages (default true). With false, journal-tag items resolve to // nothing, tag descriptions carry empty URLs and tag paths never match. const TagPagesKey = "golem15.journal.tag_pages" // maxContentPage caps the page size of a descriptor's Records; a limit of 0 // or below asks for the cap. const maxContentPage = 500 func pageLimit(n int) int { if n <= 0 || n > maxContentPage { return maxContentPage } return n } // afterID parses a Records After cursor: "" or an invalid value is the // start. func afterID(after string) uint { id, _ := menuReferenceID(after) return id } // tagPages reports TagPagesKey, true when unset. func (m menuTypes) tagPages() bool { if m.app == nil { return true } app := m.app() if app == nil || app.Config == nil || !app.Config.Has(TagPagesKey) { return true } return app.Config.Bool(TagPagesKey) } // listPosts is the Records of journal-post: every post, drafts included, in // id order. func (m menuTypes) listPosts(ctx context.Context, in pact.ContentList) (pact.ContentPage, error) { db, err := m.contentDB(ctx) if err != nil { return pact.ContentPage{}, err } limit := pageLimit(in.Limit) var posts []models.Post if err := db.Where("id > ?", afterID(in.After)).Order("id ASC").Limit(limit + 1).Find(&posts).Error; err != nil { return pact.ContentPage{}, err } more := len(posts) > limit if more { posts = posts[:limit] } records, err := describePosts(ctx, db, posts, in.Pattern, in.UnprefixedLocale) if err != nil { return pact.ContentPage{}, err } return contentPage(records, more), nil } func contentPage(records []pact.ContentRecord, more bool) pact.ContentPage { page := pact.ContentPage{Records: records} if page.Records == nil { page.Records = []pact.ContentRecord{} } if more && len(records) > 0 { page.Next = records[len(records)-1].ID } return page } // matchPost is the Match of journal-post. The post is found by its slug in // the path's locale, else its default slug, else a slug of another locale, // else by :id; drafts never match. The match is exact only when the path is // the post's own URL in that locale, so every segment (category, date, id, // locale prefix, slug) must fit the post. func (m menuTypes) matchPost(ctx context.Context, in pact.ContentMatch) (*pact.ContentMatchResult, error) { db, err := m.contentDB(ctx) if err != nil { return nil, err } morph := models.Post{}.MorphName() find := func(values map[string]string, locale string, locales menuLocales) (uint, error) { return findMatchRecord(ctx, db, values, locale, locales, morph, func(q *gorm.DB) *gorm.DB { return publishedPosts(q.Model(&models.Post{})) }) } urls := func(id uint) ([]pact.MenuItemAlternate, error) { var posts []models.Post if err := db.Where("id = ?", id).Limit(1).Find(&posts).Error; err != nil || len(posts) == 0 { return nil, err } items, err := resolvePostItems(ctx, db, posts, pact.MenuItemResolve{Pattern: in.Pattern, UnprefixedLocale: in.UnprefixedLocale}) if err != nil { return nil, err } return items[0].Alternates, nil } return matchContent(ctx, db, in, find, urls) } // categoryContent is the D-93 descriptor of journal-category. func (m menuTypes) categoryContent() *pact.ContentDescriptor { return &pact.ContentDescriptor{ Controller: categoriesControllerID, Model: models.Category{}.MorphName(), Records: m.listCategories, Describe: m.describeCategory, Match: m.matchCategory, } } func (m menuTypes) describeCategory(ctx context.Context, in pact.ContentDescribe) (*pact.ContentRecord, error) { id, ok := menuReferenceID(in.ID) if !ok { return nil, nil } db, err := m.contentDB(ctx) if err != nil { return nil, err } var cats []models.Category if err := db.Where("id = ?", id).Limit(1).Find(&cats).Error; err != nil { return nil, err } if len(cats) == 0 { return nil, nil } records, err := describeCategories(ctx, db, cats, in.Pattern, in.UnprefixedLocale) if err != nil { return nil, err } return &records[0], nil } func (m menuTypes) listCategories(ctx context.Context, in pact.ContentList) (pact.ContentPage, error) { db, err := m.contentDB(ctx) if err != nil { return pact.ContentPage{}, err } limit := pageLimit(in.Limit) var cats []models.Category if err := db.Where("id > ?", afterID(in.After)).Order("id ASC").Limit(limit + 1).Find(&cats).Error; err != nil { return pact.ContentPage{}, err } more := len(cats) > limit if more { cats = cats[:limit] } records, err := describeCategories(ctx, db, cats, in.Pattern, in.UnprefixedLocale) if err != nil { return pact.ContentPage{}, err } return contentPage(records, more), nil } // describeCategories describes categories, always published: each locale's // URL and title as resolveCategory builds them, the translated description // as plain text and the word count of the category's content. func describeCategories(ctx context.Context, db *gorm.DB, cats []models.Category, pattern, unprefixed string) ([]pact.ContentRecord, error) { ids := make([]uint, 0, len(cats)) for _, c := range cats { ids = append(ids, c.ID) } ci, err := newCategoryItems(ctx, db, pact.MenuItemResolve{Pattern: pattern, UnprefixedLocale: unprefixed}, ids) if err != nil { return nil, err } out := make([]pact.ContentRecord, 0, len(cats)) for _, c := range cats { item := ci.item(c) rec := pact.ContentRecord{ID: strconv.FormatUint(uint64(c.ID), 10), Published: true, LastMod: copyTime(c.UpdatedAt)} var words *int if c.Content != nil && strings.TrimSpace(*c.Content) != "" { n := wordCount(plainText(*c.Content)) words = &n } host := "" if c.Description != nil { host = *c.Description } for _, alt := range item.Alternates { locale := alt.Locale if locale == xDefaultLocale { locale = "" } rec.Locales = append(rec.Locales, pact.ContentLocale{ Locale: alt.Locale, URL: alt.URL, Title: alt.Title, Description: plainText(ci.tr.value(locale, c.ID, "description", host)), Words: words, }) } out = append(out, rec) } return out, nil } // matchCategory is the Match of journal-category, by translated or default // slug or :id, exact only on the category's own URL in the path's locale. func (m menuTypes) matchCategory(ctx context.Context, in pact.ContentMatch) (*pact.ContentMatchResult, error) { db, err := m.contentDB(ctx) if err != nil { return nil, err } morph := models.Category{}.MorphName() find := func(values map[string]string, locale string, locales menuLocales) (uint, error) { return findMatchRecord(ctx, db, values, locale, locales, morph, func(q *gorm.DB) *gorm.DB { return q.Model(&models.Category{}) }) } urls := func(id uint) ([]pact.MenuItemAlternate, error) { var cats []models.Category if err := db.Where("id = ?", id).Limit(1).Find(&cats).Error; err != nil || len(cats) == 0 { return nil, err } ci, err := newCategoryItems(ctx, db, pact.MenuItemResolve{Pattern: in.Pattern, UnprefixedLocale: in.UnprefixedLocale}, []uint{id}) if err != nil { return nil, err } return ci.item(cats[0]).Alternates, nil } return matchContent(ctx, db, in, find, urls) } // tagContent is the D-93 descriptor of journal-tag. func (m menuTypes) tagContent() *pact.ContentDescriptor { return &pact.ContentDescriptor{ Controller: tagsControllerID, Model: models.Tag{}.MorphName(), Records: m.listTags, Describe: m.describeTag, Match: m.matchTag, } } // tagReferences lists every tag by name, ties by id. func (m menuTypes) tagReferences(ctx context.Context) ([]pact.MenuItemReference, error) { db, err := m.db(ctx) if err != nil { return nil, err } var tags []models.Tag if err := db.Order("name ASC, id ASC").Find(&tags).Error; err != nil { return nil, err } refs := make([]pact.MenuItemReference, 0, len(tags)) for _, tag := range tags { refs = append(refs, pact.MenuItemReference{Value: strconv.FormatUint(uint64(tag.ID), 10), Label: tag.Name}) } return refs, nil } // resolveTag resolves one journal-tag item; an unknown tag, and any tag on a // host without tag pages, resolve to nothing. func (m menuTypes) resolveTag(ctx context.Context, in pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { id, ok := menuReferenceID(in.Reference) if !ok || !m.tagPages() { return nil, nil } db, err := m.db(ctx) if err != nil { return nil, err } var tags []models.Tag if err := db.Where("id = ?", id).Limit(1).Find(&tags).Error; err != nil { return nil, err } if len(tags) == 0 { return nil, nil } locales, err := loadMenuLocales(ctx, db, in.UnprefixedLocale) if err != nil { return nil, err } tag := tags[0] item := resolvedMenuItem(in.Pattern, locales, tag.Name, tag.UpdatedAt, tagParams(tag), func(string) string { return tag.Name }) return &item, nil } // tagParams are a tag's placeholder values; tags are not translatable. func tagParams(tag models.Tag) func(string) [][2]string { return func(string) [][2]string { return [][2]string{{"id", strconv.FormatUint(uint64(tag.ID), 10)}, {"slug", tag.Slug}} } } func (m menuTypes) describeTag(ctx context.Context, in pact.ContentDescribe) (*pact.ContentRecord, error) { id, ok := menuReferenceID(in.ID) if !ok { return nil, nil } db, err := m.contentDB(ctx) if err != nil { return nil, err } var tags []models.Tag if err := db.Where("id = ?", id).Limit(1).Find(&tags).Error; err != nil { return nil, err } if len(tags) == 0 { return nil, nil } records, err := m.describeTags(ctx, db, tags, in.Pattern, in.UnprefixedLocale) if err != nil { return nil, err } return &records[0], nil } func (m menuTypes) listTags(ctx context.Context, in pact.ContentList) (pact.ContentPage, error) { db, err := m.contentDB(ctx) if err != nil { return pact.ContentPage{}, err } limit := pageLimit(in.Limit) var tags []models.Tag if err := db.Where("id > ?", afterID(in.After)).Order("id ASC").Limit(limit + 1).Find(&tags).Error; err != nil { return pact.ContentPage{}, err } more := len(tags) > limit if more { tags = tags[:limit] } records, err := m.describeTags(ctx, db, tags, in.Pattern, in.UnprefixedLocale) if err != nil { return pact.ContentPage{}, err } return contentPage(records, more), nil } // describeTags describes tags, always published, one locale per enabled // locale with the same name and slug; URLs are empty without tag pages. func (m menuTypes) describeTags(ctx context.Context, db *gorm.DB, tags []models.Tag, pattern, unprefixed string) ([]pact.ContentRecord, error) { locales, err := loadMenuLocales(ctx, db, unprefixed) if err != nil { return nil, err } pages := m.tagPages() out := make([]pact.ContentRecord, 0, len(tags)) for _, tag := range tags { rec := pact.ContentRecord{ID: strconv.FormatUint(uint64(tag.ID), 10), Published: true, LastMod: copyTime(tag.UpdatedAt)} desc := "" if tag.Description != nil { desc = plainText(*tag.Description) } for _, alt := range menuAlternates(pattern, locales, tagParams(tag), func(string) string { return tag.Name }) { url := alt.URL if !pages { url = "" } rec.Locales = append(rec.Locales, pact.ContentLocale{Locale: alt.Locale, URL: url, Title: tag.Name, Description: desc}) } out = append(out, rec) } return out, nil } // matchTag is the Match of journal-tag, by slug or :id; nothing matches on // a host without tag pages. func (m menuTypes) matchTag(ctx context.Context, in pact.ContentMatch) (*pact.ContentMatchResult, error) { if !m.tagPages() { return nil, nil } db, err := m.contentDB(ctx) if err != nil { return nil, err } find := func(values map[string]string, _ string, _ menuLocales) (uint, error) { q := db.Model(&models.Tag{}) switch { case values["slug"] != "": q = q.Where("slug = ?", values["slug"]) case values["id"] != "": id, ok := menuReferenceID(values["id"]) if !ok { return 0, nil } q = q.Where("id = ?", id) default: return 0, nil } var ids []uint if err := q.Order("id ASC").Limit(1).Pluck("id", &ids).Error; err != nil || len(ids) == 0 { return 0, err } return ids[0], nil } urls := func(id uint) ([]pact.MenuItemAlternate, error) { var tags []models.Tag if err := db.Where("id = ?", id).Limit(1).Find(&tags).Error; err != nil || len(tags) == 0 { return nil, err } locales, err := loadMenuLocales(ctx, db, in.UnprefixedLocale) if err != nil { return nil, err } tag := tags[0] return menuAlternates(in.Pattern, locales, tagParams(tag), func(string) string { return tag.Name }), nil } return matchContent(ctx, db, in, find, urls) } // pathReading is one way to read a public path: the locale it addresses, // the path without its locale prefix, and whether the prefix is the one the // locale's URLs carry. type pathReading struct { locale string rest string prefixOK bool } // readPath lists the readings of path: with a prefix of an enabled locale // other than the unprefixed one, with the unprefixed locale's own code as a // prefix (stale, never exact), and as a path of the unprefixed locale. func readPath(path string, locales menuLocales) []pathReading { out := []pathReading{} seg, rest := path, "/" if trimmed := strings.TrimPrefix(path, "/"); trimmed != path { if i := strings.IndexByte(trimmed, '/'); i >= 0 { seg, rest = trimmed[:i], trimmed[i:] } else { seg = trimmed } for _, code := range locales.codes { if seg == code { out = append(out, pathReading{locale: code, rest: rest, prefixOK: code != locales.unprefixed}) } } } return append(out, pathReading{locale: locales.unprefixed, rest: path, prefixOK: true}) } // matchContent reads the path in every way readPath lists, finds the record // its placeholder values name (find) and compares the path with the // record's URL in the reading's locale (urls). The first exact reading // wins, else the first that found a record. func matchContent(ctx context.Context, db *gorm.DB, in pact.ContentMatch, find func(values map[string]string, locale string, locales menuLocales) (uint, error), urls func(id uint) ([]pact.MenuItemAlternate, error)) (*pact.ContentMatchResult, error) { locales, err := loadMenuLocales(ctx, db, in.UnprefixedLocale) if err != nil { return nil, err } var fallback *pact.ContentMatchResult for _, reading := range readPath(in.Path, locales) { values, ok := signpost.MatchPattern(in.Pattern, reading.rest) if !ok { continue } id, err := find(values, reading.locale, locales) if err != nil { return nil, err } if id == 0 { continue } alternates, err := urls(id) if err != nil { return nil, err } exact := false for _, alt := range alternates { if alt.Locale == reading.locale || (len(locales.codes) == 0 && alt.Locale == xDefaultLocale) { exact = reading.prefixOK && alt.URL == in.Path break } } res := &pact.ContentMatchResult{ID: strconv.FormatUint(uint64(id), 10), Locale: reading.locale, Exact: exact} if exact { return res, nil } if fallback == nil { fallback = res } } return fallback, nil } // findMatchRecord finds the record a path's values name among scope: by // :slug, first the slug of the path's locale (translate's slug index), then // the default slug, then a slug of any other locale; else by :id. A value // reaches the database only as a query parameter. func findMatchRecord(ctx context.Context, db *gorm.DB, values map[string]string, locale string, locales menuLocales, morph string, scope func(*gorm.DB) *gorm.DB) (uint, error) { first := func(ids []uint) (uint, error) { if len(ids) == 0 { return 0, nil } var found []uint if err := scope(db.Session(&gorm.Session{NewDB: true, Context: ctx})).Where("id IN ?", ids).Order("id ASC").Limit(1).Pluck("id", &found).Error; err != nil || len(found) == 0 { return 0, err } return found[0], nil } if slug := values["slug"]; slug != "" { if locale != "" && locale != locales.defaultCode { ids, err := slugIndexIDs(ctx, db, morph, slug, locale) if err != nil { return 0, err } if id, err := first(ids); err != nil || id != 0 { return id, err } } var hosts []uint if err := scope(db.Session(&gorm.Session{NewDB: true, Context: ctx})).Where("slug = ?", slug).Order("id ASC").Pluck("id", &hosts).Error; err != nil { return 0, err } if id, err := first(hosts); err != nil || id != 0 { return id, err } ids, err := slugIndexIDs(ctx, db, morph, slug, "") if err != nil { return 0, err } return first(ids) } if raw := values["id"]; raw != "" { id, ok := menuReferenceID(raw) if !ok { return 0, nil } return first([]uint{id}) } return 0, nil } // slugIndexIDs are the ids of morph records whose translated slug is slug // in locale ("" for any locale), from translate's index table. func slugIndexIDs(ctx context.Context, db *gorm.DB, morph, slug, locale string) ([]uint, error) { q := db.WithContext(ctx).Table(translatemodels.Index{}.TableName()). Where("model_type = ? AND item = ? AND value = ?", morph, "slug", slug) if locale != "" { q = q.Where("locale = ?", locale) } var raw []string if err := q.Order("id ASC").Pluck("model_id", &raw).Error; err != nil { return nil, err } ids := make([]uint, 0, len(raw)) for _, r := range raw { if id, ok := menuReferenceID(r); ok { ids = append(ids, id) } } return ids, nil }