diff --git a/README.md b/README.md index dee4119..8f005c8 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ WinterCMS theme components such as `journalPost` and `journalPosts` are a later ## Features -- **Posts.** `models.Post` maps `golem15_journal_posts`. Translatable fields are title, content, content_html, excerpt, metadata, and slug (indexed). Fillable does not include `redactor_id`, `content_html`, or `user_id`. Attachments `featured_images` and `content_images` use `system_files` under the same MorphName. YAML admin uses `mltext`/`mlmarkdown`. `classes.FormatHTML` regenerates `content_html` on every write path (admin save, `journal:import-posts`, Journal API writes). It renders markdown and passes raw HTML in `content` through the renderer (Winter parity), then sanitizes the result with a `bluemonday` allow-list: bluemonday's UGC policy plus `footer`, `class`, schema.org microdata, the `color`, `font-size`, `font-weight` and `text-align` styles, goldmark's footnote roles, `dir="auto"` and two pasted-transcript `data-message-*` attributes. Like Winter's `Html::clean`, it strips what the policy does not allow instead of refusing the save: script, style, iframe, object, embed, form, base and meta elements, comments, event handler attributes, and URLs other than http, https, mailto and relative ones. HTML the policy keeps whole is stored byte for byte, so imported bodies survive a save. `content` itself is stored as submitted. The API response shapes are unchanged. +- **Posts.** `models.Post` maps `golem15_journal_posts`. Translatable fields are title, content, content_html, excerpt, metadata, and slug (indexed). Fillable does not include `redactor_id`, `content_html`, or `user_id`. Attachments `featured_images` and `content_images` use `system_files` under the same MorphName. YAML admin uses `mltext`/`mlmarkdown`. `classes.FormatHTML` regenerates `content_html` on every write path (admin save, `journal:import-posts`, Journal API writes). It renders markdown and passes raw HTML in `content` through the renderer (Winter parity), then sanitizes the result with a `bluemonday` allow-list: bluemonday's UGC policy plus `footer`, `class`, schema.org microdata, the `color`, `font-size`, `font-weight` and `text-align` styles, goldmark's footnote roles, `dir="auto"` and two pasted-transcript `data-message-*` attributes. Like Winter's `Html::clean`, it strips what the policy does not allow instead of refusing the save: script, style, iframe, object, embed, form, base and meta elements, comments, event handler attributes, and URLs other than http, https, mailto and relative ones. HTML the policy keeps whole is stored byte for byte, so imported bodies survive a save. `content` itself is stored as submitted. Translated `content_html` goes through the same policy on the API and `journal:import-posts` write paths (`classes.FormatTranslation`): when the translation carries a non-blank `content`, `content_html` is rendered from it and a supplied `content_html` is replaced, as the host column is always derived; a translation with only `content_html` is sanitized with `classes.SanitizeHTML`. Winter stored a supplied translated `content_html` verbatim. The API response shapes and status codes are unchanged. - **Categories and tags.** `models.Category` is translatable (name, description, slug). It also stores `content`, an HTML host column that is not translatable, edited in the admin as a textarea. Fillable is name, slug, code, description, content and parent_id; nest_* columns exist in DDL and are not fillable. `models.Tag` is not translatable. Fillable is name, slug, description. Pivots are `golem15_journal_posts_categories` and `golem15_journal_posts_tags`. - **Settings.** Singleton row ID=1 on `golem15_journal_settings`, not PHP `system_settings`. `search_use_typesense` defaults to false. A fresh save never contacts Typesense. - **Public HTTP.** Anonymous GET `/_journal/api/v1/posts`, `/posts/{slug}`, `/categories`, `/tags`, `/rss`, `/authors/{author}`, and `/authors/{author}/posts`. Writes under the same prefix require a cabana backend JWT (`aud=backend`). @@ -177,6 +177,8 @@ func (Post) MorphName() string { return `Golem15\Journal\Models\Post | `models.Settings` | Singleton settings row; `TableName` is `golem15_journal_settings`. | | `models.All` | Models registered by this plugin. | | `updates.All` | gormigrate steps registered by this plugin. | +| `classes.SanitizeHTML` | Strips from HTML (not markdown) what the `FormatHTML` allow-list does not allow; HTML the policy keeps whole is returned byte for byte. | +| `classes.FormatTranslation` | Sets `content_html` in one locale's translated attributes: rendered from a non-blank `content` with `FormatHTML`, otherwise a supplied `content_html` sanitized with `SanitizeHTML`. | | `classes.CheckHTML` | Gate for HTML stored as is, such as category content: refuses HTML that the `FormatHTML` allow-list would strip or change, instead of stripping it. | ## CLI commands diff --git a/classes/format_html.go b/classes/format_html.go index 513f583..686c3c6 100644 --- a/classes/format_html.go +++ b/classes/format_html.go @@ -72,8 +72,54 @@ func FormatHTML(src string) (string, error) { if err := markdownEngine.Convert([]byte(strings.TrimSpace(src)), &buf); err != nil { return "", fmt.Errorf("journal: markdown: %w", err) } - out, _ := sanitizeHTML(buf.String()) - return out, nil + return SanitizeHTML(buf.String()), nil +} + +// SanitizeHTML filters doc, which is HTML and not markdown, through the +// FormatHTML allow-list and strips what the policy does not allow. HTML the +// policy keeps whole is returned byte for byte. It is the gate for +// content_html supplied without markdown, such as a translation that carries +// only content_html. +func SanitizeHTML(doc string) string { + out, _ := sanitizeHTML(doc) + return out +} + +// FormatTranslation sets attrs["content_html"] for one locale's translated +// post attributes, so a translated content_html is stored under the same +// policy as the host column. When attrs["content"] is not blank, content_html +// is rendered from it with FormatHTML and a supplied content_html is +// replaced, as the host column is always derived from content. Otherwise a +// supplied, non-null content_html is sanitized with SanitizeHTML; a value +// that is not a string is sanitized as its text form. attrs without content +// or content_html is left unchanged. +func FormatTranslation(attrs map[string]any) error { + if attrs == nil { + return nil + } + if content := textOf(attrs["content"]); strings.TrimSpace(content) != "" { + html, err := FormatHTML(content) + if err != nil { + return err + } + attrs["content_html"] = html + return nil + } + if v, has := attrs["content_html"]; has && v != nil { + attrs["content_html"] = SanitizeHTML(textOf(v)) + } + return nil +} + +func textOf(v any) string { + switch t := v.(type) { + case nil: + return "" + case string: + return t + default: + return fmt.Sprint(t) + } } // CheckHTML gates HTML that is stored as is, such as category content. It diff --git a/classes/format_translation_test.go b/classes/format_translation_test.go new file mode 100644 index 0000000..0e42806 --- /dev/null +++ b/classes/format_translation_test.go @@ -0,0 +1,124 @@ +package classes + +import "testing" + +func TestSanitizeHTML(t *testing.T) { + const clean = `

Zobacz źródło.

` + if got := SanitizeHTML(clean); got != clean { + t.Fatalf("clean HTML changed: %q", got) + } + if got, want := SanitizeHTML(`

a

`), `

a

`; got != want { + t.Fatalf("got %q, want %q", got, want) + } + // SanitizeHTML takes HTML, not markdown: markdown syntax stays text. + if got := SanitizeHTML("**bold**"); got != "**bold**" { + t.Fatalf("markdown was rendered: %q", got) + } + for _, tc := range unsafePayloads { + if tc.src[0] != '<' { + continue // markdown, not HTML + } + t.Run(tc.name, func(t *testing.T) { + assertNoActiveContent(t, SanitizeHTML(tc.src)) + }) + } +} + +func TestFormatTranslation(t *testing.T) { + const payload = `

Treść

x` + + t.Run("content derives content_html", func(t *testing.T) { + attrs := map[string]any{"title": "Witaj", "content": "Witaj **świecie**"} + if err := FormatTranslation(attrs); err != nil { + t.Fatal(err) + } + if got, want := attrs["content_html"], "

Witaj świecie

\n"; got != want { + t.Fatalf("content_html = %#v, want %q", got, want) + } + if attrs["content"] != "Witaj **świecie**" || attrs["title"] != "Witaj" { + t.Fatalf("other attributes changed: %#v", attrs) + } + }) + + t.Run("content wins over a supplied content_html", func(t *testing.T) { + attrs := map[string]any{"content": "Tekst", "content_html": payload} + if err := FormatTranslation(attrs); err != nil { + t.Fatal(err) + } + if got, want := attrs["content_html"], "

Tekst

\n"; got != want { + t.Fatalf("content_html = %#v, want %q", got, want) + } + }) + + t.Run("raw HTML in content is sanitized", func(t *testing.T) { + attrs := map[string]any{"content": payload} + if err := FormatTranslation(attrs); err != nil { + t.Fatal(err) + } + assertNoActiveContent(t, attrs["content_html"].(string)) + }) + + t.Run("content_html alone is sanitized", func(t *testing.T) { + for _, attrs := range []map[string]any{ + {"content_html": payload}, + {"content": "", "content_html": payload}, + {"content": " ", "content_html": payload}, + {"content": nil, "content_html": payload}, + } { + if err := FormatTranslation(attrs); err != nil { + t.Fatal(err) + } + got, ok := attrs["content_html"].(string) + if !ok { + t.Fatalf("content_html = %#v, want a string", attrs["content_html"]) + } + assertNoActiveContent(t, got) + if want := `

Treść

x`; got != want { + t.Fatalf("content_html = %q, want %q", got, want) + } + } + }) + + t.Run("clean content_html alone is kept byte for byte", func(t *testing.T) { + const clean = "

Lion's Mane, \"superior herb\" & inne

" + attrs := map[string]any{"content_html": clean} + if err := FormatTranslation(attrs); err != nil { + t.Fatal(err) + } + if attrs["content_html"] != clean { + t.Fatalf("content_html = %#v, want %q", attrs["content_html"], clean) + } + }) + + t.Run("non-string content_html is sanitized as text", func(t *testing.T) { + attrs := map[string]any{"content_html": []any{``}} + if err := FormatTranslation(attrs); err != nil { + t.Fatal(err) + } + got, ok := attrs["content_html"].(string) + if !ok { + t.Fatalf("content_html = %#v, want a string", attrs["content_html"]) + } + assertNoActiveContent(t, got) + }) + + t.Run("absent and null content_html stay as they are", func(t *testing.T) { + attrs := map[string]any{"title": "Tytuł"} + if err := FormatTranslation(attrs); err != nil { + t.Fatal(err) + } + if _, has := attrs["content_html"]; has { + t.Fatalf("content_html added: %#v", attrs) + } + null := map[string]any{"content_html": nil} + if err := FormatTranslation(null); err != nil { + t.Fatal(err) + } + if v, has := null["content_html"]; !has || v != nil { + t.Fatalf("null content_html changed: %#v", null) + } + if err := FormatTranslation(nil); err != nil { + t.Fatal(err) + } + }) +} diff --git a/console/import_posts.go b/console/import_posts.go index 0e04446..04b0ecc 100644 --- a/console/import_posts.go +++ b/console/import_posts.go @@ -312,6 +312,11 @@ func syncTranslations(ctx context.Context, db *gorm.DB, post *models.Post, trans } fields := []string{"title", "slug", "content", "content_html", "excerpt", "metadata"} for locale, attrs := range translations { + // A translated content_html is derived from content or sanitized, + // as the host column is; Winter's ImportPosts stored it verbatim. + if err := classes.FormatTranslation(attrs); err != nil { + return err + } for _, field := range fields { val, ok := attrs[field] if !ok || val == nil { diff --git a/controllers/api/posts_helpers.go b/controllers/api/posts_helpers.go index 86faa25..084c198 100644 --- a/controllers/api/posts_helpers.go +++ b/controllers/api/posts_helpers.go @@ -764,12 +764,11 @@ func syncTranslations(ctx context.Context, db *gorm.DB, post *models.Post, raw a if !ok { continue } - if content := asString(attrs["content"]); strings.TrimSpace(content) != "" && strings.TrimSpace(asString(attrs["content_html"])) == "" { - html, err := classes.FormatHTML(content) - if err != nil { - return err - } - attrs["content_html"] = html + // Every translated content_html goes through the host column's + // policy: derived from content, or sanitized when only content_html + // is supplied. Winter stored a supplied content_html verbatim. + if err := classes.FormatTranslation(attrs); err != nil { + return err } for _, field := range fields { if val, has := attrs[field]; has && val != nil { diff --git a/journal_translation_html_test.go b/journal_translation_html_test.go new file mode 100644 index 0000000..438d39a --- /dev/null +++ b/journal_translation_html_test.go @@ -0,0 +1,239 @@ +package journal_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.golem15.com/golem15/sm-journal-plugin/console" + "git.golem15.com/golem15/sm-journal-plugin/controllers" + "git.golem15.com/golem15/sm-journal-plugin/models" +) + +// translatedXSS is a translated content_html carrying an event handler and a +// javascript URL. Stored verbatim it would reach anonymous readers through +// the public posts API. +const translatedXSS = `

Treść

x` + +// translatedXSSClean is translatedXSS after the FormatHTML allow-list. +const translatedXSSClean = `

Treść

x` + +// assertNoXSS fails when html still holds the translatedXSS payload. +func assertNoXSS(t *testing.T, where, html string) { + t.Helper() + lower := strings.ToLower(html) + for _, bad := range []string{"onerror", "javascript:", "Witaj świecie

\n"; got != want { + t.Fatalf("pl content_html = %#v, want %q", got, want) + } + if got := tr["pl"]["content"]; got != "Witaj **świecie**" { + t.Fatalf("pl content = %#v, stored as submitted expected", got) + } + }) + + t.Run("content wins over a supplied content_html", func(t *testing.T) { + create("tr-both", map[string]any{ + "pl": map[string]any{"content": "Tekst", "content_html": translatedXSS}, + }) + tr, _ := publicTranslations(t, env, "tr-both") + if got, want := tr["pl"]["content_html"], "

Tekst

\n"; got != want { + t.Fatalf("pl content_html = %#v, want %q", got, want) + } + }) + + t.Run("default-locale translation cannot bypass the host column", func(t *testing.T) { + create("tr-default-locale", map[string]any{ + "en": map[string]any{"content_html": translatedXSS}, + }) + _, host := publicTranslations(t, env, "tr-default-locale") + if host != translatedXSSClean { + t.Fatalf("host content_html = %q, want %q", host, translatedXSSClean) + } + }) + + t.Run("update sanitizes translated content_html", func(t *testing.T) { + var post models.Post + if err := env.gdb.Where("slug = ?", "tr-content-only").Take(&post).Error; err != nil { + t.Fatal(err) + } + rec := env.call(t, http.MethodPut, fmt.Sprintf("/_journal/api/v1/posts/%d", post.ID), tok, map[string]any{ + "translations": map[string]any{ + "pl": map[string]any{"content": "", "content_html": translatedXSS}, + }, + }) + if rec.Code != http.StatusOK { + t.Fatalf("update = %d, want 200: %s", rec.Code, rec.Body.String()) + } + assertNoXSS(t, "update response", rec.Body.String()) + tr, _ := publicTranslations(t, env, "tr-content-only") + if got := tr["pl"]["content_html"]; got != translatedXSSClean { + t.Fatalf("pl content_html after update = %#v, want %q", got, translatedXSSClean) + } + }) + + list := env.call(t, http.MethodGet, "/_journal/api/v1/posts", "", nil) + if list.Code != http.StatusOK { + t.Fatalf("anonymous list = %d: %s", list.Code, list.Body.String()) + } + assertNoXSS(t, "anonymous list", list.Body.String()) +} + +// TestJournalImportTranslatedContentHTML pins W-1 on journal:import-posts: +// a translated content_html in the import file is stored sanitized, and a +// translation with only content gets a derived content_html. +func TestJournalImportTranslatedContentHTML(t *testing.T) { + env := newPostsEnv(t) + dir := t.TempDir() + published := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) + files := []console.PostFile{ + { + Slug: "import-html-only", + Title: "Import HTML", + Content: "Host", + Published: true, + PublishedAt: &published, + Translations: map[string]map[string]any{ + "pl": {"title": "Import HTML", "content_html": translatedXSS}, + }, + }, + { + Slug: "import-content-only", + Title: "Import content", + Content: "Host", + Published: true, + PublishedAt: &published, + Translations: map[string]map[string]any{ + "pl": {"title": "Import treści", "content": "Witaj **świecie**"}, + }, + }, + } + for _, f := range files { + raw, err := json.Marshal(f) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, f.Slug+".json"), raw, 0o644); err != nil { + t.Fatal(err) + } + } + stats, err := console.ImportFromDir(context.Background(), env.gdb, dir, false, false, nil, nil) + if err != nil { + t.Fatalf("import: %v", err) + } + if stats.Created != 2 || stats.Failed != 0 { + t.Fatalf("import stats = %+v, want 2 created", stats) + } + + stored := func(slug string) map[string]any { + t.Helper() + var post models.Post + if err := env.gdb.Where("slug = ?", slug).Take(&post).Error; err != nil { + t.Fatal(err) + } + var raw string + if err := env.gdb.Table("golem15_translate_attributes"). + Select("attribute_data"). + Where("model_id = ? AND model_type = ? AND locale = ?", fmt.Sprint(post.ID), post.MorphName(), "pl"). + Scan(&raw).Error; err != nil { + t.Fatal(err) + } + var data map[string]any + if err := json.Unmarshal([]byte(raw), &data); err != nil { + t.Fatalf("attribute_data %q: %v", raw, err) + } + return data + } + + if got := stored("import-html-only")["content_html"]; got != translatedXSSClean { + t.Fatalf("imported pl content_html = %#v, want %q", got, translatedXSSClean) + } + if got, want := stored("import-content-only")["content_html"], "

Witaj świecie

\n"; got != want { + t.Fatalf("imported pl content_html = %#v, want %q", got, want) + } + + tr, _ := publicTranslations(t, env, "import-html-only") + if got := tr["pl"]["content_html"]; got != translatedXSSClean { + t.Fatalf("public pl content_html = %#v, want %q", got, translatedXSSClean) + } +}