Jakub Zych
2384a61e38
- figure data-embed ^youtube$ and data-id ^[A-Za-z0-9_-]{11}$ (the embed
placeholder), img alt with any text, ol start ^[0-9]+$; nothing else
- TestEditorOutputPassesPolicy checks every editor-corpus.json entry byte
for byte with an empty report, plus the negative cases;
TestSanitizeHTMLReport pins the removal report
- CheckHTML documented as a predicate no write path refuses with; README
lists the four additions and the report API
507 lines
16 KiB
Go
507 lines
16 KiB
Go
package classes
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
|
|
"github.com/microcosm-cc/bluemonday"
|
|
"github.com/yuin/goldmark"
|
|
"github.com/yuin/goldmark/extension"
|
|
"github.com/yuin/goldmark/parser"
|
|
"github.com/yuin/goldmark/renderer/html"
|
|
xhtml "golang.org/x/net/html"
|
|
)
|
|
|
|
var (
|
|
markdownEngine = goldmark.New(
|
|
goldmark.WithExtensions(extension.Footnote, extension.Table),
|
|
goldmark.WithParserOptions(parser.WithAttribute()),
|
|
// Winter's Markdown::parse passes raw HTML through, so imported HTML
|
|
// bodies round-trip (D-03, PHP parity); htmlPolicy then sanitizes the
|
|
// rendered result on every write path.
|
|
goldmark.WithRendererOptions(html.WithUnsafe()),
|
|
)
|
|
|
|
// htmlPolicy is the allow-list for stored journal HTML.
|
|
htmlPolicy = newHTMLPolicy()
|
|
)
|
|
|
|
// newHTMLPolicy extends bluemonday's UGC policy only with what goldmark
|
|
// renders and what the imported posts and categories use.
|
|
func newHTMLPolicy() *bluemonday.Policy {
|
|
p := bluemonday.UGCPolicy()
|
|
// Winter neither adds nor requires rel="nofollow", so links keep their
|
|
// stored form.
|
|
p.RequireNoFollowOnLinks(false)
|
|
// Imported author footers: <footer class=... itemscope itemtype=...>.
|
|
p.AllowElements("footer")
|
|
p.AllowAttrs("class").Matching(bluemonday.SpaceSeparatedTokens).Globally()
|
|
p.AllowAttrs("itemscope").Matching(regexp.MustCompile(`^(|itemscope)$`)).Globally()
|
|
p.AllowAttrs("itemtype").Matching(regexp.MustCompile(`^https?://schema\.org/[A-Za-z]+$`)).Globally()
|
|
p.AllowAttrs("itemprop").Matching(bluemonday.SpaceSeparatedTokens).Globally()
|
|
// Inline styles in imported footers and goldmark's table cell alignment.
|
|
// bluemonday validates each property value.
|
|
p.AllowStyles("color", "font-size", "font-weight", "text-align").Globally()
|
|
// goldmark's footnote markup.
|
|
p.AllowAttrs("role").Matching(regexp.MustCompile(`^doc-(noteref|endnotes|backlink)$`)).OnElements("a", "div")
|
|
// Pasted chat transcripts in imported category bodies: dir="auto" (the
|
|
// UGC policy allows only ltr and rtl) and two data attributes.
|
|
p.AllowAttrs("dir").Matching(regexp.MustCompile(`(?i)^(ltr|rtl|auto)$`)).Globally()
|
|
p.AllowAttrs("data-message-author-role", "data-message-id").
|
|
Matching(regexp.MustCompile(`^[A-Za-z0-9_-]+$`)).OnElements("div")
|
|
// Blog Editor output (Phase 16.2 D-05), exactly what the editor writes
|
|
// and nothing more; plan 02's editor-corpus.json pins it
|
|
// (TestEditorOutputPassesPolicy) and the site layer mirrors these rules.
|
|
// The embed placeholder <figure data-embed="youtube" data-id="..."></figure>:
|
|
// the provider name and the 11-character video id, on figure only. The
|
|
// layer expands it; no iframe is ever stored.
|
|
p.AllowAttrs("data-embed").Matching(regexp.MustCompile(`^youtube$`)).OnElements("figure")
|
|
p.AllowAttrs("data-id").Matching(regexp.MustCompile(`^[A-Za-z0-9_-]{11}$`)).OnElements("figure")
|
|
// The editor's alt field takes free text, which UGC's Paragraph pattern
|
|
// cuts at ':', '?', '"' and more. bluemonday accepts an attribute when
|
|
// any of its rules matches and escapes the value on output; alt carries
|
|
// no URL.
|
|
p.AllowAttrs("alt").OnElements("img")
|
|
// TipTap's ordered-list input rule writes start when a list does not
|
|
// begin at 1.
|
|
p.AllowAttrs("start").Matching(regexp.MustCompile(`^[0-9]+$`)).OnElements("ol")
|
|
return p
|
|
}
|
|
|
|
// FormatHTML converts journal post markdown to stored HTML using goldmark
|
|
// footnote, table, and attribute extensions. Raw HTML in the source passes
|
|
// through the renderer, as it does in Winter's Markdown::parse. The result is
|
|
// then sanitized with a bluemonday allow-list that strips, as Winter's
|
|
// Html::clean does, what it does not allow: script, style, iframe, object,
|
|
// embed, form, base and meta elements, event handler attributes, and URLs
|
|
// other than http, https, mailto and relative ones. HTML that the policy
|
|
// keeps whole is returned byte for byte, so imported bodies round-trip. It
|
|
// does not call cabana.RenderMarkdown, whose admin preview stays without
|
|
// unsafe HTML.
|
|
func FormatHTML(src string) (string, error) {
|
|
var buf bytes.Buffer
|
|
if err := markdownEngine.Convert([]byte(strings.TrimSpace(src)), &buf); err != nil {
|
|
return "", fmt.Errorf("journal: markdown: %w", err)
|
|
}
|
|
return SanitizeHTML(buf.String()), nil
|
|
}
|
|
|
|
// SanitizeHTML filters doc, which is HTML and not markdown, through the
|
|
// FormatHTML allow-list and strips what the policy does not allow. HTML the
|
|
// policy keeps whole is returned byte for byte. It is the gate for
|
|
// content_html supplied without markdown, such as a translation that carries
|
|
// only content_html. It is the string half of SanitizeHTMLReport.
|
|
func SanitizeHTML(doc string) string {
|
|
out, _ := SanitizeHTMLReport(doc)
|
|
return out
|
|
}
|
|
|
|
// maxRemovalNames is how many distinct names an HTMLRemovals list keeps.
|
|
const maxRemovalNames = 10
|
|
|
|
// HTMLRemovals reports what SanitizeHTMLReport removed from a document.
|
|
// Elements and Attributes are the distinct removed element and attribute
|
|
// names in first-seen order, at most ten each; ElementCount and
|
|
// AttributeCount count every removal, so they stay exact when a list is cut.
|
|
// An attribute counts as removed when the policy dropped it, when it is a
|
|
// repeat of a name the tag already has, or when the policy changed its value
|
|
// (a style attribute that lost a property counts under "style"). Comments
|
|
// counts removed HTML comments. Other is set when the policy changed the
|
|
// document in a way none of the other fields describe.
|
|
type HTMLRemovals struct {
|
|
Elements []string
|
|
ElementCount int
|
|
Attributes []string
|
|
AttributeCount int
|
|
Comments int
|
|
Other bool
|
|
|
|
// moreElements and moreAttributes are set when a list was cut.
|
|
moreElements bool
|
|
moreAttributes bool
|
|
}
|
|
|
|
// Empty reports whether nothing was removed, which is exactly when
|
|
// SanitizeHTMLReport returned its input unchanged.
|
|
func (r HTMLRemovals) Empty() bool {
|
|
return r.ElementCount == 0 && r.AttributeCount == 0 && r.Comments == 0 && !r.Other
|
|
}
|
|
|
|
func (r *HTMLRemovals) addElement(name string) {
|
|
r.ElementCount++
|
|
r.Elements, r.moreElements = addRemovalName(r.Elements, r.moreElements, name)
|
|
}
|
|
|
|
func (r *HTMLRemovals) addAttribute(name string) {
|
|
r.AttributeCount++
|
|
r.Attributes, r.moreAttributes = addRemovalName(r.Attributes, r.moreAttributes, name)
|
|
}
|
|
|
|
func addRemovalName(names []string, cut bool, name string) ([]string, bool) {
|
|
if slices.Contains(names, name) {
|
|
return names, cut
|
|
}
|
|
if len(names) >= maxRemovalNames {
|
|
return names, true
|
|
}
|
|
return append(names, name), cut
|
|
}
|
|
|
|
// SanitizeHTMLReport is SanitizeHTML with a report of what the policy
|
|
// removed. HTML the policy keeps whole is returned byte for byte with an
|
|
// empty report. Otherwise the policy's output is returned and the report is
|
|
// built by aligning the start tags of doc with those of the output in order
|
|
// (the policy only removes, it never adds or reorders): an input tag without
|
|
// a counterpart is a removed element, and an attribute of a matched tag that
|
|
// the output lacks or holds with another value is a removed attribute.
|
|
// Comments are counted on their own. The report is empty exactly when the
|
|
// returned string equals doc.
|
|
func SanitizeHTMLReport(doc string) (string, HTMLRemovals) {
|
|
clean, changed := sanitizeHTML(doc)
|
|
if !changed || clean == doc {
|
|
return doc, HTMLRemovals{}
|
|
}
|
|
report := removalReport(doc, clean)
|
|
if report.Empty() {
|
|
report.Other = true
|
|
}
|
|
return clean, report
|
|
}
|
|
|
|
// reportTag is one start tag of a document as removalReport sees it.
|
|
type reportTag struct {
|
|
name string
|
|
raw []string // attribute names in source order, repeats included
|
|
attrs map[string]string // decoded values, the first of a repeated name
|
|
}
|
|
|
|
// reportTokens returns doc's start tags and its number of comments.
|
|
func reportTokens(doc string) ([]reportTag, int) {
|
|
var (
|
|
tags []reportTag
|
|
comments int
|
|
)
|
|
z := xhtml.NewTokenizer(strings.NewReader(doc))
|
|
for {
|
|
tt := z.Next()
|
|
switch tt {
|
|
case xhtml.ErrorToken:
|
|
return tags, comments
|
|
case xhtml.CommentToken:
|
|
comments++
|
|
case xhtml.StartTagToken, xhtml.SelfClosingTagToken:
|
|
raw := rawAttrNames(z.Raw())
|
|
t := z.Token()
|
|
attrs := make(map[string]string, len(t.Attr))
|
|
for _, a := range t.Attr {
|
|
key := a.Key
|
|
if a.Namespace != "" {
|
|
key = a.Namespace + ":" + a.Key
|
|
}
|
|
if _, seen := attrs[key]; !seen {
|
|
attrs[key] = a.Val
|
|
}
|
|
}
|
|
tags = append(tags, reportTag{name: t.Data, raw: raw, attrs: attrs})
|
|
}
|
|
}
|
|
}
|
|
|
|
// keeps reports whether out can be the policy's output of in: the same
|
|
// element, and every output attribute present in the input with the same
|
|
// value, except style, whose properties the policy filters one by one.
|
|
func (in reportTag) keeps(out reportTag) bool {
|
|
if in.name != out.name {
|
|
return false
|
|
}
|
|
for key, val := range out.attrs {
|
|
got, ok := in.attrs[key]
|
|
if !ok || (got != val && key != "style") {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// removalReport compares doc with the policy's output clean. It aligns the
|
|
// start tags greedily, preferring an output tag whose attributes the input
|
|
// tag can have produced; when that leaves output tags unmatched, it aligns
|
|
// by element name only, which always succeeds when the output's tags are a
|
|
// subsequence of the input's.
|
|
func removalReport(doc, clean string) HTMLRemovals {
|
|
in, inComments := reportTokens(doc)
|
|
out, outComments := reportTokens(clean)
|
|
match := alignTags(in, out, reportTag.keeps)
|
|
if match == nil {
|
|
match = alignTags(in, out, func(a, b reportTag) bool { return a.name == b.name })
|
|
}
|
|
var r HTMLRemovals
|
|
for i, tag := range in {
|
|
j := -1
|
|
if match != nil {
|
|
j = match[i]
|
|
}
|
|
if j < 0 {
|
|
r.addElement(tag.name)
|
|
continue
|
|
}
|
|
kept := out[j]
|
|
seen := make(map[string]bool, len(tag.raw))
|
|
for _, name := range tag.raw {
|
|
if seen[name] {
|
|
r.addAttribute(name) // a repeat the tokenizer dropped
|
|
continue
|
|
}
|
|
seen[name] = true
|
|
val, ok := kept.attrs[name]
|
|
if !ok || val != tag.attrs[name] {
|
|
r.addAttribute(name)
|
|
}
|
|
}
|
|
}
|
|
if inComments > outComments {
|
|
r.Comments = inComments - outComments
|
|
}
|
|
return r
|
|
}
|
|
|
|
// alignTags matches every tag of out to a tag of in, in order, with the
|
|
// first input tag for which same holds. The result maps each input index to
|
|
// its output index or -1. It is nil when an output tag stays unmatched.
|
|
func alignTags(in, out []reportTag, same func(in, out reportTag) bool) []int {
|
|
match := make([]int, len(in))
|
|
j := 0
|
|
for i := range in {
|
|
match[i] = -1
|
|
if j < len(out) && same(in[i], out[j]) {
|
|
match[i] = j
|
|
j++
|
|
}
|
|
}
|
|
if j < len(out) {
|
|
return nil
|
|
}
|
|
return match
|
|
}
|
|
|
|
// FormatTranslation sets attrs["content_html"] for one locale's translated
|
|
// post attributes, so a translated content_html is stored under the same
|
|
// policy as the host column. When attrs["content"] is not blank, content_html
|
|
// is rendered from it with FormatHTML and a supplied content_html is
|
|
// replaced, as the host column is always derived from content. Otherwise a
|
|
// supplied, non-null content_html is sanitized with SanitizeHTML; a value
|
|
// that is not a string is sanitized as its text form. attrs without content
|
|
// or content_html is left unchanged.
|
|
func FormatTranslation(attrs map[string]any) error {
|
|
if attrs == nil {
|
|
return nil
|
|
}
|
|
if content := textOf(attrs["content"]); strings.TrimSpace(content) != "" {
|
|
html, err := FormatHTML(content)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
attrs["content_html"] = html
|
|
return nil
|
|
}
|
|
if v, has := attrs["content_html"]; has && v != nil {
|
|
attrs["content_html"] = SanitizeHTML(textOf(v))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func textOf(v any) string {
|
|
switch t := v.(type) {
|
|
case nil:
|
|
return ""
|
|
case string:
|
|
return t
|
|
default:
|
|
return fmt.Sprint(t)
|
|
}
|
|
}
|
|
|
|
// CheckHTML reports whether the FormatHTML allow-list would remove or
|
|
// change any element, attribute or text in doc: it returns an error when it
|
|
// would, nil when the policy keeps doc whole. Since Phase 16.2 no admin or
|
|
// API write path refuses with it (admin saves strip and report through
|
|
// SanitizeHTMLReport, D-10 amended); it is the predicate counterpart of the
|
|
// site layer's isSafeJournalHtml, and the policy cross-check tests use it.
|
|
func CheckHTML(doc string) error {
|
|
if _, changed := sanitizeHTML(doc); changed {
|
|
return errors.New("journal: HTML contains elements, attributes or URLs that are not allowed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// sanitizeHTML returns doc unchanged when htmlPolicy keeps all of it, and the
|
|
// policy's output with changed set otherwise. Keeping all of it means doc and
|
|
// the policy output tokenize to the same HTML tokens, so an output that only
|
|
// re-escapes text or attribute values is not a change. A tag that repeats an
|
|
// attribute is always a change: the tokenizer keeps only the first of them,
|
|
// so the comparison would not see the others in the stored bytes.
|
|
func sanitizeHTML(doc string) (string, bool) {
|
|
clean := htmlPolicy.Sanitize(doc)
|
|
if sameTokens(doc, clean) {
|
|
return doc, false
|
|
}
|
|
return clean, true
|
|
}
|
|
|
|
// sameTokens reports whether a and b tokenize to the same sequence of HTML
|
|
// tokens: type, tag name or decoded text, and the set of decoded attributes.
|
|
// It is false when either repeats an attribute in a tag.
|
|
func sameTokens(a, b string) bool {
|
|
ta, oka := htmlTokens(a)
|
|
if !oka {
|
|
return false
|
|
}
|
|
tb, okb := htmlTokens(b)
|
|
return okb && slices.Equal(ta, tb)
|
|
}
|
|
|
|
// htmlTokens tokenizes doc. It is not ok when the tokenizer stops on an
|
|
// error other than EOF, or when a start tag's raw attributes differ from the
|
|
// attributes the tokenizer kept, which happens when a name repeats.
|
|
func htmlTokens(doc string) ([]string, bool) {
|
|
var out []string
|
|
z := xhtml.NewTokenizer(strings.NewReader(doc))
|
|
for {
|
|
tt := z.Next()
|
|
if tt == xhtml.ErrorToken {
|
|
return out, errors.Is(z.Err(), io.EOF)
|
|
}
|
|
var raw []string
|
|
if tt == xhtml.StartTagToken || tt == xhtml.SelfClosingTagToken {
|
|
raw = rawAttrNames(z.Raw())
|
|
}
|
|
t := z.Token()
|
|
if tt == xhtml.StartTagToken || tt == xhtml.SelfClosingTagToken {
|
|
if len(raw) != len(t.Attr) {
|
|
return out, false
|
|
}
|
|
for i, a := range t.Attr {
|
|
if raw[i] != a.Key {
|
|
return out, false
|
|
}
|
|
}
|
|
}
|
|
attrs := make([]string, 0, len(t.Attr))
|
|
for _, a := range t.Attr {
|
|
attrs = append(attrs, a.Namespace+":"+a.Key+"="+a.Val)
|
|
}
|
|
slices.Sort(attrs)
|
|
out = append(out, fmt.Sprintf("%d|%s|%q", tt, t.Data, attrs))
|
|
}
|
|
}
|
|
|
|
// rawAttrNames returns the ASCII-lowercased name of every attribute in the
|
|
// raw start tag, repeats included, in source order. It scans the tag as
|
|
// golang.org/x/net/html's tokenizer does (readTagName, readTagAttrKey and
|
|
// readTagAttrVal), which drops every repeat of a name it has seen. A scan
|
|
// that disagrees with the tokenizer only makes htmlTokens report a change.
|
|
func rawAttrNames(raw []byte) []string {
|
|
i, n := 2, len(raw) // past '<' and the first byte of the tag name
|
|
skipSpace := func() {
|
|
for i < n && isHTMLSpace(raw[i]) {
|
|
i++
|
|
}
|
|
}
|
|
for i < n {
|
|
c := raw[i]
|
|
i++
|
|
if isHTMLSpace(c) {
|
|
break
|
|
}
|
|
if c == '/' || c == '>' {
|
|
i--
|
|
break
|
|
}
|
|
}
|
|
var names []string
|
|
skipSpace()
|
|
for i < n {
|
|
if raw[i] == '>' {
|
|
break
|
|
}
|
|
// Attribute name: an '=' that starts it is part of the name.
|
|
start, end := i, n
|
|
for i < n {
|
|
c := raw[i]
|
|
i++
|
|
if c == '=' && i == start+1 {
|
|
continue
|
|
}
|
|
if c == '=' || c == '/' || c == '>' || isHTMLSpace(c) {
|
|
i--
|
|
end = i
|
|
break
|
|
}
|
|
}
|
|
// Attribute value.
|
|
skipSpace()
|
|
if i < n {
|
|
c := raw[i]
|
|
i++
|
|
switch {
|
|
case c == '/':
|
|
case c != '=':
|
|
i--
|
|
default:
|
|
skipSpace()
|
|
if i < n {
|
|
q := raw[i]
|
|
i++
|
|
switch q {
|
|
case '>':
|
|
i--
|
|
case '"', '\'':
|
|
for i < n {
|
|
i++
|
|
if raw[i-1] == q {
|
|
break
|
|
}
|
|
}
|
|
default:
|
|
for i < n {
|
|
c := raw[i]
|
|
i++
|
|
if isHTMLSpace(c) {
|
|
break
|
|
}
|
|
if c == '>' {
|
|
i--
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if end > start {
|
|
names = append(names, asciiLower(raw[start:end]))
|
|
}
|
|
skipSpace()
|
|
}
|
|
return names
|
|
}
|
|
|
|
func isHTMLSpace(c byte) bool {
|
|
return c == ' ' || c == '\n' || c == '\r' || c == '\t' || c == '\f'
|
|
}
|
|
|
|
func asciiLower(b []byte) string {
|
|
out := []byte(string(b))
|
|
for i, c := range out {
|
|
if 'A' <= c && c <= 'Z' {
|
|
out[i] = c + 'a' - 'A'
|
|
}
|
|
}
|
|
return string(out)
|
|
}
|