sm-journal-plugin/classes/format_html.go

package classes

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"regexp"
	"slices"
	"strings"

	"github.com/microcosm-cc/bluemonday"
	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/extension"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/renderer/html"
	xhtml "golang.org/x/net/html"
)

var (
	markdownEngine = goldmark.New(
		goldmark.WithExtensions(extension.Footnote, extension.Table),
		goldmark.WithParserOptions(parser.WithAttribute()),
		// Winter's Markdown::parse passes raw HTML through, so imported HTML
		// bodies round-trip (D-03, PHP parity); htmlPolicy then sanitizes the
		// rendered result on every write path.
		goldmark.WithRendererOptions(html.WithUnsafe()),
	)

	// htmlPolicy is the allow-list for stored journal HTML.
	htmlPolicy = newHTMLPolicy()
)

// newHTMLPolicy extends bluemonday's UGC policy only with what goldmark
// renders and what the imported posts and categories use.
func newHTMLPolicy() *bluemonday.Policy {
	p := bluemonday.UGCPolicy()
	// Winter neither adds nor requires rel="nofollow", so links keep their
	// stored form.
	p.RequireNoFollowOnLinks(false)
	// Imported author footers: <footer class=... itemscope itemtype=...>.
	p.AllowElements("footer")
	p.AllowAttrs("class").Matching(bluemonday.SpaceSeparatedTokens).Globally()
	p.AllowAttrs("itemscope").Matching(regexp.MustCompile(`^(|itemscope)$`)).Globally()
	p.AllowAttrs("itemtype").Matching(regexp.MustCompile(`^https?://schema\.org/[A-Za-z]+$`)).Globally()
	p.AllowAttrs("itemprop").Matching(bluemonday.SpaceSeparatedTokens).Globally()
	// Inline styles in imported footers and goldmark's table cell alignment.
	// bluemonday validates each property value.
	p.AllowStyles("color", "font-size", "font-weight", "text-align").Globally()
	// goldmark's footnote markup.
	p.AllowAttrs("role").Matching(regexp.MustCompile(`^doc-(noteref|endnotes|backlink)$`)).OnElements("a", "div")
	// Pasted chat transcripts in imported category bodies: dir="auto" (the
	// UGC policy allows only ltr and rtl) and two data attributes.
	p.AllowAttrs("dir").Matching(regexp.MustCompile(`(?i)^(ltr|rtl|auto)$`)).Globally()
	p.AllowAttrs("data-message-author-role", "data-message-id").
		Matching(regexp.MustCompile(`^[A-Za-z0-9_-]+$`)).OnElements("div")
	return p
}

// FormatHTML converts journal post markdown to stored HTML using goldmark
// footnote, table, and attribute extensions. Raw HTML in the source passes
// through the renderer, as it does in Winter's Markdown::parse. The result is
// then sanitized with a bluemonday allow-list that strips, as Winter's
// Html::clean does, what it does not allow: script, style, iframe, object,
// embed, form, base and meta elements, event handler attributes, and URLs
// other than http, https, mailto and relative ones. HTML that the policy
// keeps whole is returned byte for byte, so imported bodies round-trip. It
// does not call cabana.RenderMarkdown, whose admin preview stays without
// unsafe HTML.
func FormatHTML(src string) (string, error) {
	var buf bytes.Buffer
	if err := markdownEngine.Convert([]byte(strings.TrimSpace(src)), &buf); err != nil {
		return "", fmt.Errorf("journal: markdown: %w", err)
	}
	return SanitizeHTML(buf.String()), nil
}

// SanitizeHTML filters doc, which is HTML and not markdown, through the
// FormatHTML allow-list and strips what the policy does not allow. HTML the
// policy keeps whole is returned byte for byte. It is the gate for
// content_html supplied without markdown, such as a translation that carries
// only content_html.
func SanitizeHTML(doc string) string {
	out, _ := sanitizeHTML(doc)
	return out
}

// FormatTranslation sets attrs["content_html"] for one locale's translated
// post attributes, so a translated content_html is stored under the same
// policy as the host column. When attrs["content"] is not blank, content_html
// is rendered from it with FormatHTML and a supplied content_html is
// replaced, as the host column is always derived from content. Otherwise a
// supplied, non-null content_html is sanitized with SanitizeHTML; a value
// that is not a string is sanitized as its text form. attrs without content
// or content_html is left unchanged.
func FormatTranslation(attrs map[string]any) error {
	if attrs == nil {
		return nil
	}
	if content := textOf(attrs["content"]); strings.TrimSpace(content) != "" {
		html, err := FormatHTML(content)
		if err != nil {
			return err
		}
		attrs["content_html"] = html
		return nil
	}
	if v, has := attrs["content_html"]; has && v != nil {
		attrs["content_html"] = SanitizeHTML(textOf(v))
	}
	return nil
}

func textOf(v any) string {
	switch t := v.(type) {
	case nil:
		return ""
	case string:
		return t
	default:
		return fmt.Sprint(t)
	}
}

// CheckHTML gates HTML that is stored as is, such as category content. It
// uses the FormatHTML allow-list but refuses instead of stripping: doc is
// refused when the policy would remove or change any element, attribute or
// text in it.
func CheckHTML(doc string) error {
	if _, changed := sanitizeHTML(doc); changed {
		return errors.New("journal: HTML contains elements, attributes or URLs that are not allowed")
	}
	return nil
}

// sanitizeHTML returns doc unchanged when htmlPolicy keeps all of it, and the
// policy's output with changed set otherwise. Keeping all of it means doc and
// the policy output tokenize to the same HTML tokens, so an output that only
// re-escapes text or attribute values is not a change. A tag that repeats an
// attribute is always a change: the tokenizer keeps only the first of them,
// so the comparison would not see the others in the stored bytes.
func sanitizeHTML(doc string) (string, bool) {
	clean := htmlPolicy.Sanitize(doc)
	if sameTokens(doc, clean) {
		return doc, false
	}
	return clean, true
}

// sameTokens reports whether a and b tokenize to the same sequence of HTML
// tokens: type, tag name or decoded text, and the set of decoded attributes.
// It is false when either repeats an attribute in a tag.
func sameTokens(a, b string) bool {
	ta, oka := htmlTokens(a)
	if !oka {
		return false
	}
	tb, okb := htmlTokens(b)
	return okb && slices.Equal(ta, tb)
}

// htmlTokens tokenizes doc. It is not ok when the tokenizer stops on an
// error other than EOF, or when a start tag's raw attributes differ from the
// attributes the tokenizer kept, which happens when a name repeats.
func htmlTokens(doc string) ([]string, bool) {
	var out []string
	z := xhtml.NewTokenizer(strings.NewReader(doc))
	for {
		tt := z.Next()
		if tt == xhtml.ErrorToken {
			return out, errors.Is(z.Err(), io.EOF)
		}
		var raw []string
		if tt == xhtml.StartTagToken || tt == xhtml.SelfClosingTagToken {
			raw = rawAttrNames(z.Raw())
		}
		t := z.Token()
		if tt == xhtml.StartTagToken || tt == xhtml.SelfClosingTagToken {
			if len(raw) != len(t.Attr) {
				return out, false
			}
			for i, a := range t.Attr {
				if raw[i] != a.Key {
					return out, false
				}
			}
		}
		attrs := make([]string, 0, len(t.Attr))
		for _, a := range t.Attr {
			attrs = append(attrs, a.Namespace+":"+a.Key+"="+a.Val)
		}
		slices.Sort(attrs)
		out = append(out, fmt.Sprintf("%d|%s|%q", tt, t.Data, attrs))
	}
}

// rawAttrNames returns the ASCII-lowercased name of every attribute in the
// raw start tag, repeats included, in source order. It scans the tag as
// golang.org/x/net/html's tokenizer does (readTagName, readTagAttrKey and
// readTagAttrVal), which drops every repeat of a name it has seen. A scan
// that disagrees with the tokenizer only makes htmlTokens report a change.
func rawAttrNames(raw []byte) []string {
	i, n := 2, len(raw) // past '<' and the first byte of the tag name
	skipSpace := func() {
		for i < n && isHTMLSpace(raw[i]) {
			i++
		}
	}
	for i < n {
		c := raw[i]
		i++
		if isHTMLSpace(c) {
			break
		}
		if c == '/' || c == '>' {
			i--
			break
		}
	}
	var names []string
	skipSpace()
	for i < n {
		if raw[i] == '>' {
			break
		}
		// Attribute name: an '=' that starts it is part of the name.
		start, end := i, n
		for i < n {
			c := raw[i]
			i++
			if c == '=' && i == start+1 {
				continue
			}
			if c == '=' || c == '/' || c == '>' || isHTMLSpace(c) {
				i--
				end = i
				break
			}
		}
		// Attribute value.
		skipSpace()
		if i < n {
			c := raw[i]
			i++
			switch {
			case c == '/':
			case c != '=':
				i--
			default:
				skipSpace()
				if i < n {
					q := raw[i]
					i++
					switch q {
					case '>':
						i--
					case '"', '\'':
						for i < n {
							i++
							if raw[i-1] == q {
								break
							}
						}
					default:
						for i < n {
							c := raw[i]
							i++
							if isHTMLSpace(c) {
								break
							}
							if c == '>' {
								i--
								break
							}
						}
					}
				}
			}
		}
		if end > start {
			names = append(names, asciiLower(raw[start:end]))
		}
		skipSpace()
	}
	return names
}

func isHTMLSpace(c byte) bool {
	return c == ' ' || c == '\n' || c == '\r' || c == '\t' || c == '\f'
}

func asciiLower(b []byte) string {
	out := []byte(string(b))
	for i, c := range out {
		if 'A' <= c && c <= 'Z' {
			out[i] = c + 'a' - 'A'
		}
	}
	return string(out)
}