Jakub Zych
5af746f31f
LooksLikeHTML decided by the first character alone, so a markdown post opening with a hero <img>, a <div> or a comment was served as raw markdown and its first edit stored ## and ** markers as HTML in both columns. Load now serves such a value as HTML only when content_html is empty or missing, or holds that same HTML (the value or SanitizeHTML of it, white space at the ends ignored); otherwise it serves content_html.
427 lines
14 KiB
Go
427 lines
14 KiB
Go
package classes
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"path"
|
|
"regexp"
|
|
"strings"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/backpack"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"gocloud.dev/blob"
|
|
xhtml "golang.org/x/net/html"
|
|
)
|
|
|
|
// BlogEditorElement is the custom element of the blogeditor and mlblogeditor
|
|
// field types, defined by assets/dist/blogeditor.js.
|
|
const BlogEditorElement = "golem15-journal-blogeditor"
|
|
|
|
// Phrase keys of the Blog Editor field types.
|
|
const (
|
|
editorContentInvalid = "golem15.journal::lang.editor.content_invalid"
|
|
editorImageInvalid = "golem15.journal::lang.editor.image_invalid"
|
|
editorRemovedElements = "golem15.journal::lang.editor.removed_elements"
|
|
editorRemovedAttributes = "golem15.journal::lang.editor.removed_attributes"
|
|
editorRemovedComments = "golem15.journal::lang.editor.removed_comments"
|
|
editorMarkupChanged = "golem15.journal::lang.editor.markup_changed"
|
|
)
|
|
|
|
// blogListValueRunes is how many runes of text a list column shows.
|
|
const blogListValueRunes = 120
|
|
|
|
// blogUploadFolder is the upload folder of a field without uploadFolder.
|
|
const blogUploadFolder = "editor"
|
|
|
|
var (
|
|
blogHTMLColumnPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
|
blogUploadExtPattern = regexp.MustCompile(`^\.[a-z0-9]+$`)
|
|
// blogImageTypes maps an upload extension to the content type its first
|
|
// bytes must sniff as.
|
|
blogImageTypes = map[string]string{
|
|
"jpg": "image/jpeg",
|
|
"jpeg": "image/jpeg",
|
|
"png": "image/png",
|
|
"gif": "image/gif",
|
|
"webp": "image/webp",
|
|
}
|
|
// blogImageExtensions is the extension given to a name without one.
|
|
blogImageExtensions = map[string]string{
|
|
"image/jpeg": "jpg",
|
|
"image/png": "png",
|
|
"image/gif": "gif",
|
|
"image/webp": "webp",
|
|
}
|
|
)
|
|
|
|
// BlogEditorTypes returns the Blog Editor admin field types: blogeditor for
|
|
// a column that holds one HTML string, and mlblogeditor for a translatable
|
|
// one. Both render BlogEditorElement. Decode stores what the HTML policy
|
|
// keeps (SanitizeHTMLReport) and reports what it removed as notices; it
|
|
// never refuses a string because of its markup. Load serves HTML content as
|
|
// it is and a markdown value as the rendered HTML of the htmlColumn option,
|
|
// and a blank value as the htmlColumn attribute when that holds a body; a
|
|
// value that starts with '<' counts as HTML only when the htmlColumn
|
|
// attribute is empty or holds that same HTML (loadBlogHTML). Upload stores an image the editor
|
|
// inserts through StoreMedia, in the media bucket the app publishes. app is
|
|
// asked for the booted application at request time; it may be nil in tests,
|
|
// where Upload then fails.
|
|
//
|
|
// Options: htmlColumn (mlblogeditor only) names the translatable attribute
|
|
// that holds the rendered HTML of a markdown value; uploadFolder is the
|
|
// folder under /journal/ that editor image uploads are stored in ("editor"
|
|
// when unset).
|
|
func BlogEditorTypes(app func() *backpack.App) []pact.FormFieldType {
|
|
upload := uploadBlogImage(app)
|
|
return []pact.FormFieldType{
|
|
{
|
|
Name: "blogeditor",
|
|
Element: BlogEditorElement,
|
|
Scripts: []string{"assets/dist/blogeditor.js"},
|
|
Styles: []string{"assets/dist/blogeditor.css"},
|
|
Options: []string{"uploadFolder"},
|
|
CheckOptions: checkBlogOptions,
|
|
Decode: decodeBlogHTML,
|
|
Load: loadBlogHTML,
|
|
ListValue: blogListValue,
|
|
Upload: upload,
|
|
},
|
|
{
|
|
Name: "mlblogeditor",
|
|
Element: BlogEditorElement,
|
|
Translatable: true,
|
|
Scripts: []string{"assets/dist/blogeditor.js"},
|
|
Styles: []string{"assets/dist/blogeditor.css"},
|
|
Options: []string{"htmlColumn", "uploadFolder"},
|
|
CheckOptions: checkBlogOptions,
|
|
Decode: decodeBlogHTML,
|
|
Load: loadBlogHTML,
|
|
ListValue: blogListValue,
|
|
Upload: upload,
|
|
},
|
|
}
|
|
}
|
|
|
|
// LooksLikeHTML reports whether s may be stored HTML rather than markdown:
|
|
// its first character that is not white space is '<' (an HTML comment
|
|
// included). Phase 16.2 assumption A1. Markdown can start with a tag too,
|
|
// so Load also compares the value with its htmlColumn attribute before it
|
|
// serves the value as HTML.
|
|
func LooksLikeHTML(s string) bool {
|
|
t := strings.TrimLeftFunc(s, unicode.IsSpace)
|
|
return strings.HasPrefix(t, "<")
|
|
}
|
|
|
|
// checkBlogOptions checks a Blog Editor field's options at boot.
|
|
func checkBlogOptions(options map[string]any) error {
|
|
if raw, ok := options["htmlColumn"]; ok {
|
|
column, isString := raw.(string)
|
|
if !isString || !blogHTMLColumnPattern.MatchString(column) {
|
|
return fmt.Errorf("htmlColumn must be an attribute name, got %v", raw)
|
|
}
|
|
}
|
|
if raw, ok := options["uploadFolder"]; ok {
|
|
folder, isString := raw.(string)
|
|
if !isString {
|
|
return fmt.Errorf("uploadFolder must be a string, got %v", raw)
|
|
}
|
|
if err := checkUploadFolder(folder); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// checkUploadFolder applies the public media upload route's folder rule.
|
|
func checkUploadFolder(folder string) error {
|
|
if !validMediaFolder(folder) {
|
|
return errors.New("uploadFolder must be at most 100 letters, digits, '_', '-' or '/' without '..'")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// decodeBlogHTML stores the HTML policy's output for a submitted value: the
|
|
// exact submitted bytes with no notice when the policy keeps everything,
|
|
// otherwise the stripped HTML with notices naming what was removed. nil
|
|
// decodes to the empty string; any other value that is not a string is
|
|
// refused.
|
|
func decodeBlogHTML(_ context.Context, in pact.FormFieldValue) (pact.FormFieldDecoded, error) {
|
|
var text string
|
|
switch v := in.Value.(type) {
|
|
case nil:
|
|
return pact.FormFieldDecoded{Value: ""}, nil
|
|
case string:
|
|
text = v
|
|
default:
|
|
return pact.FormFieldDecoded{}, &pact.FormFieldError{Messages: []string{editorContentInvalid}}
|
|
}
|
|
clean, report := SanitizeHTMLReport(text)
|
|
return pact.FormFieldDecoded{Value: clean, Notices: removalNotices(report)}, nil
|
|
}
|
|
|
|
// removalNotices turns a removal report into admin notices: removed
|
|
// elements, removed attributes, removed comments and other changes, in that
|
|
// order and only when non-zero.
|
|
func removalNotices(r HTMLRemovals) []pact.FormFieldNotice {
|
|
var out []pact.FormFieldNotice
|
|
if r.ElementCount > 0 {
|
|
out = append(out, pact.FormFieldNotice{
|
|
Message: editorRemovedElements,
|
|
Params: map[string]string{"names": removalNames(r.Elements, r.moreElements)},
|
|
Count: r.ElementCount,
|
|
})
|
|
}
|
|
if r.AttributeCount > 0 {
|
|
out = append(out, pact.FormFieldNotice{
|
|
Message: editorRemovedAttributes,
|
|
Params: map[string]string{"names": removalNames(r.Attributes, r.moreAttributes)},
|
|
Count: r.AttributeCount,
|
|
})
|
|
}
|
|
if r.Comments > 0 {
|
|
out = append(out, pact.FormFieldNotice{Message: editorRemovedComments, Count: r.Comments})
|
|
}
|
|
if r.Other {
|
|
out = append(out, pact.FormFieldNotice{Message: editorMarkupChanged})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func removalNames(names []string, cut bool) string {
|
|
joined := strings.Join(names, ", ")
|
|
if cut {
|
|
joined += ", …"
|
|
}
|
|
return joined
|
|
}
|
|
|
|
// loadBlogHTML serves a stored value to the editor (D-07). Without the
|
|
// htmlColumn option every value is served as stored. With it:
|
|
//
|
|
// - A blank value is served as the htmlColumn attribute in the same locale
|
|
// when that is not empty, so a locale that carries only content_html (an
|
|
// API-written translation, FormatTranslation) opens with the body the
|
|
// site renders instead of an empty editor whose first edit would replace
|
|
// it; it is served as stored otherwise.
|
|
// - A value that looks like HTML (LooksLikeHTML) is served as it is when
|
|
// the htmlColumn attribute is empty or missing, or holds the same HTML:
|
|
// the value itself or the policy's output for it (SanitizeHTML), white
|
|
// space at either end ignored. Otherwise the value is markdown that
|
|
// merely starts with a tag (a hero <img>, a leading <div> or comment),
|
|
// and it is served as the htmlColumn attribute, so the first edit does
|
|
// not store raw markdown as HTML.
|
|
// - Any other value is markdown, served as the htmlColumn attribute, or as
|
|
// FormatHTML of the value when that is empty or cannot be read (the
|
|
// value itself when rendering fails).
|
|
func loadBlogHTML(_ context.Context, in pact.FormFieldValue) (any, error) {
|
|
text := storedText(in.Value)
|
|
column, _ := in.Options["htmlColumn"].(string)
|
|
if column == "" {
|
|
return text, nil
|
|
}
|
|
html := ""
|
|
if in.Translated != nil {
|
|
value, found, err := in.Translated(column)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if found {
|
|
html = value
|
|
}
|
|
}
|
|
switch {
|
|
case strings.TrimSpace(text) == "":
|
|
if html != "" {
|
|
return html, nil
|
|
}
|
|
return text, nil
|
|
case LooksLikeHTML(text):
|
|
if html == "" || sameStoredHTML(text, html) {
|
|
return text, nil
|
|
}
|
|
return html, nil
|
|
case html != "":
|
|
return html, nil
|
|
}
|
|
rendered, err := FormatHTML(text)
|
|
if err != nil {
|
|
return text, nil
|
|
}
|
|
return rendered, nil
|
|
}
|
|
|
|
// sameStoredHTML reports whether rendered, a stored htmlColumn value, is the
|
|
// HTML text itself: equal to text or to the policy's output for it, with
|
|
// white space at either end ignored. It is false when rendered came from
|
|
// rendering text as markdown in a way that changed it.
|
|
func sameStoredHTML(text, rendered string) bool {
|
|
rendered = strings.TrimSpace(rendered)
|
|
return rendered == strings.TrimSpace(text) || rendered == strings.TrimSpace(SanitizeHTML(text))
|
|
}
|
|
|
|
// storedText is a stored column value as text: a string, a *string (nil is
|
|
// empty) or nil.
|
|
func storedText(v any) string {
|
|
switch t := v.(type) {
|
|
case nil:
|
|
return ""
|
|
case string:
|
|
return t
|
|
case *string:
|
|
if t == nil {
|
|
return ""
|
|
}
|
|
return *t
|
|
default:
|
|
return fmt.Sprint(t)
|
|
}
|
|
}
|
|
|
|
// inlineListTags are the elements that do not separate words in a list
|
|
// value's text.
|
|
var inlineListTags = map[string]bool{
|
|
"a": true, "abbr": true, "b": true, "cite": true, "code": true, "del": true,
|
|
"em": true, "i": true, "ins": true, "kbd": true, "mark": true, "q": true,
|
|
"s": true, "small": true, "span": true, "strong": true, "sub": true,
|
|
"sup": true, "u": true,
|
|
}
|
|
|
|
// blogListValue is the text a list column of a Blog Editor type shows: the
|
|
// value without tags, white space collapsed, cut to 120 runes with an
|
|
// ellipsis.
|
|
func blogListValue(_ context.Context, value any) string {
|
|
var b strings.Builder
|
|
z := xhtml.NewTokenizer(strings.NewReader(storedText(value)))
|
|
skip := 0 // inside script or style
|
|
for {
|
|
tt := z.Next()
|
|
if tt == xhtml.ErrorToken {
|
|
break
|
|
}
|
|
switch tt {
|
|
case xhtml.TextToken:
|
|
if skip == 0 {
|
|
b.WriteString(z.Token().Data)
|
|
}
|
|
case xhtml.StartTagToken, xhtml.EndTagToken, xhtml.SelfClosingTagToken:
|
|
name, _ := z.TagName()
|
|
switch string(name) {
|
|
case "script", "style":
|
|
if tt == xhtml.StartTagToken {
|
|
skip++
|
|
} else if tt == xhtml.EndTagToken && skip > 0 {
|
|
skip--
|
|
}
|
|
}
|
|
if !inlineListTags[string(name)] {
|
|
b.WriteByte(' ')
|
|
}
|
|
}
|
|
}
|
|
text := strings.Join(strings.Fields(b.String()), " ")
|
|
if utf8.RuneCountInString(text) <= blogListValueRunes {
|
|
return text
|
|
}
|
|
runes := []rune(text)
|
|
return strings.TrimRightFunc(string(runes[:blogListValueRunes]), unicode.IsSpace) + "…"
|
|
}
|
|
|
|
// uploadBlogImage is the Blog Editor's Upload: it stores an image the
|
|
// editor inserts through StoreMedia, in the field's uploadFolder, and
|
|
// returns its URL. The file's first bytes must sniff as a JPEG, PNG, GIF or
|
|
// WebP image of the type its extension names (a name without an extension
|
|
// gets the sniffed one); anything else is refused with
|
|
// golem15.journal::lang.editor.image_invalid. The stored name is
|
|
// safeUploadName's. A MediaError is refused with its message; a missing
|
|
// application or bucket is an opaque error.
|
|
func uploadBlogImage(app func() *backpack.App) func(context.Context, pact.FormFieldUpload) (pact.FormFieldUploadResult, error) {
|
|
return func(ctx context.Context, in pact.FormFieldUpload) (pact.FormFieldUploadResult, error) {
|
|
var a *backpack.App
|
|
if app != nil {
|
|
a = app()
|
|
}
|
|
if a == nil {
|
|
return pact.FormFieldUploadResult{}, errors.New("journal: editor upload: application is not configured")
|
|
}
|
|
bucket, ok := a.Lookup[*blob.Bucket]()
|
|
if !ok || bucket == nil {
|
|
return pact.FormFieldUploadResult{}, errors.New("journal: editor upload: media bucket is not configured")
|
|
}
|
|
if in.Body == nil {
|
|
return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}}
|
|
}
|
|
head := make([]byte, 512)
|
|
n, err := io.ReadFull(in.Body, head)
|
|
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
|
|
return pact.FormFieldUploadResult{}, fmt.Errorf("journal: editor upload: read: %w", err)
|
|
}
|
|
head = head[:n]
|
|
sniffed := http.DetectContentType(head)
|
|
ext, known := blogImageExtensions[sniffed]
|
|
if !known {
|
|
return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}}
|
|
}
|
|
name := safeUploadName(in.Filename)
|
|
if named := strings.TrimPrefix(path.Ext(name), "."); named == "" {
|
|
name += "." + ext
|
|
} else if want, allowed := blogImageTypes[named]; allowed && want != sniffed {
|
|
return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}}
|
|
}
|
|
folder, _ := in.Options["uploadFolder"].(string)
|
|
if folder == "" {
|
|
folder = blogUploadFolder
|
|
}
|
|
stored, err := StoreMedia(ctx, bucket, folder, name, io.MultiReader(bytes.NewReader(head), in.Body))
|
|
if err != nil {
|
|
var refused *MediaError
|
|
if errors.As(err, &refused) && refused != nil {
|
|
return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{refused.Message}}
|
|
}
|
|
return pact.FormFieldUploadResult{}, err
|
|
}
|
|
return pact.FormFieldUploadResult{URL: stored.URL}, nil
|
|
}
|
|
}
|
|
|
|
// safeUploadName is the stored name of an editor upload: the base name,
|
|
// lowercased, every character outside a-z, 0-9, '.', '_' and '-' turned
|
|
// into '-' with runs collapsed and no leading or trailing '-' or '.', and
|
|
// the extension kept. An empty name becomes "image". The result needs no
|
|
// escaping in an src attribute.
|
|
func safeUploadName(filename string) string {
|
|
base := path.Base(strings.ReplaceAll(filename, `\`, "/"))
|
|
ext := strings.ToLower(path.Ext(base))
|
|
if !blogUploadExtPattern.MatchString(ext) {
|
|
ext = ""
|
|
}
|
|
stem := strings.ToLower(strings.TrimSuffix(base, path.Ext(base)))
|
|
if ext == "" {
|
|
stem = strings.ToLower(base)
|
|
}
|
|
var b strings.Builder
|
|
dash := false
|
|
for _, r := range stem {
|
|
ok := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-'
|
|
if !ok || r == '-' {
|
|
if !dash {
|
|
b.WriteByte('-')
|
|
dash = true
|
|
}
|
|
continue
|
|
}
|
|
b.WriteRune(r)
|
|
dash = false
|
|
}
|
|
stem = strings.Trim(b.String(), "-.")
|
|
if stem == "" {
|
|
stem = "image"
|
|
}
|
|
return stem + ext
|
|
}
|