sm-journal-plugin/classes/blog_editor.go

package classes

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"path"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"

	"git.golem15.com/golem15/sm-journal-plugin/models"
	"git.golem15.com/golem15/summercms/modules/backpack"
	"git.golem15.com/golem15/summercms/modules/compass"
	"git.golem15.com/golem15/summercms/modules/pact"
	"gocloud.dev/blob"
	xhtml "golang.org/x/net/html"
	"gorm.io/gorm"
)

// BlogEditorElement is the custom element of the blogeditor and mlblogeditor
// field types, defined by assets/dist/blogeditor.js.
const BlogEditorElement = "golem15-journal-blogeditor"

// Phrase keys of the Blog Editor field types.
const (
	editorContentInvalid    = "golem15.journal::lang.editor.content_invalid"
	editorImageInvalid      = "golem15.journal::lang.editor.image_invalid"
	editorRemovedElements   = "golem15.journal::lang.editor.removed_elements"
	editorRemovedAttributes = "golem15.journal::lang.editor.removed_attributes"
	editorRemovedComments   = "golem15.journal::lang.editor.removed_comments"
	editorMarkupChanged     = "golem15.journal::lang.editor.markup_changed"
	// EditorH1Converted is the notice of a save that turned H1 headings
	// into H2 (Count is the number of headings).
	EditorH1Converted = "golem15.journal::lang.editor.h1_converted"
	// EditorH1ConvertedCategory is EditorH1Converted for a category's
	// pillar text, which names the category name instead of the post title.
	EditorH1ConvertedCategory = "golem15.journal::lang.editor.h1_converted_category"
)

// blogListValueRunes is how many runes of text a list column shows.
const blogListValueRunes = 120

// blogUploadFolder is the upload folder of a field without uploadFolder.
const blogUploadFolder = "editor"

var (
	blogHTMLColumnPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
	blogUploadExtPattern  = regexp.MustCompile(`^\.[a-z0-9]+$`)
	// blogImageTypes maps an upload extension to the content type its first
	// bytes must sniff as.
	blogImageTypes = map[string]string{
		"jpg":  "image/jpeg",
		"jpeg": "image/jpeg",
		"png":  "image/png",
		"gif":  "image/gif",
		"webp": "image/webp",
	}
	// blogImageExtensions is the extension given to a name without one.
	blogImageExtensions = map[string]string{
		"image/jpeg": "jpg",
		"image/png":  "png",
		"image/gif":  "gif",
		"image/webp": "webp",
	}
)

// BlogEditorTypes returns the Blog Editor admin field types: blogeditor for
// a column that holds one HTML string, and mlblogeditor for a translatable
// one. Both render BlogEditorElement. Decode stores what the HTML policy
// keeps (SanitizeHTMLReport) and reports what it removed as notices; it
// never refuses a string because of its markup. Load serves HTML content as
// it is and a markdown value as the rendered HTML of the htmlColumn option,
// and a blank value as the htmlColumn attribute when that holds a body; a
// value that starts with '<' counts as HTML only when the htmlColumn
// attribute is empty or holds that same HTML (loadBlogHTML). Upload stores an image the editor
// inserts through StoreMedia, in the media bucket the app publishes. app is
// asked for the booted application at request time; it may be nil in tests,
// where Upload then fails.
//
// Options: htmlColumn (mlblogeditor only) names the translatable attribute
// that holds the rendered HTML of a markdown value; uploadFolder is the
// folder under /journal/ that editor image uploads are stored in ("editor"
// when unset).
func BlogEditorTypes(app func() *backpack.App) []pact.FormFieldType {
	upload := uploadBlogImage(app)
	decode := decodeBlogHTMLFor(app)
	options := blogRequestOptions(app)
	return []pact.FormFieldType{
		{
			Name:           "blogeditor",
			Element:        BlogEditorElement,
			Scripts:        []string{"assets/dist/blogeditor.js"},
			Styles:         []string{"assets/dist/blogeditor.css"},
			Options:        []string{"uploadFolder"},
			CheckOptions:   checkBlogOptions,
			Decode:         decode,
			Load:           loadBlogHTML,
			ListValue:      blogListValue,
			Upload:         upload,
			RequestOptions: options,
		},
		{
			Name:           "mlblogeditor",
			Element:        BlogEditorElement,
			Translatable:   true,
			Scripts:        []string{"assets/dist/blogeditor.js"},
			Styles:         []string{"assets/dist/blogeditor.css"},
			Options:        []string{"htmlColumn", "uploadFolder"},
			CheckOptions:   checkBlogOptions,
			Decode:         decode,
			Load:           loadBlogHTML,
			ListValue:      blogListValue,
			Upload:         upload,
			RequestOptions: options,
		},
	}
}

// LooksLikeHTML reports whether s may be stored HTML rather than markdown:
// its first character that is not white space is '<' (an HTML comment
// included). Phase 16.2 assumption A1. Markdown can start with a tag too,
// so Load also compares the value with its htmlColumn attribute before it
// serves the value as HTML.
func LooksLikeHTML(s string) bool {
	t := strings.TrimLeftFunc(s, unicode.IsSpace)
	return strings.HasPrefix(t, "<")
}

// checkBlogOptions checks a Blog Editor field's options at boot.
func checkBlogOptions(options map[string]any) error {
	if raw, ok := options["htmlColumn"]; ok {
		column, isString := raw.(string)
		if !isString || !blogHTMLColumnPattern.MatchString(column) {
			return fmt.Errorf("htmlColumn must be an attribute name, got %v", raw)
		}
	}
	if raw, ok := options["uploadFolder"]; ok {
		folder, isString := raw.(string)
		if !isString {
			return fmt.Errorf("uploadFolder must be a string, got %v", raw)
		}
		if err := checkUploadFolder(folder); err != nil {
			return err
		}
	}
	return nil
}

// checkUploadFolder applies the public media upload route's folder rule.
func checkUploadFolder(folder string) error {
	if !validMediaFolder(folder) {
		return errors.New("uploadFolder must be at most 100 letters, digits, '_', '-' or '/' without '..'")
	}
	return nil
}

// decodeBlogHTMLFor is the Blog Editor types' Decode: decodeBlogHTML, then,
// while the H1 setting is on (EffectiveH1ToH2 on the save transaction and
// the booted application's config), ConvertH1 on the stored HTML with a
// notice counting the converted headings: EditorH1ConvertedCategory on a
// category's pillar text, EditorH1Converted otherwise. app may be nil
// or answer nil; the setting is then read from the database alone.
func decodeBlogHTMLFor(app func() *backpack.App) func(context.Context, pact.FormFieldValue) (pact.FormFieldDecoded, error) {
	return func(ctx context.Context, in pact.FormFieldValue) (pact.FormFieldDecoded, error) {
		out, err := decodeBlogHTML(ctx, in)
		if err != nil {
			return out, err
		}
		text, isString := out.Value.(string)
		if !isString || text == "" {
			return out, nil
		}
		if !EffectiveH1ToH2(ctx, in.DB, appConfig(app)) {
			return out, nil
		}
		converted, n := ConvertH1(text)
		if n == 0 {
			return out, nil
		}
		out.Value = converted
		out.Notices = append(out.Notices, pact.FormFieldNotice{Message: h1ConvertedNotice(in.Record), Count: n})
		return out, nil
	}
}

// h1ConvertedNotice is the conversion notice for the saved record: the
// category phrase for a category, the post phrase otherwise.
func h1ConvertedNotice(record any) string {
	switch record.(type) {
	case *models.Category, models.Category:
		return EditorH1ConvertedCategory
	}
	return EditorH1Converted
}

// blogRequestOptions is the Blog Editor types' RequestOptions (33-01
// P33-02, UI-SPEC C28): headingLevels, [2, 3, 4] while the H1 setting is on
// and [1, 2, 3, 4] while it is off (J1), and genericAnchors, the settings'
// generic link texts (J2, J3). Both are read per schema request, so a
// changed setting reaches the next form that opens. db may be nil; the
// defaults then apply. fields.yaml options with the same names win.
func blogRequestOptions(app func() *backpack.App) func(context.Context, *gorm.DB) (map[string]any, error) {
	return func(ctx context.Context, db *gorm.DB) (map[string]any, error) {
		levels := []int{2, 3, 4}
		if !EffectiveH1ToH2(ctx, db, appConfig(app)) {
			levels = []int{1, 2, 3, 4}
		}
		return map[string]any{
			"headingLevels":  levels,
			"genericAnchors": GenericAnchors(ctx, db),
		}, nil
	}
}

// appConfig is the config of the application app answers, nil when there
// is none.
func appConfig(app func() *backpack.App) *compass.Config {
	if app == nil {
		return nil
	}
	a := app()
	if a == nil {
		return nil
	}
	return a.Config
}

// decodeBlogHTML stores the HTML policy's output for a submitted value: the
// exact submitted bytes with no notice when the policy keeps everything,
// otherwise the stripped HTML with notices naming what was removed. nil
// decodes to the empty string; any other value that is not a string is
// refused.
func decodeBlogHTML(_ context.Context, in pact.FormFieldValue) (pact.FormFieldDecoded, error) {
	var text string
	switch v := in.Value.(type) {
	case nil:
		return pact.FormFieldDecoded{Value: ""}, nil
	case string:
		text = v
	default:
		return pact.FormFieldDecoded{}, &pact.FormFieldError{Messages: []string{editorContentInvalid}}
	}
	clean, report := SanitizeHTMLReport(text)
	return pact.FormFieldDecoded{Value: clean, Notices: removalNotices(report)}, nil
}

// removalNotices turns a removal report into admin notices: removed
// elements, removed attributes, removed comments and other changes, in that
// order and only when non-zero.
func removalNotices(r HTMLRemovals) []pact.FormFieldNotice {
	var out []pact.FormFieldNotice
	if r.ElementCount > 0 {
		out = append(out, pact.FormFieldNotice{
			Message: editorRemovedElements,
			Params:  map[string]string{"names": removalNames(r.Elements, r.moreElements)},
			Count:   r.ElementCount,
		})
	}
	if r.AttributeCount > 0 {
		out = append(out, pact.FormFieldNotice{
			Message: editorRemovedAttributes,
			Params:  map[string]string{"names": removalNames(r.Attributes, r.moreAttributes)},
			Count:   r.AttributeCount,
		})
	}
	if r.Comments > 0 {
		out = append(out, pact.FormFieldNotice{Message: editorRemovedComments, Count: r.Comments})
	}
	if r.Other {
		out = append(out, pact.FormFieldNotice{Message: editorMarkupChanged})
	}
	return out
}

func removalNames(names []string, cut bool) string {
	joined := strings.Join(names, ", ")
	if cut {
		joined += ", …"
	}
	return joined
}

// loadBlogHTML serves a stored value to the editor (D-07). Without the
// htmlColumn option every value is served as stored. With it:
//
//   - A blank value is served as the htmlColumn attribute in the same locale
//     when that is not empty, so a locale that carries only content_html (an
//     API-written translation, FormatTranslation) opens with the body the
//     site renders instead of an empty editor whose first edit would replace
//     it; it is served as stored otherwise.
//   - A value that looks like HTML (LooksLikeHTML) is served as it is when
//     the htmlColumn attribute is empty or missing, or holds the same HTML:
//     the value itself or the policy's output for it (SanitizeHTML), white
//     space at either end ignored. Otherwise the value is markdown that
//     merely starts with a tag (a hero <img>, a leading <div> or comment),
//     and it is served as the htmlColumn attribute, so the first edit does
//     not store raw markdown as HTML.
//   - Any other value is markdown, served as the htmlColumn attribute, or as
//     FormatHTML of the value when that is empty or cannot be read (the
//     value itself when rendering fails).
func loadBlogHTML(_ context.Context, in pact.FormFieldValue) (any, error) {
	text := storedText(in.Value)
	column, _ := in.Options["htmlColumn"].(string)
	if column == "" {
		return text, nil
	}
	html := ""
	if in.Translated != nil {
		value, found, err := in.Translated(column)
		if err != nil {
			return nil, err
		}
		if found {
			html = value
		}
	}
	switch {
	case strings.TrimSpace(text) == "":
		if html != "" {
			return html, nil
		}
		return text, nil
	case LooksLikeHTML(text):
		if html == "" || sameStoredHTML(text, html) {
			return text, nil
		}
		return html, nil
	case html != "":
		return html, nil
	}
	rendered, err := FormatHTML(text)
	if err != nil {
		return text, nil
	}
	return rendered, nil
}

// sameStoredHTML reports whether rendered, a stored htmlColumn value, is the
// HTML text itself: equal to text or to the policy's output for it, with
// white space at either end ignored. It is false when rendered came from
// rendering text as markdown in a way that changed it.
func sameStoredHTML(text, rendered string) bool {
	rendered = strings.TrimSpace(rendered)
	return rendered == strings.TrimSpace(text) || rendered == strings.TrimSpace(SanitizeHTML(text))
}

// storedText is a stored column value as text: a string, a *string (nil is
// empty) or nil.
func storedText(v any) string {
	switch t := v.(type) {
	case nil:
		return ""
	case string:
		return t
	case *string:
		if t == nil {
			return ""
		}
		return *t
	default:
		return fmt.Sprint(t)
	}
}

// inlineListTags are the elements that do not separate words in a list
// value's text.
var inlineListTags = map[string]bool{
	"a": true, "abbr": true, "b": true, "cite": true, "code": true, "del": true,
	"em": true, "i": true, "ins": true, "kbd": true, "mark": true, "q": true,
	"s": true, "small": true, "span": true, "strong": true, "sub": true,
	"sup": true, "u": true,
}

// blogListValue is the text a list column of a Blog Editor type shows: the
// value without tags, white space collapsed, cut to 120 runes with an
// ellipsis.
func blogListValue(_ context.Context, value any) string {
	var b strings.Builder
	z := xhtml.NewTokenizer(strings.NewReader(storedText(value)))
	skip := 0 // inside script or style
	for {
		tt := z.Next()
		if tt == xhtml.ErrorToken {
			break
		}
		switch tt {
		case xhtml.TextToken:
			if skip == 0 {
				b.WriteString(z.Token().Data)
			}
		case xhtml.StartTagToken, xhtml.EndTagToken, xhtml.SelfClosingTagToken:
			name, _ := z.TagName()
			switch string(name) {
			case "script", "style":
				if tt == xhtml.StartTagToken {
					skip++
				} else if tt == xhtml.EndTagToken && skip > 0 {
					skip--
				}
			}
			if !inlineListTags[string(name)] {
				b.WriteByte(' ')
			}
		}
	}
	text := strings.Join(strings.Fields(b.String()), " ")
	if utf8.RuneCountInString(text) <= blogListValueRunes {
		return text
	}
	runes := []rune(text)
	return strings.TrimRightFunc(string(runes[:blogListValueRunes]), unicode.IsSpace) + "…"
}

// uploadBlogImage is the Blog Editor's Upload: it stores an image the
// editor inserts through StoreMedia, in the field's uploadFolder, and
// returns its URL. The file's first bytes must sniff as a JPEG, PNG, GIF or
// WebP image of the type its extension names (a name without an extension
// gets the sniffed one); anything else is refused with
// golem15.journal::lang.editor.image_invalid. The stored name is
// safeUploadName's. A MediaError is refused with its message; a missing
// application or bucket is an opaque error.
func uploadBlogImage(app func() *backpack.App) func(context.Context, pact.FormFieldUpload) (pact.FormFieldUploadResult, error) {
	return func(ctx context.Context, in pact.FormFieldUpload) (pact.FormFieldUploadResult, error) {
		var a *backpack.App
		if app != nil {
			a = app()
		}
		if a == nil {
			return pact.FormFieldUploadResult{}, errors.New("journal: editor upload: application is not configured")
		}
		bucket, ok := a.Lookup[*blob.Bucket]()
		if !ok || bucket == nil {
			return pact.FormFieldUploadResult{}, errors.New("journal: editor upload: media bucket is not configured")
		}
		if in.Body == nil {
			return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}}
		}
		head := make([]byte, 512)
		n, err := io.ReadFull(in.Body, head)
		if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
			return pact.FormFieldUploadResult{}, fmt.Errorf("journal: editor upload: read: %w", err)
		}
		head = head[:n]
		sniffed := http.DetectContentType(head)
		ext, known := blogImageExtensions[sniffed]
		if !known {
			return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}}
		}
		name := safeUploadName(in.Filename)
		if named := strings.TrimPrefix(path.Ext(name), "."); named == "" {
			name += "." + ext
		} else if want, allowed := blogImageTypes[named]; allowed && want != sniffed {
			return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}}
		}
		folder, _ := in.Options["uploadFolder"].(string)
		if folder == "" {
			folder = blogUploadFolder
		}
		stored, err := StoreMedia(ctx, bucket, folder, name, io.MultiReader(bytes.NewReader(head), in.Body))
		if err != nil {
			var refused *MediaError
			if errors.As(err, &refused) && refused != nil {
				return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{refused.Message}}
			}
			return pact.FormFieldUploadResult{}, err
		}
		return pact.FormFieldUploadResult{URL: stored.URL}, nil
	}
}

// safeUploadName is the stored name of an editor upload: MediaFileName of
// its base name (D-85), so it needs no escaping in an src attribute. An
// empty name becomes "image".
func safeUploadName(filename string) string {
	return MediaFileName(filename)
}