package classes import ( "bytes" "context" "errors" "fmt" "io" "net/http" "path" "regexp" "strings" "unicode" "unicode/utf8" "git.golem15.com/golem15/sm-journal-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/compass" "git.golem15.com/golem15/summercms/modules/pact" "gocloud.dev/blob" xhtml "golang.org/x/net/html" "gorm.io/gorm" ) // BlogEditorElement is the custom element of the blogeditor and mlblogeditor // field types, defined by assets/dist/blogeditor.js. const BlogEditorElement = "golem15-journal-blogeditor" // Phrase keys of the Blog Editor field types. const ( editorContentInvalid = "golem15.journal::lang.editor.content_invalid" editorImageInvalid = "golem15.journal::lang.editor.image_invalid" editorRemovedElements = "golem15.journal::lang.editor.removed_elements" editorRemovedAttributes = "golem15.journal::lang.editor.removed_attributes" editorRemovedComments = "golem15.journal::lang.editor.removed_comments" editorMarkupChanged = "golem15.journal::lang.editor.markup_changed" // EditorH1Converted is the notice of a save that turned H1 headings // into H2 (Count is the number of headings). EditorH1Converted = "golem15.journal::lang.editor.h1_converted" // EditorH1ConvertedCategory is EditorH1Converted for a category's // pillar text, which names the category name instead of the post title. EditorH1ConvertedCategory = "golem15.journal::lang.editor.h1_converted_category" ) // blogListValueRunes is how many runes of text a list column shows. const blogListValueRunes = 120 // blogUploadFolder is the upload folder of a field without uploadFolder. const blogUploadFolder = "editor" var ( blogHTMLColumnPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) blogUploadExtPattern = regexp.MustCompile(`^\.[a-z0-9]+$`) // blogImageTypes maps an upload extension to the content type its first // bytes must sniff as. blogImageTypes = map[string]string{ "jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png", "gif": "image/gif", "webp": "image/webp", } // blogImageExtensions is the extension given to a name without one. blogImageExtensions = map[string]string{ "image/jpeg": "jpg", "image/png": "png", "image/gif": "gif", "image/webp": "webp", } ) // BlogEditorTypes returns the Blog Editor admin field types: blogeditor for // a column that holds one HTML string, and mlblogeditor for a translatable // one. Both render BlogEditorElement. Decode stores what the HTML policy // keeps (SanitizeHTMLReport) and reports what it removed as notices; it // never refuses a string because of its markup. Load serves HTML content as // it is and a markdown value as the rendered HTML of the htmlColumn option, // and a blank value as the htmlColumn attribute when that holds a body; a // value that starts with '<' counts as HTML only when the htmlColumn // attribute is empty or holds that same HTML (loadBlogHTML). Upload stores an image the editor // inserts through StoreMedia, in the media bucket the app publishes. app is // asked for the booted application at request time; it may be nil in tests, // where Upload then fails. // // Options: htmlColumn (mlblogeditor only) names the translatable attribute // that holds the rendered HTML of a markdown value; uploadFolder is the // folder under /journal/ that editor image uploads are stored in ("editor" // when unset). func BlogEditorTypes(app func() *backpack.App) []pact.FormFieldType { upload := uploadBlogImage(app) decode := decodeBlogHTMLFor(app) options := blogRequestOptions(app) return []pact.FormFieldType{ { Name: "blogeditor", Element: BlogEditorElement, Scripts: []string{"assets/dist/blogeditor.js"}, Styles: []string{"assets/dist/blogeditor.css"}, Options: []string{"uploadFolder"}, CheckOptions: checkBlogOptions, Decode: decode, Load: loadBlogHTML, ListValue: blogListValue, Upload: upload, RequestOptions: options, }, { Name: "mlblogeditor", Element: BlogEditorElement, Translatable: true, Scripts: []string{"assets/dist/blogeditor.js"}, Styles: []string{"assets/dist/blogeditor.css"}, Options: []string{"htmlColumn", "uploadFolder"}, CheckOptions: checkBlogOptions, Decode: decode, Load: loadBlogHTML, ListValue: blogListValue, Upload: upload, RequestOptions: options, }, } } // LooksLikeHTML reports whether s may be stored HTML rather than markdown: // its first character that is not white space is '<' (an HTML comment // included). Phase 16.2 assumption A1. Markdown can start with a tag too, // so Load also compares the value with its htmlColumn attribute before it // serves the value as HTML. func LooksLikeHTML(s string) bool { t := strings.TrimLeftFunc(s, unicode.IsSpace) return strings.HasPrefix(t, "<") } // checkBlogOptions checks a Blog Editor field's options at boot. func checkBlogOptions(options map[string]any) error { if raw, ok := options["htmlColumn"]; ok { column, isString := raw.(string) if !isString || !blogHTMLColumnPattern.MatchString(column) { return fmt.Errorf("htmlColumn must be an attribute name, got %v", raw) } } if raw, ok := options["uploadFolder"]; ok { folder, isString := raw.(string) if !isString { return fmt.Errorf("uploadFolder must be a string, got %v", raw) } if err := checkUploadFolder(folder); err != nil { return err } } return nil } // checkUploadFolder applies the public media upload route's folder rule. func checkUploadFolder(folder string) error { if !validMediaFolder(folder) { return errors.New("uploadFolder must be at most 100 letters, digits, '_', '-' or '/' without '..'") } return nil } // decodeBlogHTMLFor is the Blog Editor types' Decode: decodeBlogHTML, then, // while the H1 setting is on (EffectiveH1ToH2 on the save transaction and // the booted application's config), ConvertH1 on the stored HTML with a // notice counting the converted headings: EditorH1ConvertedCategory on a // category's pillar text, EditorH1Converted otherwise. app may be nil // or answer nil; the setting is then read from the database alone. func decodeBlogHTMLFor(app func() *backpack.App) func(context.Context, pact.FormFieldValue) (pact.FormFieldDecoded, error) { return func(ctx context.Context, in pact.FormFieldValue) (pact.FormFieldDecoded, error) { out, err := decodeBlogHTML(ctx, in) if err != nil { return out, err } text, isString := out.Value.(string) if !isString || text == "" { return out, nil } if !EffectiveH1ToH2(ctx, in.DB, appConfig(app)) { return out, nil } converted, n := ConvertH1(text) if n == 0 { return out, nil } out.Value = converted out.Notices = append(out.Notices, pact.FormFieldNotice{Message: h1ConvertedNotice(in.Record), Count: n}) return out, nil } } // h1ConvertedNotice is the conversion notice for the saved record: the // category phrase for a category, the post phrase otherwise. func h1ConvertedNotice(record any) string { switch record.(type) { case *models.Category, models.Category: return EditorH1ConvertedCategory } return EditorH1Converted } // blogRequestOptions is the Blog Editor types' RequestOptions (33-01 // P33-02, UI-SPEC C28): headingLevels, [2, 3, 4] while the H1 setting is on // and [1, 2, 3, 4] while it is off (J1), and genericAnchors, the settings' // generic link texts (J2, J3). Both are read per schema request, so a // changed setting reaches the next form that opens. db may be nil; the // defaults then apply. fields.yaml options with the same names win. func blogRequestOptions(app func() *backpack.App) func(context.Context, *gorm.DB) (map[string]any, error) { return func(ctx context.Context, db *gorm.DB) (map[string]any, error) { levels := []int{2, 3, 4} if !EffectiveH1ToH2(ctx, db, appConfig(app)) { levels = []int{1, 2, 3, 4} } return map[string]any{ "headingLevels": levels, "genericAnchors": GenericAnchors(ctx, db), }, nil } } // appConfig is the config of the application app answers, nil when there // is none. func appConfig(app func() *backpack.App) *compass.Config { if app == nil { return nil } a := app() if a == nil { return nil } return a.Config } // decodeBlogHTML stores the HTML policy's output for a submitted value: the // exact submitted bytes with no notice when the policy keeps everything, // otherwise the stripped HTML with notices naming what was removed. nil // decodes to the empty string; any other value that is not a string is // refused. func decodeBlogHTML(_ context.Context, in pact.FormFieldValue) (pact.FormFieldDecoded, error) { var text string switch v := in.Value.(type) { case nil: return pact.FormFieldDecoded{Value: ""}, nil case string: text = v default: return pact.FormFieldDecoded{}, &pact.FormFieldError{Messages: []string{editorContentInvalid}} } clean, report := SanitizeHTMLReport(text) return pact.FormFieldDecoded{Value: clean, Notices: removalNotices(report)}, nil } // removalNotices turns a removal report into admin notices: removed // elements, removed attributes, removed comments and other changes, in that // order and only when non-zero. func removalNotices(r HTMLRemovals) []pact.FormFieldNotice { var out []pact.FormFieldNotice if r.ElementCount > 0 { out = append(out, pact.FormFieldNotice{ Message: editorRemovedElements, Params: map[string]string{"names": removalNames(r.Elements, r.moreElements)}, Count: r.ElementCount, }) } if r.AttributeCount > 0 { out = append(out, pact.FormFieldNotice{ Message: editorRemovedAttributes, Params: map[string]string{"names": removalNames(r.Attributes, r.moreAttributes)}, Count: r.AttributeCount, }) } if r.Comments > 0 { out = append(out, pact.FormFieldNotice{Message: editorRemovedComments, Count: r.Comments}) } if r.Other { out = append(out, pact.FormFieldNotice{Message: editorMarkupChanged}) } return out } func removalNames(names []string, cut bool) string { joined := strings.Join(names, ", ") if cut { joined += ", …" } return joined } // loadBlogHTML serves a stored value to the editor (D-07). Without the // htmlColumn option every value is served as stored. With it: // // - A blank value is served as the htmlColumn attribute in the same locale // when that is not empty, so a locale that carries only content_html (an // API-written translation, FormatTranslation) opens with the body the // site renders instead of an empty editor whose first edit would replace // it; it is served as stored otherwise. // - A value that looks like HTML (LooksLikeHTML) is served as it is when // the htmlColumn attribute is empty or missing, or holds the same HTML: // the value itself or the policy's output for it (SanitizeHTML), white // space at either end ignored. Otherwise the value is markdown that // merely starts with a tag (a hero , a leading
or comment), // and it is served as the htmlColumn attribute, so the first edit does // not store raw markdown as HTML. // - Any other value is markdown, served as the htmlColumn attribute, or as // FormatHTML of the value when that is empty or cannot be read (the // value itself when rendering fails). func loadBlogHTML(_ context.Context, in pact.FormFieldValue) (any, error) { text := storedText(in.Value) column, _ := in.Options["htmlColumn"].(string) if column == "" { return text, nil } html := "" if in.Translated != nil { value, found, err := in.Translated(column) if err != nil { return nil, err } if found { html = value } } switch { case strings.TrimSpace(text) == "": if html != "" { return html, nil } return text, nil case LooksLikeHTML(text): if html == "" || sameStoredHTML(text, html) { return text, nil } return html, nil case html != "": return html, nil } rendered, err := FormatHTML(text) if err != nil { return text, nil } return rendered, nil } // sameStoredHTML reports whether rendered, a stored htmlColumn value, is the // HTML text itself: equal to text or to the policy's output for it, with // white space at either end ignored. It is false when rendered came from // rendering text as markdown in a way that changed it. func sameStoredHTML(text, rendered string) bool { rendered = strings.TrimSpace(rendered) return rendered == strings.TrimSpace(text) || rendered == strings.TrimSpace(SanitizeHTML(text)) } // storedText is a stored column value as text: a string, a *string (nil is // empty) or nil. func storedText(v any) string { switch t := v.(type) { case nil: return "" case string: return t case *string: if t == nil { return "" } return *t default: return fmt.Sprint(t) } } // inlineListTags are the elements that do not separate words in a list // value's text. var inlineListTags = map[string]bool{ "a": true, "abbr": true, "b": true, "cite": true, "code": true, "del": true, "em": true, "i": true, "ins": true, "kbd": true, "mark": true, "q": true, "s": true, "small": true, "span": true, "strong": true, "sub": true, "sup": true, "u": true, } // blogListValue is the text a list column of a Blog Editor type shows: the // value without tags, white space collapsed, cut to 120 runes with an // ellipsis. func blogListValue(_ context.Context, value any) string { var b strings.Builder z := xhtml.NewTokenizer(strings.NewReader(storedText(value))) skip := 0 // inside script or style for { tt := z.Next() if tt == xhtml.ErrorToken { break } switch tt { case xhtml.TextToken: if skip == 0 { b.WriteString(z.Token().Data) } case xhtml.StartTagToken, xhtml.EndTagToken, xhtml.SelfClosingTagToken: name, _ := z.TagName() switch string(name) { case "script", "style": if tt == xhtml.StartTagToken { skip++ } else if tt == xhtml.EndTagToken && skip > 0 { skip-- } } if !inlineListTags[string(name)] { b.WriteByte(' ') } } } text := strings.Join(strings.Fields(b.String()), " ") if utf8.RuneCountInString(text) <= blogListValueRunes { return text } runes := []rune(text) return strings.TrimRightFunc(string(runes[:blogListValueRunes]), unicode.IsSpace) + "…" } // uploadBlogImage is the Blog Editor's Upload: it stores an image the // editor inserts through StoreMedia, in the field's uploadFolder, and // returns its URL. The file's first bytes must sniff as a JPEG, PNG, GIF or // WebP image of the type its extension names (a name without an extension // gets the sniffed one); anything else is refused with // golem15.journal::lang.editor.image_invalid. The stored name is // safeUploadName's. A MediaError is refused with its message; a missing // application or bucket is an opaque error. func uploadBlogImage(app func() *backpack.App) func(context.Context, pact.FormFieldUpload) (pact.FormFieldUploadResult, error) { return func(ctx context.Context, in pact.FormFieldUpload) (pact.FormFieldUploadResult, error) { var a *backpack.App if app != nil { a = app() } if a == nil { return pact.FormFieldUploadResult{}, errors.New("journal: editor upload: application is not configured") } bucket, ok := a.Lookup[*blob.Bucket]() if !ok || bucket == nil { return pact.FormFieldUploadResult{}, errors.New("journal: editor upload: media bucket is not configured") } if in.Body == nil { return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}} } head := make([]byte, 512) n, err := io.ReadFull(in.Body, head) if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { return pact.FormFieldUploadResult{}, fmt.Errorf("journal: editor upload: read: %w", err) } head = head[:n] sniffed := http.DetectContentType(head) ext, known := blogImageExtensions[sniffed] if !known { return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}} } name := safeUploadName(in.Filename) if named := strings.TrimPrefix(path.Ext(name), "."); named == "" { name += "." + ext } else if want, allowed := blogImageTypes[named]; allowed && want != sniffed { return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}} } folder, _ := in.Options["uploadFolder"].(string) if folder == "" { folder = blogUploadFolder } stored, err := StoreMedia(ctx, bucket, folder, name, io.MultiReader(bytes.NewReader(head), in.Body)) if err != nil { var refused *MediaError if errors.As(err, &refused) && refused != nil { return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{refused.Message}} } return pact.FormFieldUploadResult{}, err } return pact.FormFieldUploadResult{URL: stored.URL}, nil } } // safeUploadName is the stored name of an editor upload: MediaFileName of // its base name (D-85), so it needs no escaping in an src attribute. An // empty name becomes "image". func safeUploadName(filename string) string { return MediaFileName(filename) }