package classes import ( "bytes" "context" "errors" "fmt" "io" "net/http" "path" "regexp" "strings" "unicode" "unicode/utf8" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/pact" "gocloud.dev/blob" xhtml "golang.org/x/net/html" ) // BlogEditorElement is the custom element of the blogeditor and mlblogeditor // field types, defined by assets/dist/blogeditor.js. const BlogEditorElement = "golem15-journal-blogeditor" // Phrase keys of the Blog Editor field types. const ( editorContentInvalid = "golem15.journal::lang.editor.content_invalid" editorImageInvalid = "golem15.journal::lang.editor.image_invalid" editorRemovedElements = "golem15.journal::lang.editor.removed_elements" editorRemovedAttributes = "golem15.journal::lang.editor.removed_attributes" editorRemovedComments = "golem15.journal::lang.editor.removed_comments" editorMarkupChanged = "golem15.journal::lang.editor.markup_changed" ) // blogListValueRunes is how many runes of text a list column shows. const blogListValueRunes = 120 // blogUploadFolder is the upload folder of a field without uploadFolder. const blogUploadFolder = "editor" var ( blogHTMLColumnPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) blogUploadExtPattern = regexp.MustCompile(`^\.[a-z0-9]+$`) // blogImageTypes maps an upload extension to the content type its first // bytes must sniff as. blogImageTypes = map[string]string{ "jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png", "gif": "image/gif", "webp": "image/webp", } // blogImageExtensions is the extension given to a name without one. blogImageExtensions = map[string]string{ "image/jpeg": "jpg", "image/png": "png", "image/gif": "gif", "image/webp": "webp", } ) // BlogEditorTypes returns the Blog Editor admin field types: blogeditor for // a column that holds one HTML string, and mlblogeditor for a translatable // one. Both render BlogEditorElement. Decode stores what the HTML policy // keeps (SanitizeHTMLReport) and reports what it removed as notices; it // never refuses a string because of its markup. Load serves HTML content as // it is and a markdown value as the rendered HTML of the htmlColumn option, // and a blank value as the htmlColumn attribute when that holds a body; a // value that starts with '<' counts as HTML only when the htmlColumn // attribute is empty or holds that same HTML (loadBlogHTML). Upload stores an image the editor // inserts through StoreMedia, in the media bucket the app publishes. app is // asked for the booted application at request time; it may be nil in tests, // where Upload then fails. // // Options: htmlColumn (mlblogeditor only) names the translatable attribute // that holds the rendered HTML of a markdown value; uploadFolder is the // folder under /journal/ that editor image uploads are stored in ("editor" // when unset). func BlogEditorTypes(app func() *backpack.App) []pact.FormFieldType { upload := uploadBlogImage(app) return []pact.FormFieldType{ { Name: "blogeditor", Element: BlogEditorElement, Scripts: []string{"assets/dist/blogeditor.js"}, Styles: []string{"assets/dist/blogeditor.css"}, Options: []string{"uploadFolder"}, CheckOptions: checkBlogOptions, Decode: decodeBlogHTML, Load: loadBlogHTML, ListValue: blogListValue, Upload: upload, }, { Name: "mlblogeditor", Element: BlogEditorElement, Translatable: true, Scripts: []string{"assets/dist/blogeditor.js"}, Styles: []string{"assets/dist/blogeditor.css"}, Options: []string{"htmlColumn", "uploadFolder"}, CheckOptions: checkBlogOptions, Decode: decodeBlogHTML, Load: loadBlogHTML, ListValue: blogListValue, Upload: upload, }, } } // LooksLikeHTML reports whether s may be stored HTML rather than markdown: // its first character that is not white space is '<' (an HTML comment // included). Phase 16.2 assumption A1. Markdown can start with a tag too, // so Load also compares the value with its htmlColumn attribute before it // serves the value as HTML. func LooksLikeHTML(s string) bool { t := strings.TrimLeftFunc(s, unicode.IsSpace) return strings.HasPrefix(t, "<") } // checkBlogOptions checks a Blog Editor field's options at boot. func checkBlogOptions(options map[string]any) error { if raw, ok := options["htmlColumn"]; ok { column, isString := raw.(string) if !isString || !blogHTMLColumnPattern.MatchString(column) { return fmt.Errorf("htmlColumn must be an attribute name, got %v", raw) } } if raw, ok := options["uploadFolder"]; ok { folder, isString := raw.(string) if !isString { return fmt.Errorf("uploadFolder must be a string, got %v", raw) } if err := checkUploadFolder(folder); err != nil { return err } } return nil } // checkUploadFolder applies the public media upload route's folder rule. func checkUploadFolder(folder string) error { if !validMediaFolder(folder) { return errors.New("uploadFolder must be at most 100 letters, digits, '_', '-' or '/' without '..'") } return nil } // decodeBlogHTML stores the HTML policy's output for a submitted value: the // exact submitted bytes with no notice when the policy keeps everything, // otherwise the stripped HTML with notices naming what was removed. nil // decodes to the empty string; any other value that is not a string is // refused. func decodeBlogHTML(_ context.Context, in pact.FormFieldValue) (pact.FormFieldDecoded, error) { var text string switch v := in.Value.(type) { case nil: return pact.FormFieldDecoded{Value: ""}, nil case string: text = v default: return pact.FormFieldDecoded{}, &pact.FormFieldError{Messages: []string{editorContentInvalid}} } clean, report := SanitizeHTMLReport(text) return pact.FormFieldDecoded{Value: clean, Notices: removalNotices(report)}, nil } // removalNotices turns a removal report into admin notices: removed // elements, removed attributes, removed comments and other changes, in that // order and only when non-zero. func removalNotices(r HTMLRemovals) []pact.FormFieldNotice { var out []pact.FormFieldNotice if r.ElementCount > 0 { out = append(out, pact.FormFieldNotice{ Message: editorRemovedElements, Params: map[string]string{"names": removalNames(r.Elements, r.moreElements)}, Count: r.ElementCount, }) } if r.AttributeCount > 0 { out = append(out, pact.FormFieldNotice{ Message: editorRemovedAttributes, Params: map[string]string{"names": removalNames(r.Attributes, r.moreAttributes)}, Count: r.AttributeCount, }) } if r.Comments > 0 { out = append(out, pact.FormFieldNotice{Message: editorRemovedComments, Count: r.Comments}) } if r.Other { out = append(out, pact.FormFieldNotice{Message: editorMarkupChanged}) } return out } func removalNames(names []string, cut bool) string { joined := strings.Join(names, ", ") if cut { joined += ", …" } return joined } // loadBlogHTML serves a stored value to the editor (D-07). Without the // htmlColumn option every value is served as stored. With it: // // - A blank value is served as the htmlColumn attribute in the same locale // when that is not empty, so a locale that carries only content_html (an // API-written translation, FormatTranslation) opens with the body the // site renders instead of an empty editor whose first edit would replace // it; it is served as stored otherwise. // - A value that looks like HTML (LooksLikeHTML) is served as it is when // the htmlColumn attribute is empty or missing, or holds the same HTML: // the value itself or the policy's output for it (SanitizeHTML), white // space at either end ignored. Otherwise the value is markdown that // merely starts with a tag (a hero , a leading
or comment), // and it is served as the htmlColumn attribute, so the first edit does // not store raw markdown as HTML. // - Any other value is markdown, served as the htmlColumn attribute, or as // FormatHTML of the value when that is empty or cannot be read (the // value itself when rendering fails). func loadBlogHTML(_ context.Context, in pact.FormFieldValue) (any, error) { text := storedText(in.Value) column, _ := in.Options["htmlColumn"].(string) if column == "" { return text, nil } html := "" if in.Translated != nil { value, found, err := in.Translated(column) if err != nil { return nil, err } if found { html = value } } switch { case strings.TrimSpace(text) == "": if html != "" { return html, nil } return text, nil case LooksLikeHTML(text): if html == "" || sameStoredHTML(text, html) { return text, nil } return html, nil case html != "": return html, nil } rendered, err := FormatHTML(text) if err != nil { return text, nil } return rendered, nil } // sameStoredHTML reports whether rendered, a stored htmlColumn value, is the // HTML text itself: equal to text or to the policy's output for it, with // white space at either end ignored. It is false when rendered came from // rendering text as markdown in a way that changed it. func sameStoredHTML(text, rendered string) bool { rendered = strings.TrimSpace(rendered) return rendered == strings.TrimSpace(text) || rendered == strings.TrimSpace(SanitizeHTML(text)) } // storedText is a stored column value as text: a string, a *string (nil is // empty) or nil. func storedText(v any) string { switch t := v.(type) { case nil: return "" case string: return t case *string: if t == nil { return "" } return *t default: return fmt.Sprint(t) } } // inlineListTags are the elements that do not separate words in a list // value's text. var inlineListTags = map[string]bool{ "a": true, "abbr": true, "b": true, "cite": true, "code": true, "del": true, "em": true, "i": true, "ins": true, "kbd": true, "mark": true, "q": true, "s": true, "small": true, "span": true, "strong": true, "sub": true, "sup": true, "u": true, } // blogListValue is the text a list column of a Blog Editor type shows: the // value without tags, white space collapsed, cut to 120 runes with an // ellipsis. func blogListValue(_ context.Context, value any) string { var b strings.Builder z := xhtml.NewTokenizer(strings.NewReader(storedText(value))) skip := 0 // inside script or style for { tt := z.Next() if tt == xhtml.ErrorToken { break } switch tt { case xhtml.TextToken: if skip == 0 { b.WriteString(z.Token().Data) } case xhtml.StartTagToken, xhtml.EndTagToken, xhtml.SelfClosingTagToken: name, _ := z.TagName() switch string(name) { case "script", "style": if tt == xhtml.StartTagToken { skip++ } else if tt == xhtml.EndTagToken && skip > 0 { skip-- } } if !inlineListTags[string(name)] { b.WriteByte(' ') } } } text := strings.Join(strings.Fields(b.String()), " ") if utf8.RuneCountInString(text) <= blogListValueRunes { return text } runes := []rune(text) return strings.TrimRightFunc(string(runes[:blogListValueRunes]), unicode.IsSpace) + "…" } // uploadBlogImage is the Blog Editor's Upload: it stores an image the // editor inserts through StoreMedia, in the field's uploadFolder, and // returns its URL. The file's first bytes must sniff as a JPEG, PNG, GIF or // WebP image of the type its extension names (a name without an extension // gets the sniffed one); anything else is refused with // golem15.journal::lang.editor.image_invalid. The stored name is // safeUploadName's. A MediaError is refused with its message; a missing // application or bucket is an opaque error. func uploadBlogImage(app func() *backpack.App) func(context.Context, pact.FormFieldUpload) (pact.FormFieldUploadResult, error) { return func(ctx context.Context, in pact.FormFieldUpload) (pact.FormFieldUploadResult, error) { var a *backpack.App if app != nil { a = app() } if a == nil { return pact.FormFieldUploadResult{}, errors.New("journal: editor upload: application is not configured") } bucket, ok := a.Lookup[*blob.Bucket]() if !ok || bucket == nil { return pact.FormFieldUploadResult{}, errors.New("journal: editor upload: media bucket is not configured") } if in.Body == nil { return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}} } head := make([]byte, 512) n, err := io.ReadFull(in.Body, head) if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { return pact.FormFieldUploadResult{}, fmt.Errorf("journal: editor upload: read: %w", err) } head = head[:n] sniffed := http.DetectContentType(head) ext, known := blogImageExtensions[sniffed] if !known { return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}} } name := safeUploadName(in.Filename) if named := strings.TrimPrefix(path.Ext(name), "."); named == "" { name += "." + ext } else if want, allowed := blogImageTypes[named]; allowed && want != sniffed { return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{editorImageInvalid}} } folder, _ := in.Options["uploadFolder"].(string) if folder == "" { folder = blogUploadFolder } stored, err := StoreMedia(ctx, bucket, folder, name, io.MultiReader(bytes.NewReader(head), in.Body)) if err != nil { var refused *MediaError if errors.As(err, &refused) && refused != nil { return pact.FormFieldUploadResult{}, &pact.FormFieldError{Messages: []string{refused.Message}} } return pact.FormFieldUploadResult{}, err } return pact.FormFieldUploadResult{URL: stored.URL}, nil } } // safeUploadName is the stored name of an editor upload: the base name, // lowercased, every character outside a-z, 0-9, '.', '_' and '-' turned // into '-' with runs collapsed and no leading or trailing '-' or '.', and // the extension kept. An empty name becomes "image". The result needs no // escaping in an src attribute. func safeUploadName(filename string) string { base := path.Base(strings.ReplaceAll(filename, `\`, "/")) ext := strings.ToLower(path.Ext(base)) if !blogUploadExtPattern.MatchString(ext) { ext = "" } stem := strings.ToLower(strings.TrimSuffix(base, path.Ext(base))) if ext == "" { stem = strings.ToLower(base) } var b strings.Builder dash := false for _, r := range stem { ok := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' if !ok || r == '-' { if !dash { b.WriteByte('-') dash = true } continue } b.WriteRune(r) dash = false } stem = strings.Trim(b.String(), "-.") if stem == "" { stem = "image" } return stem + ext }