Jakub Zych
924e6aa0dc
- classes/uploads.go: tickets signed with HMAC-SHA256 under the
jz.bm.upload.v1: label (app.key), 10-minute expiry, the nonce pulled
from the cache once (duplicated from golem15.presentations on purpose)
- POST /api/v1/bm/materials/{id}/files/upload-url issues the URL with a
curl example; POST /_bm/api/v1/uploads/{ticket} takes one file part,
no bearer, throttled per IP and per ticket, user and permission
re-checked, stored through storeMaterialFile
- bm_manage_create_material_upload; README documents the file routes
377 lines
12 KiB
Go
377 lines
12 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
userclasses "git.golem15.com/golem15/sm-user-plugin/classes"
|
|
"git.golem15.com/golem15/summercms/modules/cabana"
|
|
"git.golem15.com/golem15/summercms/modules/cooler"
|
|
"git.golem15.com/golem15/summercms/modules/lagoon"
|
|
"git.golem15.com/golem15/summercms/modules/lagoon/attach"
|
|
"git.golem15.com/golem15/summercms/modules/wire"
|
|
"git.golem15.com/jakub/sm-bm-plugin/classes"
|
|
"git.golem15.com/jakub/sm-bm-plugin/controllers"
|
|
"git.golem15.com/jakub/sm-bm-plugin/models"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// Material files over MCP (D-10, D-11, D-18): a small file goes in as base64
|
|
// in one tool call (AttachMaterialInline). Every path stores through
|
|
// storeMaterialFile, so attach.Store and the material's file cap apply.
|
|
|
|
// MaterialInlineMaxBytes is the largest decoded file of POST
|
|
// /materials/{id}/files/inline. pier refuses MCP request bodies over 4 MiB,
|
|
// and base64 grows a file by a third, so about 3 MB is what fits (D-18).
|
|
const MaterialInlineMaxBytes = 3_000_000
|
|
|
|
// MaterialInlineBodyLimit is the body cap of POST
|
|
// /materials/{id}/files/inline: the base64 of MaterialInlineMaxBytes plus
|
|
// the JSON around it.
|
|
const MaterialInlineBodyLimit = 4_300_000
|
|
|
|
// MaterialFileWarning is the sentence every material file tool and answer
|
|
// carries (D-11).
|
|
const MaterialFileWarning = "Files attached to a material are public to anyone with the link."
|
|
|
|
// inlineFile is the body of POST /materials/{id}/files/inline.
|
|
type inlineFile struct {
|
|
Name string `json:"name"`
|
|
Mime string `json:"mime"`
|
|
ContentBase64 string `json:"content_base64"`
|
|
}
|
|
|
|
// decodeInline reads and checks the body of POST /materials/{id}/files/inline.
|
|
// The encoded content is measured before it is decoded, so an oversized one
|
|
// costs no decoding.
|
|
func decodeInline(raw []byte) (name string, mime string, data []byte, errs map[string][]string) {
|
|
var in inlineFile
|
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
|
dec.DisallowUnknownFields()
|
|
if err := dec.Decode(&in); err != nil {
|
|
return "", "", nil, map[string][]string{"json": {"Send a JSON object with name, content_base64 and optionally mime: " + err.Error()}}
|
|
}
|
|
errs = map[string][]string{}
|
|
name = strings.TrimSpace(in.Name)
|
|
if name == "" || len(name) > 255 {
|
|
errs["name"] = []string{"The name is required (the file name with its extension) and may not be longer than 255 characters."}
|
|
}
|
|
mime = strings.ToLower(strings.TrimSpace(in.Mime))
|
|
if mime != "" && (len(mime) > 100 || !strings.Contains(mime, "/") || strings.ContainsAny(mime, " \t;,")) {
|
|
errs["mime"] = []string{"The mime must be one MIME type such as application/pdf."}
|
|
}
|
|
switch {
|
|
case in.ContentBase64 == "":
|
|
errs["content_base64"] = []string{"The content_base64 is required."}
|
|
case len(in.ContentBase64) > base64.StdEncoding.EncodedLen(MaterialInlineMaxBytes)+inlineSlack(in.ContentBase64):
|
|
errs["content_base64"] = []string{"The file is larger than " + strconv.Itoa(MaterialInlineMaxBytes) + " bytes: use bm_manage_create_material_upload for larger files."}
|
|
default:
|
|
b, err := base64.StdEncoding.DecodeString(in.ContentBase64)
|
|
switch {
|
|
case err != nil:
|
|
errs["content_base64"] = []string{"The content_base64 is not valid standard base64."}
|
|
case len(b) == 0:
|
|
errs["content_base64"] = []string{"The file is empty."}
|
|
case len(b) > MaterialInlineMaxBytes:
|
|
errs["content_base64"] = []string{"The file is larger than " + strconv.Itoa(MaterialInlineMaxBytes) + " bytes: use bm_manage_create_material_upload for larger files."}
|
|
default:
|
|
data = b
|
|
}
|
|
}
|
|
if len(errs) > 0 {
|
|
return "", "", nil, errs
|
|
}
|
|
return name, mime, data, nil
|
|
}
|
|
|
|
// inlineSlack is the number of line breaks in s, which the base64 decoder
|
|
// skips: wrapped base64 is accepted at the same decoded size.
|
|
func inlineSlack(s string) int {
|
|
return strings.Count(s, "\n") + strings.Count(s, "\r")
|
|
}
|
|
|
|
// AttachMaterialInline answers POST /materials/{id}/files/inline: one file
|
|
// sent as JSON {name, mime?, content_base64}, stored as a public file of the
|
|
// material through attach.Store, at most MaterialInlineMaxBytes decoded. A
|
|
// sent mime must match the stored content type. The answer is the material
|
|
// as GET /materials/{id} shows it, with the public-file warning, 201.
|
|
func (h *Handlers) AttachMaterialInline(w http.ResponseWriter, r *http.Request) {
|
|
raw, ok := body(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
name, mime, data, errs := decodeInline(raw)
|
|
if errs != nil {
|
|
writeValidation(w, errs)
|
|
return
|
|
}
|
|
db, ok := h.db(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
bucket, ok := h.bucket(w)
|
|
if !ok {
|
|
return
|
|
}
|
|
m, ok := findMaterial(w, r, db)
|
|
if !ok {
|
|
return
|
|
}
|
|
lim := attach.Limits{MaxBytes: MaterialInlineMaxBytes}
|
|
if mime != "" {
|
|
lim.MIMETypes = []string{mime}
|
|
}
|
|
var stored *attach.File
|
|
var refusal string
|
|
err := lagoon.Transaction(r.Context(), db, func(ctx context.Context, tx *gorm.DB) error {
|
|
f, why, err := storeMaterialFile(ctx, tx, bucket, m, name, bytes.NewReader(data), lim)
|
|
stored, refusal = f, why
|
|
return err
|
|
})
|
|
if err != nil || refusal != "" {
|
|
// The blob is deleted again when the transaction did not commit.
|
|
if stored != nil {
|
|
_ = attach.DeleteKeys(context.WithoutCancel(r.Context()), bucket, attach.BlobKeys(*stored))
|
|
}
|
|
if refusal != "" {
|
|
writeValidation(w, map[string][]string{"file": {refusal}})
|
|
return
|
|
}
|
|
writeOpaque500(w)
|
|
return
|
|
}
|
|
h.writeMaterialDetailWith(w, r, db, m.ID, http.StatusCreated, map[string]any{"warning": MaterialFileWarning})
|
|
}
|
|
|
|
// Signed one-time uploads (D-10, D-12, D-19): for anything larger, the
|
|
// agent asks for an upload URL (CreateMaterialUpload) and sends the file
|
|
// with curl, so the bytes never pass through MCP. The upload route
|
|
// (SignedMaterialUpload) has no bearer: the ticket in its path is the
|
|
// credential. It is bound to one material and one user, valid
|
|
// classes.UploadTTL and works once; the user and the permission are
|
|
// checked again when it is redeemed.
|
|
|
|
// UploadPath is the path of the signed upload route; a ticket follows it.
|
|
// The plugin mounts the route at APIPrefix + "/uploads/{ticket}", and
|
|
// TestRoutes pins the two together.
|
|
const UploadPath = "/_bm/api/v1/uploads/"
|
|
|
|
// uploadKind is the ticket kind of a material file upload.
|
|
const uploadKind = "material"
|
|
|
|
// materialUploadJSON is the answer of CreateMaterialUpload.
|
|
type materialUploadJSON struct {
|
|
UploadURL string `json:"upload_url"`
|
|
ExpiresAt string `json:"expires_at"`
|
|
MaxBytes int `json:"max_bytes"`
|
|
Accept string `json:"accept"`
|
|
CurlExample string `json:"curl_example"`
|
|
Warning string `json:"warning"`
|
|
}
|
|
|
|
// uploadedFileJSON is the file in the answer of SignedMaterialUpload.
|
|
type uploadedFileJSON struct {
|
|
ID uint `json:"id"`
|
|
Name string `json:"name"`
|
|
Size int64 `json:"size"`
|
|
Mime string `json:"mime"`
|
|
URL string `json:"url"`
|
|
}
|
|
|
|
// CreateMaterialUpload answers POST /materials/{id}/files/upload-url: a
|
|
// signed, single-use URL, valid classes.UploadTTL, for one file upload to
|
|
// this material by the caller, with a curl example. No URL is issued
|
|
// without app.key or a cache (503).
|
|
func (h *Handlers) CreateMaterialUpload(w http.ResponseWriter, r *http.Request) {
|
|
user, ok := h.caller(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
db, ok := h.db(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
m, ok := findMaterial(w, r, db)
|
|
if !ok {
|
|
return
|
|
}
|
|
key, cache := h.uploadKey(), h.uploadCache()
|
|
if len(key) == 0 || cache == nil {
|
|
wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]any{"error": true, "message": "Uploads are not configured."})
|
|
return
|
|
}
|
|
ticket, expires, err := classes.IssueUpload(r.Context(), cache, key, uploadKind, m.ID, user, h.now())
|
|
if err != nil {
|
|
writeOpaque500(w)
|
|
return
|
|
}
|
|
url := h.appBase(r) + UploadPath + ticket
|
|
wire.WriteJSON(w, http.StatusCreated, map[string]any{"data": materialUploadJSON{
|
|
UploadURL: url,
|
|
ExpiresAt: expires.UTC().Format(time.RFC3339),
|
|
MaxBytes: MaterialFileMaxBytes,
|
|
Accept: strings.Join(attach.DefaultFileExtensions, ", "),
|
|
CurlExample: "curl -fsS -F 'file=@<path>' '" + url + "'",
|
|
Warning: MaterialFileWarning,
|
|
}})
|
|
}
|
|
|
|
// SignedMaterialUpload answers POST /_bm/api/v1/uploads/{ticket}. The
|
|
// ticket is redeemed first (once; any failure is the same 403 before the
|
|
// database), then the ticket's user must still exist and hold
|
|
// jz.bm.access_materials, and the material must still exist. The body is
|
|
// multipart with exactly one part, named file, stored through
|
|
// storeMaterialFile with the material file limits.
|
|
func (h *Handlers) SignedMaterialUpload(w http.ResponseWriter, r *http.Request) {
|
|
ticket, err := classes.RedeemUpload(r.Context(), h.uploadCache(), h.uploadKey(), r.PathValue("ticket"), h.now())
|
|
if err != nil || ticket.Kind != uploadKind {
|
|
writeUploadForbidden(w)
|
|
return
|
|
}
|
|
db, ok := h.db(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
principal, err := userclasses.PrincipalByID(r.Context(), db, ticket.User)
|
|
if err != nil {
|
|
writeOpaque500(w)
|
|
return
|
|
}
|
|
if principal == nil || !cabana.Allows(principal, []string{controllers.PermMaterials}) {
|
|
writeUploadForbidden(w)
|
|
return
|
|
}
|
|
var m models.Material
|
|
res := db.Where("id = ?", ticket.Target).Limit(1).Find(&m)
|
|
if res.Error != nil {
|
|
writeOpaque500(w)
|
|
return
|
|
}
|
|
if res.RowsAffected == 0 {
|
|
writeNotFound(w)
|
|
return
|
|
}
|
|
bucket, ok := h.bucket(w)
|
|
if !ok {
|
|
return
|
|
}
|
|
mr, err := r.MultipartReader()
|
|
if err != nil {
|
|
writeValidation(w, map[string][]string{"file": {singleFilePart}})
|
|
return
|
|
}
|
|
var stored *attach.File
|
|
var refusal string
|
|
err = lagoon.Transaction(r.Context(), db, func(ctx context.Context, tx *gorm.DB) error {
|
|
part, err := mr.NextPart()
|
|
if err != nil && err != io.EOF {
|
|
return err
|
|
}
|
|
if err == io.EOF || part.FormName() != "file" || part.FileName() == "" {
|
|
if part != nil {
|
|
part.Close()
|
|
}
|
|
refusal = singleFilePart
|
|
return errUploadRefused
|
|
}
|
|
f, why, err := storeMaterialFile(ctx, tx, bucket, m, part.FileName(), part, fileLimits)
|
|
part.Close()
|
|
stored = f
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if why != "" {
|
|
refusal = why
|
|
return errUploadRefused
|
|
}
|
|
if next, err := mr.NextPart(); err != io.EOF {
|
|
if next != nil {
|
|
next.Close()
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
refusal = singleFilePart
|
|
return errUploadRefused
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
// The blob is deleted again when the transaction did not commit.
|
|
if stored != nil {
|
|
_ = attach.DeleteKeys(context.WithoutCancel(r.Context()), bucket, attach.BlobKeys(*stored))
|
|
}
|
|
var tooLarge *http.MaxBytesError
|
|
switch {
|
|
case refusal != "":
|
|
writeValidation(w, map[string][]string{"file": {refusal}})
|
|
case errors.As(err, &tooLarge):
|
|
wire.WriteJSON(w, http.StatusRequestEntityTooLarge, map[string]any{"error": true, "message": "Request body too large."})
|
|
default:
|
|
writeOpaque500(w)
|
|
}
|
|
return
|
|
}
|
|
url := stored.URL()
|
|
if strings.HasPrefix(url, "/") && !strings.HasPrefix(url, "//") {
|
|
url = h.appBase(r) + url
|
|
}
|
|
wire.WriteJSON(w, http.StatusCreated, map[string]any{
|
|
"data": map[string]any{
|
|
"material_id": m.ID,
|
|
"file": uploadedFileJSON{ID: stored.ID, Name: stored.FileName, Size: stored.FileSize, Mime: stored.ContentType, URL: url},
|
|
},
|
|
"warning": MaterialFileWarning,
|
|
})
|
|
}
|
|
|
|
// singleFilePart is the refusal of a signed upload body that is not one
|
|
// multipart part named file.
|
|
const singleFilePart = "Send exactly one file as multipart/form-data in a part named \"file\", for example curl -F 'file=@<path>'."
|
|
|
|
// errUploadRefused rolls back a refused signed upload.
|
|
var errUploadRefused = errors.New("api: upload refused")
|
|
|
|
// writeUploadForbidden is the one answer of every refused upload ticket.
|
|
func writeUploadForbidden(w http.ResponseWriter) {
|
|
wire.WriteJSON(w, http.StatusForbidden, map[string]any{"error": true, "message": "Invalid or expired upload URL."})
|
|
}
|
|
|
|
func (h *Handlers) uploadKey() []byte {
|
|
if h.UploadKey == nil {
|
|
return nil
|
|
}
|
|
return h.UploadKey()
|
|
}
|
|
|
|
func (h *Handlers) uploadCache() cooler.Cache {
|
|
if h.Cache == nil {
|
|
return nil
|
|
}
|
|
return h.Cache()
|
|
}
|
|
|
|
// appBase is the base of the absolute links of the upload answers (D-16):
|
|
// app.url, else jz.bm.public_url, else the scheme (TLS or
|
|
// X-Forwarded-Proto) and Host of the request.
|
|
func (h *Handlers) appBase(r *http.Request) string {
|
|
for _, base := range []string{h.AppURL, h.PublicURL} {
|
|
if base = strings.TrimRight(strings.TrimSpace(base), "/"); base != "" {
|
|
return base
|
|
}
|
|
}
|
|
scheme := "http"
|
|
if r.TLS != nil || strings.EqualFold(strings.TrimSpace(r.Header.Get("X-Forwarded-Proto")), "https") {
|
|
scheme = "https"
|
|
}
|
|
return scheme + "://" + r.Host
|
|
}
|