feat(19.3-03): storage.uploads.public_url for direct bucket or CDN file URLs
- OpenBucket validates public_url (absolute http/https, host, no user info, query or fragment; the error names the key) and records it next to public_path_prefix - a set public_url logs one start-up line: the bucket behind it must hold only public files, since it bypasses the is_public check - attach.PublicURLBase; PublicURL, File.URL and thumbnail URLs use the base when set and keep the public_path_prefix proxy URLs otherwise - TestPublicURLSmoke; storage Public URL base section, attachments, configuration and lagoon README
This commit is contained in:
@@ -3,6 +3,7 @@ package attach
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -24,6 +25,7 @@ const S3BuildTag = "summercms_s3"
|
||||
var (
|
||||
prefixMu sync.RWMutex
|
||||
publicPathPrefix = defaultPublicPathPrefix
|
||||
publicURLBase string
|
||||
)
|
||||
|
||||
func setPublicPathPrefix(prefix string) {
|
||||
@@ -32,6 +34,38 @@ func setPublicPathPrefix(prefix string) {
|
||||
prefixMu.Unlock()
|
||||
}
|
||||
|
||||
func setPublicURLBase(base string) {
|
||||
prefixMu.Lock()
|
||||
publicURLBase = base
|
||||
prefixMu.Unlock()
|
||||
}
|
||||
|
||||
// PublicURLBase is storage.uploads.public_url as recorded by OpenBucket: an
|
||||
// absolute http or https base (a public bucket or a CDN) that file and
|
||||
// thumbnail URLs start with instead of PublicPathPrefix. Empty when unset.
|
||||
func PublicURLBase() string {
|
||||
prefixMu.RLock()
|
||||
defer prefixMu.RUnlock()
|
||||
return publicURLBase
|
||||
}
|
||||
|
||||
// parsePublicURL validates storage.uploads.public_url: empty means unset,
|
||||
// anything else must be an absolute http or https URL with a host and
|
||||
// without user info, query or fragment.
|
||||
func parsePublicURL(raw string) (*url.URL, error) {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
return nil, nil
|
||||
}
|
||||
u, err := url.Parse(value)
|
||||
bad := err != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" ||
|
||||
(!strings.EqualFold(u.Scheme, "http") && !strings.EqualFold(u.Scheme, "https"))
|
||||
if bad {
|
||||
return nil, fmt.Errorf("attach: storage.uploads.public_url must be an absolute http or https URL such as https://cdn.example.com/uploads, without user info, query or fragment (set SUMMER_STORAGE__UPLOADS__PUBLIC_URL)")
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// PublicPathPrefix is the URL prefix prepended to partition+filename.
|
||||
func PublicPathPrefix() string {
|
||||
prefixMu.RLock()
|
||||
@@ -79,7 +113,9 @@ func checkS3Driver(bucketURL string) error {
|
||||
|
||||
// OpenBucket opens storage.uploads.bucket_url (file://, mem://, or s3:// in a
|
||||
// binary built with S3BuildTag) and records
|
||||
// storage.uploads.public_path_prefix. An empty bucket_url fails boot.
|
||||
// storage.uploads.public_path_prefix and storage.uploads.public_url. An
|
||||
// empty bucket_url or a malformed public_url fails boot; a set public_url
|
||||
// logs one info line saying the bucket behind it must hold only public files.
|
||||
func OpenBucket(ctx context.Context, cfg *compass.Config) (*blob.Bucket, error) {
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("attach: config is nil")
|
||||
@@ -95,6 +131,10 @@ func OpenBucket(ctx context.Context, cfg *compass.Config) (*blob.Bucket, error)
|
||||
if err := checkS3Driver(bucketURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
publicURL, err := parsePublicURL(cfg.String("storage.uploads.public_url"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket, err := blob.OpenBucket(ctx, bucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("attach: open bucket %q: %w", raw, err)
|
||||
@@ -104,6 +144,14 @@ func OpenBucket(ctx context.Context, cfg *compass.Config) (*blob.Bucket, error)
|
||||
prefix = defaultPublicPathPrefix
|
||||
}
|
||||
setPublicPathPrefix(prefix)
|
||||
if publicURL == nil {
|
||||
setPublicURLBase("")
|
||||
return bucket, nil
|
||||
}
|
||||
setPublicURLBase(publicURL.String())
|
||||
slog.Default().InfoContext(ctx, "attach: storage.uploads.public_url is set; every file and thumbnail URL points at this host, "+
|
||||
"which serves any key without the is_public check, so the bucket behind it must hold only public files",
|
||||
slog.String("host", publicURL.Host))
|
||||
return bucket, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -58,3 +58,49 @@ func TestOpenBucketS3Smoke(t *testing.T) {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublicURLSmoke checks storage.uploads.public_url (D-12): set, URLs
|
||||
// are the base plus the partition path; unset, they keep the proxy prefix;
|
||||
// malformed, OpenBucket fails naming the key.
|
||||
func TestPublicURLSmoke(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
setPublicPathPrefix(defaultPublicPathPrefix)
|
||||
setPublicURLBase("")
|
||||
})
|
||||
key := BlobKey("651a2b3c4d5e61234567.png")
|
||||
|
||||
cfg := openTestConfig(t, "uploads:\n bucket_url: \"mem://\"\n public_url: \"https://cdn.example.com/uploads/\"\n")
|
||||
bucket, err := OpenBucket(t.Context(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = bucket.Close()
|
||||
if got, want := PublicURL(key), "https://cdn.example.com/uploads/"+key; got != want {
|
||||
t.Fatalf("PublicURL with public_url = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := (&File{DiskName: "651a2b3c4d5e61234567.png"}).URL(), "https://cdn.example.com/uploads/651/a2b/3c4/651a2b3c4d5e61234567.png"; got != want {
|
||||
t.Fatalf("File.URL with public_url = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
cfg = openTestConfig(t, "uploads:\n bucket_url: \"mem://\"\n")
|
||||
bucket, err = OpenBucket(t.Context(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = bucket.Close()
|
||||
if PublicURLBase() != "" || !strings.HasPrefix(PublicURL(key), "/storage/uploads/") {
|
||||
t.Fatalf("without public_url: base %q, URL %q", PublicURLBase(), PublicURL(key))
|
||||
}
|
||||
|
||||
for _, bad := range []string{"ftp://x", "https://u:p@cdn.example.com", "/storage/uploads", "https://cdn.example.com/?v=1"} {
|
||||
cfg = openTestConfig(t, "uploads:\n bucket_url: \"mem://\"\n public_url: \""+bad+"\"\n")
|
||||
bucket, err = OpenBucket(t.Context(), cfg)
|
||||
if err == nil {
|
||||
_ = bucket.Close()
|
||||
t.Fatalf("public_url %q accepted", bad)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "storage.uploads.public_url") || strings.Contains(err.Error(), "u:p") {
|
||||
t.Fatalf("public_url %q: error %q", bad, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,13 +76,18 @@ func fileExt(diskName string) string {
|
||||
return strings.ToLower(ext)
|
||||
}
|
||||
|
||||
// PublicURL returns the public URL of a blob key: storage.uploads.
|
||||
// public_path_prefix and the key joined by exactly one slash. With the
|
||||
// WinterCMS layout (bucket rooted at storage/app/uploads/public, prefix
|
||||
// /storage/app/uploads/public) it is Winter's File::getPath() path.
|
||||
// PublicURL returns the public URL of a blob key. With
|
||||
// storage.uploads.public_url set (PublicURLBase) it is that base and the key
|
||||
// joined by exactly one slash. Otherwise it is
|
||||
// storage.uploads.public_path_prefix and the key joined the same way; with
|
||||
// the WinterCMS layout (bucket rooted at storage/app/uploads/public, prefix
|
||||
// /storage/app/uploads/public) that is Winter's File::getPath() path.
|
||||
func PublicURL(key string) string {
|
||||
prefix := strings.TrimRight(PublicPathPrefix(), "/")
|
||||
key = strings.TrimLeft(key, "/")
|
||||
if base := PublicURLBase(); base != "" {
|
||||
return strings.TrimRight(base, "/") + "/" + key
|
||||
}
|
||||
prefix := strings.TrimRight(PublicPathPrefix(), "/")
|
||||
if prefix == "" {
|
||||
return "/" + key
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user