Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion internal/api/handler/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,20 @@ import (
"go.uber.org/zap"
)

const (
// contentTypeXML is the content type for XML responses
contentTypeXML = "application/xml"

// headerXAmzMeta is the prefix for user-defined metadata headers
headerXAmzMeta = "X-Amz-Meta-"
// headerXAmzMetaLen is the length of the metadata header prefix
headerXAmzMetaLen = len(headerXAmzMeta)

// minPartSize is the minimum allowed part size for multipart uploads (5MB)
// This is an S3 constraint that we enforce across all backends for consistency
minPartSize = 5 * 1024 * 1024
)

// Handler handles S3 API requests
type Handler struct {
backend types.Backend
Expand All @@ -24,7 +38,7 @@ func New(be types.Backend, logger *zap.Logger) *Handler {

// sendXML sends an XML response
func (h *Handler) sendXML(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/xml")
w.Header().Set("Content-Type", contentTypeXML)
w.WriteHeader(status)
_ = xml.NewEncoder(w).Encode(v)
}
Expand Down
10 changes: 2 additions & 8 deletions internal/api/handler/multipart.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,6 @@ import (
"go.uber.org/zap"
)

const (
// minPartSize is the minimum allowed part size for multipart uploads (5MB)
// This is an S3 constraint that we enforce across all backends for consistency
minPartSize = 5 * 1024 * 1024 // 5MB
)

// CreateMultipartUpload handles POST /{bucket}/{key}?uploads - initiate multipart upload
func (h *Handler) CreateMultipartUpload(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
Expand All @@ -26,8 +20,8 @@ func (h *Handler) CreateMultipartUpload(w http.ResponseWriter, r *http.Request)

metadata := make(map[string]string)
for k, v := range r.Header {
if len(v) > 0 && len(k) > 11 && k[:11] == "X-Amz-Meta-" {
metadata[k[11:]] = v[0]
if len(v) > 0 && len(k) > headerXAmzMetaLen && k[:headerXAmzMetaLen] == headerXAmzMeta {
metadata[k[headerXAmzMetaLen:]] = v[0]
}
}

Expand Down
8 changes: 4 additions & 4 deletions internal/api/handler/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func (h *Handler) HeadObject(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Last-Modified", info.LastModified.Format(http.TimeFormat))

for k, v := range info.Metadata {
w.Header().Set("x-amz-meta-"+k, v)
w.Header().Set(headerXAmzMeta+k, v)
}

w.WriteHeader(http.StatusOK)
Expand Down Expand Up @@ -74,7 +74,7 @@ func (h *Handler) GetObject(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Last-Modified", obj.LastModified.Format(http.TimeFormat))

for k, v := range obj.Metadata {
w.Header().Set("x-amz-meta-"+k, v)
w.Header().Set(headerXAmzMeta+k, v)
}

// Handle range response - check if this is a range request
Expand Down Expand Up @@ -175,8 +175,8 @@ func (h *Handler) PutObject(w http.ResponseWriter, r *http.Request) {

// Capture custom metadata (X-Amz-Meta-*)
for k, v := range r.Header {
if len(v) > 0 && len(k) > 11 && k[:11] == "X-Amz-Meta-" {
metadata[k[11:]] = v[0]
if len(v) > 0 && len(k) > headerXAmzMetaLen && k[:headerXAmzMetaLen] == headerXAmzMeta {
metadata[k[headerXAmzMetaLen:]] = v[0]
}
}

Expand Down
11 changes: 8 additions & 3 deletions internal/auth/middleware.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ import (
"go.uber.org/zap"
)

const (
// contentTypeXML is the content type for XML responses
contentTypeXML = "application/xml"
)

// Middleware provides authentication middleware for S3 API requests
type Middleware struct {
verifier *SignatureV4
Expand Down Expand Up @@ -96,15 +101,15 @@ type errorResponse struct {
}

func sendAuthError(w http.ResponseWriter, code, message string) {
w.Header().Set("Content-Type", "application/xml")
w.Header().Set("Content-Type", contentTypeXML)
w.WriteHeader(http.StatusForbidden)
resp := errorResponse{Code: code, Message: message}
_, _ = w.Write([]byte(xml.Header))
_ = xml.NewEncoder(w).Encode(resp)
}

func sendPayloadTooLargeError(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/xml")
w.Header().Set("Content-Type", contentTypeXML)
w.WriteHeader(http.StatusBadRequest)
resp := errorResponse{
Code: "XAmzContentSHA256Mismatch",
Expand All @@ -115,7 +120,7 @@ func sendPayloadTooLargeError(w http.ResponseWriter) {
}

func sendPayloadHashMismatchError(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/xml")
w.Header().Set("Content-Type", contentTypeXML)
w.WriteHeader(http.StatusBadRequest)
resp := errorResponse{
Code: "XAmzContentSHA256Mismatch",
Expand Down
23 changes: 15 additions & 8 deletions internal/auth/signature.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,18 @@ import (
"time"
)

const (
// defaultMaxClockSkew is the default maximum allowed clock skew for request timestamps (AWS default)
defaultMaxClockSkew = 15 * time.Minute

// maxBodyReadSize is the maximum body size for signature calculation (10 MiB)
// Requests larger than this must use UNSIGNED-PAYLOAD
maxBodyReadSize = 10 * 1024 * 1024

// emptyPayloadHash is the SHA256 hash of an empty payload
emptyPayloadHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
)

var (
// ErrMissingAuthHeader is returned when Authorization header is missing
ErrMissingAuthHeader = errors.New("missing Authorization header")
Expand Down Expand Up @@ -88,7 +100,7 @@ func NewSignatureV4(store CredentialStore, region, service string) *SignatureV4
credentialStore: store,
region: region,
service: service,
maxClockSkew: 15 * time.Minute, // AWS default
maxClockSkew: defaultMaxClockSkew,
}
}

Expand Down Expand Up @@ -217,8 +229,6 @@ func (s *SignatureV4) Verify(r *http.Request) error {
} else if payloadHash != "" {
// Client provided a specific payload hash - verify it matches the body
// Protect against memory exhaustion by limiting body read size
const maxBodyReadSize = 10 * 1024 * 1024 // 10 MiB max for signature calculation

if r.ContentLength > maxBodyReadSize {
// Request is too large to buffer for hash verification
// Client should use X-Amz-Content-Sha256: UNSIGNED-PAYLOAD header
Expand Down Expand Up @@ -246,16 +256,13 @@ func (s *SignatureV4) Verify(r *http.Request) error {
}
} else {
// Empty body - verify client sent empty payload hash
emptyHash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
if subtle.ConstantTimeCompare([]byte(emptyHash), []byte(payloadHash)) != 1 {
if subtle.ConstantTimeCompare([]byte(emptyPayloadHash), []byte(payloadHash)) != 1 {
return ErrPayloadHashMismatch
}
}
} else {
// No payload hash header provided - we need to read body to calculate hash
// Protect against memory exhaustion by limiting body read size
const maxBodyReadSize = 10 * 1024 * 1024 // 10 MiB max for signature calculation

if r.ContentLength > maxBodyReadSize {
// Request is too large to buffer for signature verification
// Client should use X-Amz-Content-Sha256: UNSIGNED-PAYLOAD header
Expand All @@ -276,7 +283,7 @@ func (s *SignatureV4) Verify(r *http.Request) error {
r.Body = io.NopCloser(bytes.NewReader(payload))
payloadHash = hex.EncodeToString(sha256Hash(payload))
} else {
payloadHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" // empty payload hash
payloadHash = emptyPayloadHash
}
}

Expand Down
10 changes: 5 additions & 5 deletions internal/backend/awss3/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
// PutObject uploads an object using the S3 Upload Manager for efficient streaming.
func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.Reader, size int64, metadata map[string]string) (*types.ObjectInfo, error) {
// Extract Content-Type from metadata
contentType := "application/octet-stream"
contentType := types.DefaultContentType
if ct, ok := metadata["Content-Type"]; ok {
contentType = ct
delete(metadata, "Content-Type")
Expand All @@ -38,8 +38,8 @@ func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.R

// Use S3 Upload Manager which handles streaming and multipart uploads automatically
uploader := manager.NewUploader(b.client, func(u *manager.Uploader) {
u.PartSize = 5 * 1024 * 1024 // 5 MiB minimum part size
u.Concurrency = 1 // Single concurrency to minimize memory
u.PartSize = types.PartSize5MiB
u.Concurrency = 1 // Single concurrency to minimize memory
})

input := &s3.PutObjectInput{
Expand Down Expand Up @@ -131,7 +131,7 @@ func (b *Backend) GetObject(ctx context.Context, bucket, key string, opts *types
}

// Get content type
contentType := "application/octet-stream"
contentType := types.DefaultContentType
if resp.ContentType != nil {
contentType = *resp.ContentType
}
Expand Down Expand Up @@ -189,7 +189,7 @@ func (b *Backend) HeadObject(ctx context.Context, bucket, key string) (*types.Ob
}

// Get content type
contentType := "application/octet-stream"
contentType := types.DefaultContentType
if resp.ContentType != nil {
contentType = *resp.ContentType
}
Expand Down
8 changes: 4 additions & 4 deletions internal/backend/azblob/bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,8 @@ func (b *Backend) ListObjects(ctx context.Context, bucket string, opts types.Lis
containerClient := b.client.ServiceClient().NewContainerClient(bucket)

// Set up list options
maxResults := int32(1000)
if opts.MaxKeys > 0 && opts.MaxKeys < 1000 {
maxResults := int32(types.DefaultMaxKeys)
if opts.MaxKeys > 0 && opts.MaxKeys < types.DefaultMaxKeys {
maxResults = int32(opts.MaxKeys)
}

Expand Down Expand Up @@ -178,8 +178,8 @@ func (b *Backend) ListObjects(ctx context.Context, bucket string, opts types.Lis
func (b *Backend) listObjectsHierarchical(ctx context.Context, bucket string, opts types.ListObjectsOptions) (*types.ListObjectsResult, error) {
containerClient := b.client.ServiceClient().NewContainerClient(bucket)

maxResults := int32(1000)
if opts.MaxKeys > 0 && opts.MaxKeys < 1000 {
maxResults := int32(types.DefaultMaxKeys)
if opts.MaxKeys > 0 && opts.MaxKeys < types.DefaultMaxKeys {
maxResults = int32(opts.MaxKeys)
}

Expand Down
4 changes: 2 additions & 2 deletions internal/backend/azblob/multipart.go
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,7 @@ func (b *Backend) ListParts(ctx context.Context, bucket, key, uploadID string, o
// Apply pagination
maxParts := opts.MaxParts
if maxParts <= 0 {
maxParts = 1000
maxParts = types.DefaultMaxParts
}

// Filter parts after the marker
Expand Down Expand Up @@ -600,7 +600,7 @@ func (b *Backend) ListMultipartUploads(ctx context.Context, bucket string, opts
// Apply max uploads limit
maxUploads := opts.MaxUploads
if maxUploads <= 0 {
maxUploads = 1000
maxUploads = types.DefaultMaxUploads
}

isTruncated := false
Expand Down
10 changes: 5 additions & 5 deletions internal/backend/azblob/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
// PutObject uploads a blob using streaming upload with MD5 calculation.
func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.Reader, size int64, metadata map[string]string) (*types.ObjectInfo, error) {
// Extract Content-Type from metadata
contentType := "application/octet-stream"
contentType := types.DefaultContentType
if ct, ok := metadata["Content-Type"]; ok {
contentType = ct
delete(metadata, "Content-Type")
Expand All @@ -42,8 +42,8 @@ func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.R

// Upload using UploadStream which handles chunking internally
_, err := blockBlobClient.UploadStream(ctx, teeReader, &blockblob.UploadStreamOptions{
BlockSize: 4 * 1024 * 1024, // 4 MiB blocks
Concurrency: 1, // Single concurrency to minimize memory
BlockSize: types.ChunkSize4MiB,
Concurrency: 1, // Single concurrency to minimize memory
Metadata: azureMetadata,
HTTPHeaders: &blob.HTTPHeaders{
BlobContentType: &contentType,
Expand Down Expand Up @@ -155,7 +155,7 @@ func (b *Backend) GetObject(ctx context.Context, bucket, key string, opts *types
}

// Get content type
contentType := "application/octet-stream"
contentType := types.DefaultContentType
if resp.ContentType != nil {
contentType = *resp.ContentType
}
Expand Down Expand Up @@ -219,7 +219,7 @@ func (b *Backend) HeadObject(ctx context.Context, bucket, key string) (*types.Ob
}

// Get content type
contentType := "application/octet-stream"
contentType := types.DefaultContentType
if props.ContentType != nil {
contentType = *props.ContentType
}
Expand Down
10 changes: 5 additions & 5 deletions internal/backend/disk/bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,12 @@ func (b *Backend) CreateBucket(ctx context.Context, bucket string) error {
}

// Create bucket directory structure
if err := os.MkdirAll(bucketPath, 0755); err != nil {
if err := os.MkdirAll(bucketPath, dirPermissions); err != nil {
return fmt.Errorf("failed to create bucket directory: %w", err)
}

// Create subdirectory for multipart uploads
if err := os.MkdirAll(filepath.Join(bucketPath, multipartDir), 0755); err != nil {
if err := os.MkdirAll(filepath.Join(bucketPath, multipartDir), dirPermissions); err != nil {
// Cleanup on failure
_ = os.RemoveAll(bucketPath)
return fmt.Errorf("failed to create bucket subdirectory %s: %w", multipartDir, err)
Expand Down Expand Up @@ -81,7 +81,7 @@ func (b *Backend) DeleteBucket(ctx context.Context, bucket string) error {
}
return nil
}
if strings.HasPrefix(d.Name(), ".") && strings.HasSuffix(d.Name(), ".object.json") {
if strings.HasPrefix(d.Name(), ".") && strings.HasSuffix(d.Name(), objectMetadataSuffix) {
hasObjects = true
return filepath.SkipAll // Stop walking, we found an object
}
Expand Down Expand Up @@ -180,7 +180,7 @@ func (b *Backend) ListObjects(ctx context.Context, bucket string, opts types.Lis

maxKeys := opts.MaxKeys
if maxKeys <= 0 {
maxKeys = 1000
maxKeys = types.DefaultMaxKeys
}

startAfter := opts.Marker
Expand Down Expand Up @@ -274,7 +274,7 @@ func (b *Backend) ListObjects(ctx context.Context, bucket string, opts types.Lis
}

// Only process hidden .object.json files
if !strings.HasPrefix(d.Name(), ".") || !strings.HasSuffix(d.Name(), ".object.json") {
if !strings.HasPrefix(d.Name(), ".") || !strings.HasSuffix(d.Name(), objectMetadataSuffix) {
return nil
}

Expand Down
Loading