From 5a0afd0d363ca40f7343cfcb7fdefaecae76d98f Mon Sep 17 00:00:00 2001 From: gerdos82 <37865635+gerdos82@users.noreply.github.com> Date: Sun, 25 Jan 2026 18:18:44 +0100 Subject: [PATCH] Replace magic values with constants --- internal/api/handler/handler.go | 16 +++++++++++- internal/api/handler/multipart.go | 10 ++------ internal/api/handler/object.go | 8 +++--- internal/auth/middleware.go | 11 +++++--- internal/auth/signature.go | 23 +++++++++++------ internal/backend/awss3/object.go | 10 ++++---- internal/backend/azblob/bucket.go | 8 +++--- internal/backend/azblob/multipart.go | 4 +-- internal/backend/azblob/object.go | 10 ++++---- internal/backend/disk/bucket.go | 10 ++++---- internal/backend/disk/disk.go | 38 ++++++++++++++++++++++------ internal/backend/disk/multipart.go | 26 +++++++++---------- internal/backend/disk/object.go | 10 ++++---- internal/backend/gcs/bucket.go | 4 +-- internal/backend/gcs/multipart.go | 14 +++++----- internal/backend/gcs/object.go | 4 +-- internal/backend/types/types.go | 20 +++++++++++++++ 17 files changed, 144 insertions(+), 82 deletions(-) diff --git a/internal/api/handler/handler.go b/internal/api/handler/handler.go index 2622eba..813c22b 100644 --- a/internal/api/handler/handler.go +++ b/internal/api/handler/handler.go @@ -8,6 +8,20 @@ import ( "go.uber.org/zap" ) +const ( + // contentTypeXML is the content type for XML responses + contentTypeXML = "application/xml" + + // headerXAmzMeta is the prefix for user-defined metadata headers + headerXAmzMeta = "X-Amz-Meta-" + // headerXAmzMetaLen is the length of the metadata header prefix + headerXAmzMetaLen = len(headerXAmzMeta) + + // minPartSize is the minimum allowed part size for multipart uploads (5MB) + // This is an S3 constraint that we enforce across all backends for consistency + minPartSize = 5 * 1024 * 1024 +) + // Handler handles S3 API requests type Handler struct { backend types.Backend @@ -24,7 +38,7 @@ func New(be types.Backend, logger *zap.Logger) *Handler { // sendXML sends an XML response func (h *Handler) sendXML(w http.ResponseWriter, status int, v any) { - w.Header().Set("Content-Type", "application/xml") + w.Header().Set("Content-Type", contentTypeXML) w.WriteHeader(status) _ = xml.NewEncoder(w).Encode(v) } diff --git a/internal/api/handler/multipart.go b/internal/api/handler/multipart.go index 341a2f7..f8534f5 100644 --- a/internal/api/handler/multipart.go +++ b/internal/api/handler/multipart.go @@ -12,12 +12,6 @@ import ( "go.uber.org/zap" ) -const ( - // minPartSize is the minimum allowed part size for multipart uploads (5MB) - // This is an S3 constraint that we enforce across all backends for consistency - minPartSize = 5 * 1024 * 1024 // 5MB -) - // CreateMultipartUpload handles POST /{bucket}/{key}?uploads - initiate multipart upload func (h *Handler) CreateMultipartUpload(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -26,8 +20,8 @@ func (h *Handler) CreateMultipartUpload(w http.ResponseWriter, r *http.Request) metadata := make(map[string]string) for k, v := range r.Header { - if len(v) > 0 && len(k) > 11 && k[:11] == "X-Amz-Meta-" { - metadata[k[11:]] = v[0] + if len(v) > 0 && len(k) > headerXAmzMetaLen && k[:headerXAmzMetaLen] == headerXAmzMeta { + metadata[k[headerXAmzMetaLen:]] = v[0] } } diff --git a/internal/api/handler/object.go b/internal/api/handler/object.go index a4c981a..da0b01d 100644 --- a/internal/api/handler/object.go +++ b/internal/api/handler/object.go @@ -32,7 +32,7 @@ func (h *Handler) HeadObject(w http.ResponseWriter, r *http.Request) { w.Header().Set("Last-Modified", info.LastModified.Format(http.TimeFormat)) for k, v := range info.Metadata { - w.Header().Set("x-amz-meta-"+k, v) + w.Header().Set(headerXAmzMeta+k, v) } w.WriteHeader(http.StatusOK) @@ -74,7 +74,7 @@ func (h *Handler) GetObject(w http.ResponseWriter, r *http.Request) { w.Header().Set("Last-Modified", obj.LastModified.Format(http.TimeFormat)) for k, v := range obj.Metadata { - w.Header().Set("x-amz-meta-"+k, v) + w.Header().Set(headerXAmzMeta+k, v) } // Handle range response - check if this is a range request @@ -175,8 +175,8 @@ func (h *Handler) PutObject(w http.ResponseWriter, r *http.Request) { // Capture custom metadata (X-Amz-Meta-*) for k, v := range r.Header { - if len(v) > 0 && len(k) > 11 && k[:11] == "X-Amz-Meta-" { - metadata[k[11:]] = v[0] + if len(v) > 0 && len(k) > headerXAmzMetaLen && k[:headerXAmzMetaLen] == headerXAmzMeta { + metadata[k[headerXAmzMetaLen:]] = v[0] } } diff --git a/internal/auth/middleware.go b/internal/auth/middleware.go index ea30a22..7159dac 100644 --- a/internal/auth/middleware.go +++ b/internal/auth/middleware.go @@ -7,6 +7,11 @@ import ( "go.uber.org/zap" ) +const ( + // contentTypeXML is the content type for XML responses + contentTypeXML = "application/xml" +) + // Middleware provides authentication middleware for S3 API requests type Middleware struct { verifier *SignatureV4 @@ -96,7 +101,7 @@ type errorResponse struct { } func sendAuthError(w http.ResponseWriter, code, message string) { - w.Header().Set("Content-Type", "application/xml") + w.Header().Set("Content-Type", contentTypeXML) w.WriteHeader(http.StatusForbidden) resp := errorResponse{Code: code, Message: message} _, _ = w.Write([]byte(xml.Header)) @@ -104,7 +109,7 @@ func sendAuthError(w http.ResponseWriter, code, message string) { } func sendPayloadTooLargeError(w http.ResponseWriter) { - w.Header().Set("Content-Type", "application/xml") + w.Header().Set("Content-Type", contentTypeXML) w.WriteHeader(http.StatusBadRequest) resp := errorResponse{ Code: "XAmzContentSHA256Mismatch", @@ -115,7 +120,7 @@ func sendPayloadTooLargeError(w http.ResponseWriter) { } func sendPayloadHashMismatchError(w http.ResponseWriter) { - w.Header().Set("Content-Type", "application/xml") + w.Header().Set("Content-Type", contentTypeXML) w.WriteHeader(http.StatusBadRequest) resp := errorResponse{ Code: "XAmzContentSHA256Mismatch", diff --git a/internal/auth/signature.go b/internal/auth/signature.go index e778987..bc81ad6 100644 --- a/internal/auth/signature.go +++ b/internal/auth/signature.go @@ -16,6 +16,18 @@ import ( "time" ) +const ( + // defaultMaxClockSkew is the default maximum allowed clock skew for request timestamps (AWS default) + defaultMaxClockSkew = 15 * time.Minute + + // maxBodyReadSize is the maximum body size for signature calculation (10 MiB) + // Requests larger than this must use UNSIGNED-PAYLOAD + maxBodyReadSize = 10 * 1024 * 1024 + + // emptyPayloadHash is the SHA256 hash of an empty payload + emptyPayloadHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +) + var ( // ErrMissingAuthHeader is returned when Authorization header is missing ErrMissingAuthHeader = errors.New("missing Authorization header") @@ -88,7 +100,7 @@ func NewSignatureV4(store CredentialStore, region, service string) *SignatureV4 credentialStore: store, region: region, service: service, - maxClockSkew: 15 * time.Minute, // AWS default + maxClockSkew: defaultMaxClockSkew, } } @@ -217,8 +229,6 @@ func (s *SignatureV4) Verify(r *http.Request) error { } else if payloadHash != "" { // Client provided a specific payload hash - verify it matches the body // Protect against memory exhaustion by limiting body read size - const maxBodyReadSize = 10 * 1024 * 1024 // 10 MiB max for signature calculation - if r.ContentLength > maxBodyReadSize { // Request is too large to buffer for hash verification // Client should use X-Amz-Content-Sha256: UNSIGNED-PAYLOAD header @@ -246,16 +256,13 @@ func (s *SignatureV4) Verify(r *http.Request) error { } } else { // Empty body - verify client sent empty payload hash - emptyHash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - if subtle.ConstantTimeCompare([]byte(emptyHash), []byte(payloadHash)) != 1 { + if subtle.ConstantTimeCompare([]byte(emptyPayloadHash), []byte(payloadHash)) != 1 { return ErrPayloadHashMismatch } } } else { // No payload hash header provided - we need to read body to calculate hash // Protect against memory exhaustion by limiting body read size - const maxBodyReadSize = 10 * 1024 * 1024 // 10 MiB max for signature calculation - if r.ContentLength > maxBodyReadSize { // Request is too large to buffer for signature verification // Client should use X-Amz-Content-Sha256: UNSIGNED-PAYLOAD header @@ -276,7 +283,7 @@ func (s *SignatureV4) Verify(r *http.Request) error { r.Body = io.NopCloser(bytes.NewReader(payload)) payloadHash = hex.EncodeToString(sha256Hash(payload)) } else { - payloadHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" // empty payload hash + payloadHash = emptyPayloadHash } } diff --git a/internal/backend/awss3/object.go b/internal/backend/awss3/object.go index 58fa0ef..bd6cd17 100644 --- a/internal/backend/awss3/object.go +++ b/internal/backend/awss3/object.go @@ -18,7 +18,7 @@ import ( // PutObject uploads an object using the S3 Upload Manager for efficient streaming. func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.Reader, size int64, metadata map[string]string) (*types.ObjectInfo, error) { // Extract Content-Type from metadata - contentType := "application/octet-stream" + contentType := types.DefaultContentType if ct, ok := metadata["Content-Type"]; ok { contentType = ct delete(metadata, "Content-Type") @@ -38,8 +38,8 @@ func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.R // Use S3 Upload Manager which handles streaming and multipart uploads automatically uploader := manager.NewUploader(b.client, func(u *manager.Uploader) { - u.PartSize = 5 * 1024 * 1024 // 5 MiB minimum part size - u.Concurrency = 1 // Single concurrency to minimize memory + u.PartSize = types.PartSize5MiB + u.Concurrency = 1 // Single concurrency to minimize memory }) input := &s3.PutObjectInput{ @@ -131,7 +131,7 @@ func (b *Backend) GetObject(ctx context.Context, bucket, key string, opts *types } // Get content type - contentType := "application/octet-stream" + contentType := types.DefaultContentType if resp.ContentType != nil { contentType = *resp.ContentType } @@ -189,7 +189,7 @@ func (b *Backend) HeadObject(ctx context.Context, bucket, key string) (*types.Ob } // Get content type - contentType := "application/octet-stream" + contentType := types.DefaultContentType if resp.ContentType != nil { contentType = *resp.ContentType } diff --git a/internal/backend/azblob/bucket.go b/internal/backend/azblob/bucket.go index 71c3f7b..d6d8cc3 100644 --- a/internal/backend/azblob/bucket.go +++ b/internal/backend/azblob/bucket.go @@ -106,8 +106,8 @@ func (b *Backend) ListObjects(ctx context.Context, bucket string, opts types.Lis containerClient := b.client.ServiceClient().NewContainerClient(bucket) // Set up list options - maxResults := int32(1000) - if opts.MaxKeys > 0 && opts.MaxKeys < 1000 { + maxResults := int32(types.DefaultMaxKeys) + if opts.MaxKeys > 0 && opts.MaxKeys < types.DefaultMaxKeys { maxResults = int32(opts.MaxKeys) } @@ -178,8 +178,8 @@ func (b *Backend) ListObjects(ctx context.Context, bucket string, opts types.Lis func (b *Backend) listObjectsHierarchical(ctx context.Context, bucket string, opts types.ListObjectsOptions) (*types.ListObjectsResult, error) { containerClient := b.client.ServiceClient().NewContainerClient(bucket) - maxResults := int32(1000) - if opts.MaxKeys > 0 && opts.MaxKeys < 1000 { + maxResults := int32(types.DefaultMaxKeys) + if opts.MaxKeys > 0 && opts.MaxKeys < types.DefaultMaxKeys { maxResults = int32(opts.MaxKeys) } diff --git a/internal/backend/azblob/multipart.go b/internal/backend/azblob/multipart.go index a06d57a..76b456e 100644 --- a/internal/backend/azblob/multipart.go +++ b/internal/backend/azblob/multipart.go @@ -467,7 +467,7 @@ func (b *Backend) ListParts(ctx context.Context, bucket, key, uploadID string, o // Apply pagination maxParts := opts.MaxParts if maxParts <= 0 { - maxParts = 1000 + maxParts = types.DefaultMaxParts } // Filter parts after the marker @@ -600,7 +600,7 @@ func (b *Backend) ListMultipartUploads(ctx context.Context, bucket string, opts // Apply max uploads limit maxUploads := opts.MaxUploads if maxUploads <= 0 { - maxUploads = 1000 + maxUploads = types.DefaultMaxUploads } isTruncated := false diff --git a/internal/backend/azblob/object.go b/internal/backend/azblob/object.go index 64677f8..df2ac87 100644 --- a/internal/backend/azblob/object.go +++ b/internal/backend/azblob/object.go @@ -18,7 +18,7 @@ import ( // PutObject uploads a blob using streaming upload with MD5 calculation. func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.Reader, size int64, metadata map[string]string) (*types.ObjectInfo, error) { // Extract Content-Type from metadata - contentType := "application/octet-stream" + contentType := types.DefaultContentType if ct, ok := metadata["Content-Type"]; ok { contentType = ct delete(metadata, "Content-Type") @@ -42,8 +42,8 @@ func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.R // Upload using UploadStream which handles chunking internally _, err := blockBlobClient.UploadStream(ctx, teeReader, &blockblob.UploadStreamOptions{ - BlockSize: 4 * 1024 * 1024, // 4 MiB blocks - Concurrency: 1, // Single concurrency to minimize memory + BlockSize: types.ChunkSize4MiB, + Concurrency: 1, // Single concurrency to minimize memory Metadata: azureMetadata, HTTPHeaders: &blob.HTTPHeaders{ BlobContentType: &contentType, @@ -155,7 +155,7 @@ func (b *Backend) GetObject(ctx context.Context, bucket, key string, opts *types } // Get content type - contentType := "application/octet-stream" + contentType := types.DefaultContentType if resp.ContentType != nil { contentType = *resp.ContentType } @@ -219,7 +219,7 @@ func (b *Backend) HeadObject(ctx context.Context, bucket, key string) (*types.Ob } // Get content type - contentType := "application/octet-stream" + contentType := types.DefaultContentType if props.ContentType != nil { contentType = *props.ContentType } diff --git a/internal/backend/disk/bucket.go b/internal/backend/disk/bucket.go index 98c8619..5e8f3c0 100644 --- a/internal/backend/disk/bucket.go +++ b/internal/backend/disk/bucket.go @@ -28,12 +28,12 @@ func (b *Backend) CreateBucket(ctx context.Context, bucket string) error { } // Create bucket directory structure - if err := os.MkdirAll(bucketPath, 0755); err != nil { + if err := os.MkdirAll(bucketPath, dirPermissions); err != nil { return fmt.Errorf("failed to create bucket directory: %w", err) } // Create subdirectory for multipart uploads - if err := os.MkdirAll(filepath.Join(bucketPath, multipartDir), 0755); err != nil { + if err := os.MkdirAll(filepath.Join(bucketPath, multipartDir), dirPermissions); err != nil { // Cleanup on failure _ = os.RemoveAll(bucketPath) return fmt.Errorf("failed to create bucket subdirectory %s: %w", multipartDir, err) @@ -81,7 +81,7 @@ func (b *Backend) DeleteBucket(ctx context.Context, bucket string) error { } return nil } - if strings.HasPrefix(d.Name(), ".") && strings.HasSuffix(d.Name(), ".object.json") { + if strings.HasPrefix(d.Name(), ".") && strings.HasSuffix(d.Name(), objectMetadataSuffix) { hasObjects = true return filepath.SkipAll // Stop walking, we found an object } @@ -180,7 +180,7 @@ func (b *Backend) ListObjects(ctx context.Context, bucket string, opts types.Lis maxKeys := opts.MaxKeys if maxKeys <= 0 { - maxKeys = 1000 + maxKeys = types.DefaultMaxKeys } startAfter := opts.Marker @@ -274,7 +274,7 @@ func (b *Backend) ListObjects(ctx context.Context, bucket string, opts types.Lis } // Only process hidden .object.json files - if !strings.HasPrefix(d.Name(), ".") || !strings.HasSuffix(d.Name(), ".object.json") { + if !strings.HasPrefix(d.Name(), ".") || !strings.HasSuffix(d.Name(), objectMetadataSuffix) { return nil } diff --git a/internal/backend/disk/disk.go b/internal/backend/disk/disk.go index 2d5ca50..52b0965 100644 --- a/internal/backend/disk/disk.go +++ b/internal/backend/disk/disk.go @@ -16,6 +16,28 @@ const ( // Directory names for internal storage multipartDir = ".multipart" // Directory to store multipart uploads locksDir = ".locks" // Directory to store lock files + + // Metadata file suffixes + bucketMetadataSuffix = ".bucket.json" // Bucket metadata file + objectMetadataSuffix = ".object.json" // Object metadata file + tagsMetadataSuffix = ".tags.json" // Tags metadata file + + // Lock file names + globalLockFile = "global.lock" // Global lock file name + bucketLockSuffix = ".lock" // Bucket lock file suffix + + // Multipart upload file names + uploadMetadataFile = "upload.json" // Upload metadata file name + partFilePrefix = "part-" // Part file prefix + partFileFormat = "part-%05d" // Part file format string + partMetadataFormat = "part-%05d.json" // Part metadata format string + + // Temp file pattern + tempFilePattern = ".tmp-*" // Pattern for temporary files + + // File permissions + dirPermissions = 0755 // Directory permissions (rwxr-xr-x) + filePermissions = 0644 // File permissions (rw-r--r--) ) // bucketMetadata stores bucket-level metadata @@ -49,12 +71,12 @@ func New(cfg config.DiskConfig, logger *zap.Logger) (*Backend, error) { } // Ensure root directory exists - if err := os.MkdirAll(rootDir, 0755); err != nil { + if err := os.MkdirAll(rootDir, dirPermissions); err != nil { return nil, err } // Ensure locks directory exists - if err := os.MkdirAll(filepath.Join(rootDir, locksDir), 0755); err != nil { + if err := os.MkdirAll(filepath.Join(rootDir, locksDir), dirPermissions); err != nil { return nil, err } @@ -107,17 +129,17 @@ func (b *Backend) objectMetadataPath(bucket, key string) string { objPath := b.objectFilePath(bucket, key) dir := filepath.Dir(objPath) base := filepath.Base(objPath) - return filepath.Join(dir, "."+base+".object.json") + return filepath.Join(dir, "."+base+objectMetadataSuffix) } // tagsPath returns the path to a bucket's tags file func (b *Backend) tagsPath(bucket string) string { - return filepath.Join(b.bucketPath(bucket), ".tags.json") + return filepath.Join(b.bucketPath(bucket), tagsMetadataSuffix) } // bucketMetadataPath returns the path to a bucket's metadata file func (b *Backend) bucketMetadataPath(bucket string) string { - return filepath.Join(b.bucketPath(bucket), ".bucket.json") + return filepath.Join(b.bucketPath(bucket), bucketMetadataSuffix) } // sanitizePathComponent sanitizes a key for use as a filesystem path @@ -166,7 +188,7 @@ func writeJSON(path string, v interface{}) error { if err != nil { return err } - return os.WriteFile(path, data, 0644) + return os.WriteFile(path, data, filePermissions) } // lockFile represents a file-based lock that works across processes @@ -176,12 +198,12 @@ type lockFile struct { // bucketLockPath returns the path to a bucket's lock file func (b *Backend) bucketLockPath(bucket string) string { - return filepath.Join(b.rootDir, locksDir, bucket+".lock") + return filepath.Join(b.rootDir, locksDir, bucket+bucketLockSuffix) } // globalLockPath returns the path to the global lock file (for ListBuckets, etc.) func (b *Backend) globalLockPath() string { - return filepath.Join(b.rootDir, locksDir, "global.lock") + return filepath.Join(b.rootDir, locksDir, globalLockFile) } // lockBucket acquires an exclusive lock on a bucket for write operations diff --git a/internal/backend/disk/multipart.go b/internal/backend/disk/multipart.go index 257090f..9d0e704 100644 --- a/internal/backend/disk/multipart.go +++ b/internal/backend/disk/multipart.go @@ -39,17 +39,17 @@ func (b *Backend) uploadPath(bucket, uploadID string) string { // partFilePath returns the path to a part's data file func (b *Backend) partFilePath(bucket, uploadID string, partNumber int) string { - return filepath.Join(b.uploadPath(bucket, uploadID), fmt.Sprintf("part-%05d", partNumber)) + return filepath.Join(b.uploadPath(bucket, uploadID), fmt.Sprintf(partFileFormat, partNumber)) } // partMetadataPath returns the path to a part's metadata file func (b *Backend) partMetadataPath(bucket, uploadID string, partNumber int) string { - return filepath.Join(b.uploadPath(bucket, uploadID), fmt.Sprintf("part-%05d.json", partNumber)) + return filepath.Join(b.uploadPath(bucket, uploadID), fmt.Sprintf(partMetadataFormat, partNumber)) } // uploadMetadataPath returns the path to an upload's metadata file func (b *Backend) uploadMetadataPath(bucket, uploadID string) string { - return filepath.Join(b.uploadPath(bucket, uploadID), "upload.json") + return filepath.Join(b.uploadPath(bucket, uploadID), uploadMetadataFile) } // CreateMultipartUpload initiates a multipart upload @@ -70,7 +70,7 @@ func (b *Backend) CreateMultipartUpload(ctx context.Context, bucket, key string, // Create upload directory uploadPath := b.uploadPath(bucket, uploadID) - if err := os.MkdirAll(uploadPath, 0755); err != nil { + if err := os.MkdirAll(uploadPath, dirPermissions); err != nil { return "", fmt.Errorf("failed to create upload directory: %w", err) } @@ -113,7 +113,7 @@ func (b *Backend) UploadPart(ctx context.Context, bucket, key, uploadID string, partPath := b.partFilePath(bucket, uploadID, partNumber) // Create temp file for atomic write - tmpFile, err := os.CreateTemp(filepath.Dir(partPath), ".tmp-*") + tmpFile, err := os.CreateTemp(filepath.Dir(partPath), tempFilePattern) if err != nil { return nil, fmt.Errorf("failed to create temp file: %w", err) } @@ -213,11 +213,11 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, bucket, key, uplo // Create final object file objectPath := b.objectFilePath(bucket, key) - if err := os.MkdirAll(filepath.Dir(objectPath), 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(objectPath), dirPermissions); err != nil { return nil, fmt.Errorf("failed to create object directory: %w", err) } - tmpFile, err := os.CreateTemp(filepath.Dir(objectPath), ".tmp-*") + tmpFile, err := os.CreateTemp(filepath.Dir(objectPath), tempFilePattern) if err != nil { return nil, fmt.Errorf("failed to create temp file: %w", err) } @@ -255,7 +255,7 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, bucket, key, uplo } // Determine content type - contentType := "application/octet-stream" + contentType := types.DefaultContentType if uploadMeta.Metadata != nil { if ct, ok := uploadMeta.Metadata["Content-Type"]; ok { contentType = ct @@ -274,7 +274,7 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, bucket, key, uplo } metaPath := b.objectMetadataPath(bucket, key) - if err := os.MkdirAll(filepath.Dir(metaPath), 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(metaPath), dirPermissions); err != nil { return nil, fmt.Errorf("failed to create metadata directory: %w", err) } if err := writeJSON(metaPath, &meta); err != nil { @@ -347,7 +347,7 @@ func (b *Backend) ListParts(ctx context.Context, bucket, key, uploadID string, o var parts []types.PartInfo for _, entry := range entries { - if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") || entry.Name() == "upload.json" { + if entry.IsDir() || !strings.HasPrefix(entry.Name(), partFilePrefix) || !strings.HasSuffix(entry.Name(), ".json") { continue } @@ -382,7 +382,7 @@ func (b *Backend) ListParts(ctx context.Context, bucket, key, uploadID string, o // Apply max parts limit maxParts := opts.MaxParts if maxParts <= 0 { - maxParts = 1000 + maxParts = types.DefaultMaxParts } if len(result.Parts) > maxParts { @@ -428,7 +428,7 @@ func (b *Backend) ListMultipartUploads(ctx context.Context, bucket string, opts } var meta multipartUploadMetadata - metaPath := filepath.Join(multipartPath, entry.Name(), "upload.json") + metaPath := filepath.Join(multipartPath, entry.Name(), uploadMetadataFile) if err := readJSON(metaPath, &meta); err != nil { continue } @@ -492,7 +492,7 @@ func (b *Backend) ListMultipartUploads(ctx context.Context, bucket string, opts // Apply max uploads limit maxUploads := opts.MaxUploads if maxUploads <= 0 { - maxUploads = 1000 + maxUploads = types.DefaultMaxUploads } if len(result.Uploads) > maxUploads { diff --git a/internal/backend/disk/object.go b/internal/backend/disk/object.go index c99b844..10f15f4 100644 --- a/internal/backend/disk/object.go +++ b/internal/backend/disk/object.go @@ -30,15 +30,15 @@ func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.R metaPath := b.objectMetadataPath(bucket, key) // Ensure parent directories exist - if err := os.MkdirAll(filepath.Dir(objectPath), 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(objectPath), dirPermissions); err != nil { return nil, fmt.Errorf("failed to create object directory: %w", err) } - if err := os.MkdirAll(filepath.Dir(metaPath), 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(metaPath), dirPermissions); err != nil { return nil, fmt.Errorf("failed to create metadata directory: %w", err) } // Create temp file for atomic write - tmpFile, err := os.CreateTemp(filepath.Dir(objectPath), ".tmp-*") + tmpFile, err := os.CreateTemp(filepath.Dir(objectPath), tempFilePattern) if err != nil { return nil, fmt.Errorf("failed to create temp file: %w", err) } @@ -68,7 +68,7 @@ func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.R } // Determine content type - contentType := "application/octet-stream" + contentType := types.DefaultContentType if metadata != nil { if ct, ok := metadata["Content-Type"]; ok { contentType = ct @@ -328,7 +328,7 @@ func (b *Backend) objectTagsPath(bucket, key string) string { objPath := b.objectFilePath(bucket, key) dir := filepath.Dir(objPath) base := filepath.Base(objPath) - return filepath.Join(dir, "."+base+".tags.json") + return filepath.Join(dir, "."+base+tagsMetadataSuffix) } // GetObjectTagging retrieves the tags for an object diff --git a/internal/backend/gcs/bucket.go b/internal/backend/gcs/bucket.go index 2b0bb87..75a88cb 100644 --- a/internal/backend/gcs/bucket.go +++ b/internal/backend/gcs/bucket.go @@ -136,8 +136,8 @@ func (b *Backend) ListObjects(ctx context.Context, bucket string, opts types.Lis CommonPrefixes: []string{}, } - maxKeys := 1000 - if opts.MaxKeys > 0 && opts.MaxKeys < 1000 { + maxKeys := types.DefaultMaxKeys + if opts.MaxKeys > 0 && opts.MaxKeys < types.DefaultMaxKeys { maxKeys = opts.MaxKeys } diff --git a/internal/backend/gcs/multipart.go b/internal/backend/gcs/multipart.go index 47622e2..a7951f1 100644 --- a/internal/backend/gcs/multipart.go +++ b/internal/backend/gcs/multipart.go @@ -149,7 +149,7 @@ func (b *Backend) CreateMultipartUpload(ctx context.Context, bucket, key string, metaObj := bucketHandle.Object(getMetadataObjectName(key, uploadID)) writer := metaObj.NewWriter(ctx) writer.ContentType = "application/json" - writer.ChunkSize = 4 * 1024 * 1024 // 4 MiB chunks + writer.ChunkSize = types.ChunkSize4MiB if _, err := writer.Write(metaData); err != nil { _ = writer.Close() @@ -199,8 +199,8 @@ func (b *Backend) UploadPart(ctx context.Context, bucket, key, uploadID string, // Upload the part as an object partObj := bucketHandle.Object(getPartObjectName(key, uploadID, partNumber)) writer := partObj.NewWriter(ctx) - writer.ContentType = "application/octet-stream" - writer.ChunkSize = 4 * 1024 * 1024 // 4 MiB chunks + writer.ContentType = types.DefaultContentType + writer.ChunkSize = types.ChunkSize4MiB if _, err := io.Copy(writer, bytes.NewReader(data)); err != nil { _ = writer.Close() @@ -264,8 +264,8 @@ func (b *Backend) UploadPartCopy(ctx context.Context, bucket, key, uploadID stri dstBucketHandle := b.client.Bucket(bucket) partObj := dstBucketHandle.Object(getPartObjectName(key, uploadID, partNumber)) writer := partObj.NewWriter(ctx) - writer.ContentType = "application/octet-stream" - writer.ChunkSize = 4 * 1024 * 1024 // 4 MiB chunks + writer.ContentType = types.DefaultContentType + writer.ChunkSize = types.ChunkSize4MiB if _, err := io.Copy(writer, bytes.NewReader(data)); err != nil { _ = writer.Close() @@ -615,7 +615,7 @@ func (b *Backend) ListParts(ctx context.Context, bucket, key, uploadID string, o // Apply pagination maxParts := opts.MaxParts if maxParts <= 0 { - maxParts = 1000 + maxParts = types.DefaultMaxParts } // Filter parts after the marker @@ -749,7 +749,7 @@ func (b *Backend) ListMultipartUploads(ctx context.Context, bucket string, opts // Apply max uploads limit maxUploads := opts.MaxUploads if maxUploads <= 0 { - maxUploads = 1000 + maxUploads = types.DefaultMaxUploads } isTruncated := false diff --git a/internal/backend/gcs/object.go b/internal/backend/gcs/object.go index 03300f9..a5208e5 100644 --- a/internal/backend/gcs/object.go +++ b/internal/backend/gcs/object.go @@ -27,7 +27,7 @@ func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.R } // Extract Content-Type from metadata - contentType := "application/octet-stream" + contentType := types.DefaultContentType if ct, ok := metadata["Content-Type"]; ok { contentType = ct delete(metadata, "Content-Type") @@ -50,7 +50,7 @@ func (b *Backend) PutObject(ctx context.Context, bucket, key string, reader io.R writer.Metadata = gcsMetadata // Use 4 MiB chunks for consistency with other backends. // This enables resumable uploads for large files. - writer.ChunkSize = 4 * 1024 * 1024 // 4 MiB chunks + writer.ChunkSize = types.ChunkSize4MiB // Stream data to GCS while computing MD5 if _, err := io.Copy(writer, teeReader); err != nil { diff --git a/internal/backend/types/types.go b/internal/backend/types/types.go index b3e3c54..947d79f 100644 --- a/internal/backend/types/types.go +++ b/internal/backend/types/types.go @@ -7,6 +7,26 @@ import ( "time" ) +const ( + // DefaultContentType is the default MIME type for objects without a Content-Type + DefaultContentType = "application/octet-stream" + + // DefaultMaxKeys is the default maximum number of keys returned in ListObjects + DefaultMaxKeys = 1000 + + // DefaultMaxParts is the default maximum number of parts returned in ListParts + DefaultMaxParts = 1000 + + // DefaultMaxUploads is the default maximum number of uploads returned in ListMultipartUploads + DefaultMaxUploads = 1000 + + // ChunkSize4MiB is the chunk/block size for streaming uploads (4 MiB) + ChunkSize4MiB = 4 * 1024 * 1024 + + // PartSize5MiB is the minimum part size for multipart uploads (5 MiB) + PartSize5MiB = 5 * 1024 * 1024 +) + var ( // ErrBucketNotFound is returned when a bucket does not exist ErrBucketNotFound = errors.New("bucket not found")