BLA-4251 classify backend uploads with create-if-absent semantics

cache/s3proxy/s3proxy.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
 	"path"
 
 	"github.com/buchgr/bazel-remote/v2/cache"
@@ -219,6 +220,18 @@ func (c *s3Cache) UploadFile(item backendproxy.UploadReq) {
 		c.logMissingRequiredStoragePrefix("UPLOAD", item.Kind, item.Hash)
 	}
 	objectKey := c.objectKeyForPrefix(prefix, item.Hash, item.Kind)
+
+	opts := minio.PutObjectOptions{
+		UserMetadata: map[string]string{
+			"Content-Type": "application/octet-stream",
+		},
+	}
+	// Create-if-absent: a backend upload only counts toward the storage footprint
+	// when it stores a net-new object. If the object already exists, MinIO rejects
+	// the conditional PUT with a precondition failure and we classify it as
+	// already_exists instead of created.
+	opts.SetMatchETagExcept("*")
+
 	_, err := c.mcore.PutObject(
 		context.Background(),
 		c.bucket,        // bucketName
@@ -227,21 +240,41 @@ func (c *s3Cache) UploadFile(item backendproxy.UploadReq) {
 		item.SizeOnDisk, // objectSize
 		"",              // md5base64
 		"",              // sha256
-		minio.PutObjectOptions{
-			UserMetadata: map[string]string{
-				"Content-Type": "application/octet-stream",
-			},
-		}, // metadata
+		opts,            // metadata
 	)
 
 	logResponse(c.accessLogger, "UPLOAD", c.bucket, objectKey, err)
-	if err != nil {
-		c.observeUpload(context.Background(), item, "error", "s3_put_failed")
-	}
+
+	status, reason := classifyUploadOutcome(err)
+	c.observeUpload(context.Background(), item, status, reason)
 
 	item.Rc.Close()
 }
 
+// classifyUploadOutcome maps a create-if-absent backend PutObject result to a
+// terminal storage-accounting status. Only a net-new object (no error) is
+// "created" and counts toward the footprint. A precondition failure means the
+// object already existed: RFC-compliant servers return 412 PreconditionFailed,
+// while some older MinIO releases return 304 NotModified, so both map to
+// already_exists. Anything else is a genuine failure.
+//
+// The failure status is "error" (not "failed") to match the shared build-cache
+// status taxonomy: FA emits "error" (buildCacheStatusError) and the web reader
+// counts status IN ('error', 'rejected', 'dropped') as failures
+// (BuildCacheMetricsService::FAILURE_STATUSES). A divergent "failed" would be
+// silently dropped by those consumers.
+func classifyUploadOutcome(err error) (status string, reason string) {
+	if err == nil {
+		return "created", ""
+	}
+	switch minio.ToErrorResponse(err).StatusCode {
+	case http.StatusPreconditionFailed, http.StatusNotModified:
+		return "already_exists", "precondition_failed"
+	default:
+		return "error", "s3_put_failed"
+	}
+}
+
 func (c *s3Cache) Put(ctx context.Context, kind cache.EntryKind, hash string, logicalSize int64, sizeOnDisk int64, rc io.ReadCloser) {
 	if c.uploadQueue == nil {
 		rc.Close()
@@ -269,7 +302,7 @@ func (c *s3Cache) Put(ctx context.Context, kind cache.EntryKind, hash string, lo
 			Status: "dropped",
 			Reason: "upload_queue_full",
 			Ops:    1,
-			Bytes:  nonNegativeUint64(logicalSize),
+			Bytes:  nonNegativeUint64(sizeOnDisk),
 		})
 		rc.Close()
 	}
@@ -359,12 +392,14 @@ func (c *s3Cache) Contains(ctx context.Context, kind cache.EntryKind, hash strin
 }
 
 func (c *s3Cache) observeUpload(ctx context.Context, item backendproxy.UploadReq, status string, reason string) {
+	// SizeOnDisk is the compressed/stored byte count, matching what MinIO actually
+	// persists; the footprint accumulator and the MinIO drift scan share this unit.
 	cache.ObserveOperation(cache.WithMetricsLabels(ctx, item.MetricsLabels), c.observer, cache.OperationOutcome{
 		Method: "backend_upload",
 		Status: status,
 		Reason: reason,
 		Ops:    1,
-		Bytes:  nonNegativeUint64(item.LogicalSize),
+		Bytes:  nonNegativeUint64(item.SizeOnDisk),
 	})
 }
 

cache/s3proxy/s3proxy_test.go

@@ -5,11 +5,13 @@ import (
 	"context"
 	"io"
 	stdlog "log"
+	"net/http"
 	"strings"
 	"testing"
 
 	"github.com/buchgr/bazel-remote/v2/cache"
 	"github.com/buchgr/bazel-remote/v2/utils/backendproxy"
+	"github.com/minio/minio-go/v7"
 )
 
 type recordingObserver struct {
@@ -232,6 +234,8 @@ func TestPutRecordsUploadQueueDrop(t *testing.T) {
 		VMID:           "vm-123",
 		JobID:          "job-456",
 	})
+	// Put is called with logicalSize=4, sizeOnDisk=4; the dropped outcome now
+	// reports SizeOnDisk bytes.
 	c.Put(ctx, cache.CAS, hash, 4, 4, io.NopCloser(strings.NewReader("blob")))
 
 	if len(observer.outcomes) != 1 {
@@ -241,16 +245,21 @@ func TestPutRecordsUploadQueueDrop(t *testing.T) {
 	if outcome.Method != "backend_upload" || outcome.Status != "dropped" || outcome.Reason != "upload_queue_full" {
 		t.Fatalf("unexpected outcome: %+v", outcome)
 	}
+	if outcome.Bytes != 4 {
+		t.Fatalf("dropped outcome bytes = %d, want 4 (SizeOnDisk)", outcome.Bytes)
+	}
 	if outcome.Labels.RepositoryID != "717982840" || outcome.Labels.JobID != "job-456" {
 		t.Fatalf("unexpected labels: %+v", outcome.Labels)
 	}
 }
 
-func TestObserveUploadRecordsBackendUploadError(t *testing.T) {
+func TestObserveUploadReportsSizeOnDisk(t *testing.T) {
 	observer := &recordingObserver{}
 	c := &s3Cache{observer: observer}
 	c.observeUpload(context.Background(), backendproxy.UploadReq{
-		LogicalSize: 12,
+		// LogicalSize must be ignored; only SizeOnDisk (stored bytes) is reported.
+		LogicalSize: 99,
+		SizeOnDisk:  12,
 		Kind:        cache.CAS,
 		MetricsLabels: cache.MetricsLabels{
 			RepositoryID: "717982840",
@@ -266,13 +275,38 @@ func TestObserveUploadRecordsBackendUploadError(t *testing.T) {
 		t.Fatalf("unexpected outcome: %+v", outcome)
 	}
 	if outcome.Bytes != 12 {
-		t.Fatalf("outcome bytes = %d, want 12", outcome.Bytes)
+		t.Fatalf("outcome bytes = %d, want 12 (SizeOnDisk, not LogicalSize)", outcome.Bytes)
 	}
 	if outcome.Labels.RepositoryID != "717982840" || outcome.Labels.JobID != "job-456" {
 		t.Fatalf("unexpected labels: %+v", outcome.Labels)
 	}
 }
 
+func TestClassifyUploadOutcome(t *testing.T) {
+	testCases := []struct {
+		name           string
+		err            error
+		expectedStatus string
+		expectedReason string
+	}{
+		{"net-new object", nil, "created", ""},
+		{"precondition failed 412", minio.ErrorResponse{StatusCode: http.StatusPreconditionFailed}, "already_exists", "precondition_failed"},
+		{"not modified 304 (older minio)", minio.ErrorResponse{StatusCode: http.StatusNotModified}, "already_exists", "precondition_failed"},
+		{"server error", minio.ErrorResponse{StatusCode: http.StatusInternalServerError}, "error", "s3_put_failed"},
+		{"non-minio error", errNotFound, "error", "s3_put_failed"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			status, reason := classifyUploadOutcome(tc.err)
+			if status != tc.expectedStatus || reason != tc.expectedReason {
+				t.Fatalf("classifyUploadOutcome(%v) = (%q, %q), want (%q, %q)",
+					tc.err, status, reason, tc.expectedStatus, tc.expectedReason)
+			}
+		})
+	}
+}
+
 func TestLogMissingRequiredStoragePrefix(t *testing.T) {
 	var buf bytes.Buffer
 	c := &s3Cache{

deps.bzl

@@ -339,12 +339,7 @@ def go_dependencies():
         sum = "h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=",
         version = "v0.6.0",
     )
-    go_repository(
-        name = "com_github_google_gofuzz",
-        importpath = "github.com/google/gofuzz",
-        sum = "h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=",
-        version = "v1.0.0",
-    )
+
     go_repository(
         name = "com_github_google_s2a_go",
         importpath = "github.com/google/s2a-go",
@@ -573,14 +568,8 @@ def go_dependencies():
     go_repository(
         name = "com_github_minio_minio_go_v7",
         importpath = "github.com/minio/minio-go/v7",
-        sum = "h1:l8AnsQFyY1xiwa/DaQskY4NXSLA2yrGsW5iD9nRPVS0=",
-        version = "v7.0.69",
-    )
-    go_repository(
-        name = "com_github_minio_sha256_simd",
-        importpath = "github.com/minio/sha256-simd",
-        sum = "h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=",
-        version = "v1.0.1",
+        sum = "h1:ZSbxs2BfJensLyHdVOgHv+pfmvxYraaUy07ER04dWnA=",
+        version = "v7.0.72",
     )
 
     go_repository(

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/klauspost/compress v1.17.8
-	github.com/minio/minio-go/v7 v7.0.69
+	github.com/minio/minio-go/v7 v7.0.72
 	github.com/mostynb/go-grpc-compression v1.2.2
 	github.com/mostynb/zstdpool-syncpool v0.0.13
 	github.com/prometheus/client_golang v1.19.0
@@ -47,16 +47,13 @@ require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/sha256-simd v1.0.1 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/prometheus/common v0.52.3 // indirect
 	github.com/prometheus/procfs v0.13.0 // indirect

go.sum

@@ -34,6 +34,8 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
 github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
@@ -42,7 +44,6 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
@@ -52,8 +53,6 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877 h1:O7syWuYGzre3s73s+NkgB8e0ZvsIVhT/zxNU7V1gHK8=
 github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
 github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -67,15 +66,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.69 h1:l8AnsQFyY1xiwa/DaQskY4NXSLA2yrGsW5iD9nRPVS0=
-github.com/minio/minio-go/v7 v7.0.69/go.mod h1:XAvOPJQ5Xlzk5o3o/ArO2NMbhSGkimC+bpW/ngRKDmQ=
-github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
-github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/minio/minio-go/v7 v7.0.72 h1:ZSbxs2BfJensLyHdVOgHv+pfmvxYraaUy07ER04dWnA=
+github.com/minio/minio-go/v7 v7.0.72/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo=
 github.com/mostynb/go-grpc-compression v1.2.2 h1:XaDbnRvt2+1vgr0b/l0qh4mJAfIxE0bKXtz2Znl3GGI=
 github.com/mostynb/go-grpc-compression v1.2.2/go.mod h1:GOCr2KBxXcblCuczg3YdLQlcin1/NfyDA348ckuCH6w=
 github.com/mostynb/zstdpool-syncpool v0.0.13 h1:AIzAvQ9hNum4Fh5jYXyfZTd2aDi1leq7grKDkVZX4+s=
@@ -109,7 +101,6 @@ github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0=
 github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=