From b40fc5cbd6610ac7e0349b46f78d9ed971f447ae Mon Sep 17 00:00:00 2001 From: Josh Heinrichs Date: Sun, 22 Mar 2026 12:55:24 -0600 Subject: [PATCH] Add support for reading from gcs substituter --- src/libstore/gcs-binary-cache-store.cc | 139 ++++++++++++++++++ src/libstore/gcs-binary-cache-store.md | 48 ++++++ src/libstore/http-binary-cache-store.cc | 6 +- .../nix/store/gcs-binary-cache-store.hh | 54 +++++++ src/libstore/include/nix/store/meson.build | 1 + src/libstore/meson.build | 1 + 6 files changed, 246 insertions(+), 3 deletions(-) create mode 100644 src/libstore/gcs-binary-cache-store.cc create mode 100644 src/libstore/gcs-binary-cache-store.md create mode 100644 src/libstore/include/nix/store/gcs-binary-cache-store.hh diff --git a/src/libstore/gcs-binary-cache-store.cc b/src/libstore/gcs-binary-cache-store.cc new file mode 100644 index 000000000000..9549f57341a4 --- /dev/null +++ b/src/libstore/gcs-binary-cache-store.cc @@ -0,0 +1,139 @@ +#include "nix/store/gcs-binary-cache-store.hh" +#include "nix/store/gcs-creds.hh" +#include "nix/store/gcs-url.hh" +#include "nix/store/http-binary-cache-store.hh" +#include "nix/store/store-registration.hh" +#include "nix/util/error.hh" +#include "nix/util/logging.hh" +#include "nix/util/serialise.hh" + +namespace nix { + +MakeError(UploadToGcs, Error); + +class GcsBinaryCacheStore : public virtual HttpBinaryCacheStore +{ +public: + GcsBinaryCacheStore(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , HttpBinaryCacheStore{config} + , gcsConfig{config} + { + } + + void upsertFile( + const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint) override; + +private: + ref gcsConfig; + + /** + * Uploads a file to GCS using the JSON API simple upload. + * Supports files up to 5 GiB, which is sufficient for binary cache objects. + * + * @see https://cloud.google.com/storage/docs/uploading-objects#upload-object-json + */ + void upload( + std::string_view path, + RestartableSource & source, + uint64_t sizeHint, + std::string_view mimeType, + std::optional headers); +}; + +void GcsBinaryCacheStore::upsertFile( + const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint) +{ + try { + if (auto compressionMethod = getCompressionMethod(path)) { + CompressedSource compressed(source, *compressionMethod); + Headers headers = {{"Content-Encoding", *compressionMethod}}; + upload(path, compressed, compressed.size(), mimeType, std::move(headers)); + } else { + upload(path, source, sizeHint, mimeType, std::nullopt); + } + } catch (FileTransferError & e) { + UploadToGcs err(e.message()); + err.addTrace({}, "while uploading to GCS binary cache at '%s'", config->cacheUri.to_string()); + throw err; + } +} + +void GcsBinaryCacheStore::upload( + std::string_view path, + RestartableSource & source, + uint64_t sizeHint, + std::string_view mimeType, + std::optional headers) +{ + debug("uploading to GCS '%s' (%d bytes)", path, sizeHint); + + auto parsedGcs = ParsedGcsURL::parse(config->cacheUri); + + // Build the upload URL using the GCS JSON API: + // POST https://storage.googleapis.com/upload/storage/v1/b/{bucket}/o?uploadType=media&name={object} + ParsedURL uploadUrl; + uploadUrl.scheme = "https"; + uploadUrl.authority = ParsedURL::Authority{.host = "storage.googleapis.com"}; + uploadUrl.path = {"", "upload", "storage", "v1", "b", parsedGcs.bucket, "o"}; + uploadUrl.query["uploadType"] = "media"; + uploadUrl.query["name"] = std::string{path}; + + FileTransferRequest req(uploadUrl); + req.method = HttpMethod::Post; + + // Authenticate with write scope via OAuth2 + auto token = getGcsCredentialsProvider()->getAccessToken(/* writable = */ true); + req.bearerToken = std::move(token); + + if (headers) { + req.headers.reserve(req.headers.size() + headers->size()); + std::move(headers->begin(), headers->end(), std::back_inserter(req.headers)); + } + + if (auto storageClass = gcsConfig->storageClass.get()) { + req.headers.emplace_back("x-goog-storage-class", *storageClass); + } + + req.data = {sizeHint, source}; + req.mimeType = mimeType; + + getFileTransfer()->upload(req); +} + +StringSet GcsBinaryCacheStoreConfig::uriSchemes() +{ + return {"gs"}; +} + +GcsBinaryCacheStoreConfig::GcsBinaryCacheStoreConfig( + std::string_view scheme, std::string_view _cacheUri, const Params & params) + : StoreConfig(params) + , HttpBinaryCacheStoreConfig(scheme, _cacheUri, params) +{ + assert(cacheUri.scheme == "gs"); +} + +std::string GcsBinaryCacheStoreConfig::getHumanReadableURI() const +{ + return getReference().render(); +} + +std::string GcsBinaryCacheStoreConfig::doc() +{ + return +#include "gcs-binary-cache-store.md" + ; +} + +ref GcsBinaryCacheStoreConfig::openStore() const +{ + auto sharedThis = std::const_pointer_cast( + std::static_pointer_cast(shared_from_this())); + return make_ref(ref{sharedThis}); +} + +static RegisterStoreImplementation registerGcsBinaryCacheStore; + +} // namespace nix diff --git a/src/libstore/gcs-binary-cache-store.md b/src/libstore/gcs-binary-cache-store.md new file mode 100644 index 000000000000..1747e7099e64 --- /dev/null +++ b/src/libstore/gcs-binary-cache-store.md @@ -0,0 +1,48 @@ +R"( + +**Store URL format**: `gs://`*bucket-name* + +This store allows reading and writing a binary cache stored in a Google Cloud Storage (GCS) bucket. +This store shares many idioms with the [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md). + +For a bucket named `example-nix-cache`, the binary cache URL is . + +### Authentication + +Nix uses [Application Default Credentials (ADC)](https://cloud.google.com/docs/authentication/application-default-credentials) +for authenticating requests to Google Cloud Storage: + +1. The `GOOGLE_APPLICATION_CREDENTIALS` environment variable pointing to a service account key file +2. User credentials from `gcloud auth application-default login` (at `~/.config/gcloud/application_default_credentials.json`) +3. The GCE metadata server (automatic on Compute Engine, GKE, Cloud Run, etc.) + +For read operations, Nix requests the `devstorage.read_only` OAuth2 scope. +For write operations (uploads), Nix requests the `devstorage.read_write` scope. + +### Anonymous reads + +If your bucket is publicly accessible and does not require authentication, +you can use the [HTTP Binary Cache Store] with +`https://storage.googleapis.com/example-nix-cache` instead of `gs://example-nix-cache`. + +### Examples + +- To use a GCS bucket as a substituter: + + ```console + $ nix build --substituters 'gs://example-nix-cache' --no-require-sigs ... + ``` + +- To upload to a GCS binary cache: + + ```console + $ nix copy nixpkgs.hello --to 'gs://example-nix-cache' + ``` + +- To specify a storage class for uploads: + + ```console + $ nix copy nixpkgs.hello --to 'gs://example-nix-cache?storage-class=NEARLINE' + ``` + +)" diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index ef6ae92a44d3..5caf2b466550 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -186,10 +186,10 @@ FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path) (note the query param) and that gets passed here. */ auto result = parseURLRelative(path, cacheUriWithTrailingSlash); - /* For S3 URLs, preserve query parameters from the base URL when the + /* For S3/GCS URLs, preserve query parameters from the base URL when the relative path doesn't have its own query parameters. This is needed - to preserve S3-specific parameters like endpoint and region. */ - if (config->cacheUri.scheme == "s3" && result.query.empty()) { + to preserve scheme-specific parameters like endpoint and region. */ + if ((config->cacheUri.scheme == "s3" || config->cacheUri.scheme == "gs") && result.query.empty()) { result.query = config->cacheUri.query; } diff --git a/src/libstore/include/nix/store/gcs-binary-cache-store.hh b/src/libstore/include/nix/store/gcs-binary-cache-store.hh new file mode 100644 index 000000000000..eb6e736e12b1 --- /dev/null +++ b/src/libstore/include/nix/store/gcs-binary-cache-store.hh @@ -0,0 +1,54 @@ +#pragma once +///@file + +#include "nix/store/config.hh" +#include "nix/store/http-binary-cache-store.hh" + +namespace nix { + +struct GcsBinaryCacheStoreConfig : HttpBinaryCacheStoreConfig +{ + using HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig; + + GcsBinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); + + const Setting> projectId{ + this, + std::nullopt, + "project-id", + R"( + The Google Cloud project ID. When not set (default), the project is + inferred from the service account credentials or GCE metadata. + )"}; + + const Setting> storageClass{ + this, + std::nullopt, + "storage-class", + R"( + The GCS storage class to use for uploaded objects. When not set (default), + uses the bucket's default storage class. Valid values include: + - STANDARD (frequently accessed data) + - NEARLINE (accessed less than once per 30 days) + - COLDLINE (accessed less than once per 90 days) + - ARCHIVE (accessed less than once per year) + + See Google Cloud Storage documentation for detailed storage class descriptions: + https://cloud.google.com/storage/docs/storage-classes + )"}; + + static const std::string name() + { + return "GCS Binary Cache Store"; + } + + static StringSet uriSchemes(); + + static std::string doc(); + + std::string getHumanReadableURI() const override; + + ref openStore() const override; +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index fad5bcbec003..4b02b1183fbb 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -43,6 +43,7 @@ headers = [ config_pub_h ] + files( 'export-import.hh', 'filetransfer.hh', 'gc-store.hh', + 'gcs-binary-cache-store.hh', 'gcs-creds.hh', 'gcs-url.hh', 'globals.hh', diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 5da970e0299b..13a4ac765705 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -372,6 +372,7 @@ endif # GCS (Google Cloud Storage) support sources += files( + 'gcs-binary-cache-store.cc', 'gcs-creds.cc', 'gcs-url.cc', )