Add support for reading from gcs substituter

src/libstore/gcs-binary-cache-store.cc

@@ -0,0 +1,139 @@
+#include "nix/store/gcs-binary-cache-store.hh"
+#include "nix/store/gcs-creds.hh"
+#include "nix/store/gcs-url.hh"
+#include "nix/store/http-binary-cache-store.hh"
+#include "nix/store/store-registration.hh"
+#include "nix/util/error.hh"
+#include "nix/util/logging.hh"
+#include "nix/util/serialise.hh"
+
+namespace nix {
+
+MakeError(UploadToGcs, Error);
+
+class GcsBinaryCacheStore : public virtual HttpBinaryCacheStore
+{
+public:
+    GcsBinaryCacheStore(ref<GcsBinaryCacheStoreConfig> config)
+        : Store{*config}
+        , BinaryCacheStore{*config}
+        , HttpBinaryCacheStore{config}
+        , gcsConfig{config}
+    {
+    }
+
+    void upsertFile(
+        const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint) override;
+
+private:
+    ref<GcsBinaryCacheStoreConfig> gcsConfig;
+
+    /**
+     * Uploads a file to GCS using the JSON API simple upload.
+     * Supports files up to 5 GiB, which is sufficient for binary cache objects.
+     *
+     * @see https://cloud.google.com/storage/docs/uploading-objects#upload-object-json
+     */
+    void upload(
+        std::string_view path,
+        RestartableSource & source,
+        uint64_t sizeHint,
+        std::string_view mimeType,
+        std::optional<Headers> headers);
+};
+
+void GcsBinaryCacheStore::upsertFile(
+    const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint)
+{
+    try {
+        if (auto compressionMethod = getCompressionMethod(path)) {
+            CompressedSource compressed(source, *compressionMethod);
+            Headers headers = {{"Content-Encoding", *compressionMethod}};
+            upload(path, compressed, compressed.size(), mimeType, std::move(headers));
+        } else {
+            upload(path, source, sizeHint, mimeType, std::nullopt);
+        }
+    } catch (FileTransferError & e) {
+        UploadToGcs err(e.message());
+        err.addTrace({}, "while uploading to GCS binary cache at '%s'", config->cacheUri.to_string());
+        throw err;
+    }
+}
+
+void GcsBinaryCacheStore::upload(
+    std::string_view path,
+    RestartableSource & source,
+    uint64_t sizeHint,
+    std::string_view mimeType,
+    std::optional<Headers> headers)
+{
+    debug("uploading to GCS '%s' (%d bytes)", path, sizeHint);
+
+    auto parsedGcs = ParsedGcsURL::parse(config->cacheUri);
+
+    // Build the upload URL using the GCS JSON API:
+    // POST https://storage.googleapis.com/upload/storage/v1/b/{bucket}/o?uploadType=media&name={object}
+    ParsedURL uploadUrl;
+    uploadUrl.scheme = "https";
+    uploadUrl.authority = ParsedURL::Authority{.host = "storage.googleapis.com"};
+    uploadUrl.path = {"", "upload", "storage", "v1", "b", parsedGcs.bucket, "o"};
+    uploadUrl.query["uploadType"] = "media";
+    uploadUrl.query["name"] = std::string{path};
+
+    FileTransferRequest req(uploadUrl);
+    req.method = HttpMethod::Post;
+
+    // Authenticate with write scope via OAuth2
+    auto token = getGcsCredentialsProvider()->getAccessToken(/* writable = */ true);
+    req.bearerToken = std::move(token);
+
+    if (headers) {
+        req.headers.reserve(req.headers.size() + headers->size());
+        std::move(headers->begin(), headers->end(), std::back_inserter(req.headers));
+    }
+
+    if (auto storageClass = gcsConfig->storageClass.get()) {
+        req.headers.emplace_back("x-goog-storage-class", *storageClass);
+    }
+
+    req.data = {sizeHint, source};
+    req.mimeType = mimeType;
+
+    getFileTransfer()->upload(req);
+}
+
+StringSet GcsBinaryCacheStoreConfig::uriSchemes()
+{
+    return {"gs"};
+}
+
+GcsBinaryCacheStoreConfig::GcsBinaryCacheStoreConfig(
+    std::string_view scheme, std::string_view _cacheUri, const Params & params)
+    : StoreConfig(params)
+    , HttpBinaryCacheStoreConfig(scheme, _cacheUri, params)
+{
+    assert(cacheUri.scheme == "gs");
+}
+
+std::string GcsBinaryCacheStoreConfig::getHumanReadableURI() const
+{
+    return getReference().render();
+}
+
+std::string GcsBinaryCacheStoreConfig::doc()
+{
+    return
+#include "gcs-binary-cache-store.md"
+        ;
+}
+
+ref<Store> GcsBinaryCacheStoreConfig::openStore() const
+{
+    auto sharedThis = std::const_pointer_cast<GcsBinaryCacheStoreConfig>(
+        std::static_pointer_cast<const GcsBinaryCacheStoreConfig>(shared_from_this()));
+    return make_ref<GcsBinaryCacheStore>(ref{sharedThis});
+}
+
+static RegisterStoreImplementation<GcsBinaryCacheStoreConfig> registerGcsBinaryCacheStore;
+
+} // namespace nix

src/libstore/gcs-binary-cache-store.md

@@ -0,0 +1,48 @@
+R"(
+
+**Store URL format**: `gs://`*bucket-name*
+
+This store allows reading and writing a binary cache stored in a Google Cloud Storage (GCS) bucket.
+This store shares many idioms with the [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md).
+
+For a bucket named `example-nix-cache`, the binary cache URL is <gs://example-nix-cache>.
+
+### Authentication
+
+Nix uses [Application Default Credentials (ADC)](https://cloud.google.com/docs/authentication/application-default-credentials)
+for authenticating requests to Google Cloud Storage:
+
+1. The `GOOGLE_APPLICATION_CREDENTIALS` environment variable pointing to a service account key file
+2. User credentials from `gcloud auth application-default login` (at `~/.config/gcloud/application_default_credentials.json`)
+3. The GCE metadata server (automatic on Compute Engine, GKE, Cloud Run, etc.)
+
+For read operations, Nix requests the `devstorage.read_only` OAuth2 scope.
+For write operations (uploads), Nix requests the `devstorage.read_write` scope.
+
+### Anonymous reads
+
+If your bucket is publicly accessible and does not require authentication,
+you can use the [HTTP Binary Cache Store] with
+`https://storage.googleapis.com/example-nix-cache` instead of `gs://example-nix-cache`.
+
+### Examples
+
+- To use a GCS bucket as a substituter:
+
+  ```console
+  $ nix build --substituters 'gs://example-nix-cache' --no-require-sigs ...
+  ```
+
+- To upload to a GCS binary cache:
+
+  ```console
+  $ nix copy nixpkgs.hello --to 'gs://example-nix-cache'
+  ```
+
+- To specify a storage class for uploads:
+
+  ```console
+  $ nix copy nixpkgs.hello --to 'gs://example-nix-cache?storage-class=NEARLINE'
+  ```
+
+)"

src/libstore/http-binary-cache-store.cc

@@ -186,10 +186,10 @@ FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path)
        (note the query param) and that gets passed here. */
     auto result = parseURLRelative(path, cacheUriWithTrailingSlash);
 
-    /* For S3 URLs, preserve query parameters from the base URL when the
+    /* For S3/GCS URLs, preserve query parameters from the base URL when the
        relative path doesn't have its own query parameters. This is needed
-       to preserve S3-specific parameters like endpoint and region. */
-    if (config->cacheUri.scheme == "s3" && result.query.empty()) {
+       to preserve scheme-specific parameters like endpoint and region. */
+    if ((config->cacheUri.scheme == "s3" || config->cacheUri.scheme == "gs") && result.query.empty()) {
         result.query = config->cacheUri.query;
     }
 

src/libstore/include/nix/store/gcs-binary-cache-store.hh

@@ -0,0 +1,54 @@
+#pragma once
+///@file
+
+#include "nix/store/config.hh"
+#include "nix/store/http-binary-cache-store.hh"
+
+namespace nix {
+
+struct GcsBinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
+{
+    using HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig;
+
+    GcsBinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params);
+
+    const Setting<std::optional<std::string>> projectId{
+        this,
+        std::nullopt,
+        "project-id",
+        R"(
+          The Google Cloud project ID. When not set (default), the project is
+          inferred from the service account credentials or GCE metadata.
+        )"};
+
+    const Setting<std::optional<std::string>> storageClass{
+        this,
+        std::nullopt,
+        "storage-class",
+        R"(
+          The GCS storage class to use for uploaded objects. When not set (default),
+          uses the bucket's default storage class. Valid values include:
+          - STANDARD (frequently accessed data)
+          - NEARLINE (accessed less than once per 30 days)
+          - COLDLINE (accessed less than once per 90 days)
+          - ARCHIVE (accessed less than once per year)
+
+          See Google Cloud Storage documentation for detailed storage class descriptions:
+          https://cloud.google.com/storage/docs/storage-classes
+        )"};
+
+    static const std::string name()
+    {
+        return "GCS Binary Cache Store";
+    }
+
+    static StringSet uriSchemes();
+
+    static std::string doc();
+
+    std::string getHumanReadableURI() const override;
+
+    ref<Store> openStore() const override;
+};
+
+} // namespace nix

src/libstore/include/nix/store/meson.build

@@ -43,6 +43,7 @@ headers = [ config_pub_h ] + files(
   'export-import.hh',
   'filetransfer.hh',
   'gc-store.hh',
+  'gcs-binary-cache-store.hh',
   'gcs-creds.hh',
   'gcs-url.hh',
   'globals.hh',

src/libstore/meson.build

@@ -372,6 +372,7 @@ endif
 
 # GCS (Google Cloud Storage) support
 sources += files(
+  'gcs-binary-cache-store.cc',
   'gcs-creds.cc',
   'gcs-url.cc',
 )