diff --git a/.clang-tidy b/.clang-tidy deleted file mode 100644 index 0887b867087a..000000000000 --- a/.clang-tidy +++ /dev/null @@ -1,3 +0,0 @@ -# We use pointers to aggregates in a couple of places, intentionally. -# void * would look weird. -Checks: '-bugprone-sizeof-expression' diff --git a/.clang-tidy b/.clang-tidy new file mode 120000 index 000000000000..871c4b83d57d --- /dev/null +++ b/.clang-tidy @@ -0,0 +1 @@ +nix-meson-build-support/common/clang-tidy/.clang-tidy \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5ace4600a1f2..1880a89da556 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,11 @@ updates: directory: "/" schedule: interval: "weekly" + - package-ecosystem: "nix" + directory: "/" + schedule: + interval: "weekly" + groups: + flake-inputs: + patterns: + - "*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96f0c44671e6..133d19c2173c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,7 +157,7 @@ jobs: run: mkdir -p ./artifacts - name: Fetch artifacts - uses: actions/download-artifact@v7 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: downloaded - name: Move downloaded artifacts to artifacts directory diff --git a/.gitignore b/.gitignore index 96c134335232..c0d09b7e189c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,23 +2,8 @@ /build # Meson creates this file too src/.wraplock - -# /tests/functional/ -/tests/functional/common/subst-vars.sh -/tests/functional/restricted-innocent -/tests/functional/debugger-test-out -/tests/functional/test-libstoreconsumer/test-libstoreconsumer -/tests/functional/nix-shell - -# /tests/functional/lang/ -/tests/functional/lang/*.out -/tests/functional/lang/*.out.xml -/tests/functional/lang/*.err -/tests/functional/lang/*.ast - -# /tests/functional/cli-characterisation/ -/tests/functional/cli-characterisation/*.out -/tests/functional/cli-characterisation/*.err +# Python bytecode (clang-tidy runner scripts) +__pycache__/ /outputs diff --git a/.version b/.version index fb61f64bad1e..d07233cc933c 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.34.8 +2.35.1 diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index 7d206750fcfe..94327adc9f0d 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -57,8 +57,6 @@ rec { nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; }; mesonComponentOverrides = lib.composeManyExtensions componentOverrides; - # Unclear how to make Perl bindings work with a dynamically linked ASAN. - nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings; } ); @@ -78,6 +76,7 @@ rec { installerScriptForGHA = hydraJobs.installerScriptForGHA.${system}; nixpkgsLibTests = hydraJobs.tests.nixpkgsLibTests.${system}; nixpkgsLibTestsLazy = hydraJobs.tests.nixpkgsLibTestsLazy.${system}; + filetransfer-retry-backoff = hydraJobs.tests.filetransfer-retry-backoff.${system}; rl-next = pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${../../../doc/manual/rl-next} >$out ''; diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index e66611affe08..a58d4f79e132 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -46,10 +46,12 @@ let ## Settings - ${showSettings { - prefix = "store-${slug}"; - inherit inlineHTML; - } settings} + ${replaceStrings [ "@store-slug@" ] [ "store-${slug}" ] ( + showSettings { + prefix = "store-${slug}"; + inherit inlineHTML; + } settings + )} ''; experimentalFeatureNote = optionalString (experimentalFeature != null) '' diff --git a/doc/manual/meson.build b/doc/manual/meson.build index d842fe30ea0a..1edf11afc1f4 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -1,7 +1,7 @@ project( 'nix-manual', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -231,7 +231,9 @@ foreach command : nix_nested_manpages meson.current_source_dir() / 'source', meson.current_build_dir() / 'source', doc_url, - meson.current_source_dir() / 'source/command-ref' / command[0] / (page + '.md'), + meson.current_source_dir() / 'source/command-ref' / command[0] / ( + page + '.md' + ), '@OUTPUT0@', ], input : [ @@ -320,8 +322,6 @@ nix3_manpages = [ 'nix3-provenance-show', 'nix3-provenance-verify', 'nix3-ps', - 'nix3-realisation', - 'nix3-realisation-info', 'nix3-registry', 'nix3-registry-add', 'nix3-registry-list', @@ -336,6 +336,8 @@ nix3_manpages = [ 'nix3-store-add', 'nix3-store-add-file', 'nix3-store-add-path', + 'nix3-store-build-trace-info', + 'nix3-store-build-trace', 'nix3-store-cat', 'nix3-store-copy-log', 'nix3-store-copy-sigs', diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 9e69156ac0ca..c66163e285e3 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -145,6 +145,21 @@ mkMesonDerivation (finalAttrs: { # Exclude undocumented builtins ".*/language/builtins\\.html#builtins-addErrorContext" ".*/language/builtins\\.html#builtins-appendContext" + # `print.html` aggregates content from all pages, including + # the JSON schema pages and builtins pages excluded above, + # so it inherits the same broken fragment links. + ".*/print\\.html#algorithm" + ".*/print\\.html#root" + ".*/print\\.html#builtins-addErrorContext" + ".*/print\\.html#builtins-appendContext" + ".*/print\\.html#derivations_pattern1_structuredAttrs_additionalProperties" + ".*/print\\.html#structuredAttrs_additionalProperties" + ]; + # `404.html` uses `` so that absolute links + # work on the deployed site. Lychee cannot resolve `/` against + # a local file path, so skip the file entirely. + exclude_path = [ + ".*/404\\.html" ]; }; }; diff --git a/doc/manual/redirects.json b/doc/manual/redirects.json index 07a6f36627ff..76ce242178f3 100644 --- a/doc/manual/redirects.json +++ b/doc/manual/redirects.json @@ -283,7 +283,7 @@ "string-literal": "string-literals.html" }, "language/derivations.html": { - "builder-execution": "../store/building.html#builder-execution" + "builder-execution": "../store/building.html" }, "installation/installing-binary.html": { "uninstalling": "uninstall.html" @@ -297,7 +297,6 @@ "debugging-failing-functional-tests": "testing.html#debugging-failing-functional-tests", "integration-tests": "testing.html#integration-tests", "installer-tests": "testing.html#installer-tests", - "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", "characterization-testing": "testing.html#characterisation-testing-unit", @@ -307,6 +306,9 @@ "reverting": "contributing.html#reverting", "branches": "contributing.html#branches" }, + "store/store-path.html": { + "store-directory": "#store-directory-path" + }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", "package-attribute-set": "#package", diff --git a/doc/manual/remove_before_wrapper.py b/doc/manual/remove_before_wrapper.py index 6da4c19b0ce7..a0fcb6a55776 100644 --- a/doc/manual/remove_before_wrapper.py +++ b/doc/manual/remove_before_wrapper.py @@ -22,7 +22,7 @@ def main(): shutil.rmtree(output, ignore_errors=True) shutil.rmtree(output_temp, ignore_errors=True) - # Execute nix command with `--write-to` tempary output + # Execute nix command with `--write-to` temporary output nix_command_write_to = nix_command + ['--write-to', output_temp] subprocess.run(nix_command_write_to, check=True) diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 7b5bf0a8b0cf..e808ca66b887 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -14,9 +14,10 @@ - [Nix Store](store/index.md) - [File System Object](store/file-system-object.md) - [Content-Addressing File System Objects](store/file-system-object/content-address.md) + - [Exposing in OS File Systems](store/file-system-object/os-file-system.md) - [Store Object](store/store-object.md) - [Content-Addressing Store Objects](store/store-object/content-address.md) - - [Store Path](store/store-path.md) + - [Store Path and Store Directory](store/store-path.md) - [Store Derivation and Deriving Path](store/derivation/index.md) - [Derivation Outputs and Types of Derivations](store/derivation/outputs/index.md) - [Content-addressing derivation outputs](store/derivation/outputs/content-address.md) @@ -118,8 +119,10 @@ - [Hash](protocols/json/hash.md) - [Content Address](protocols/json/content-address.md) - [Store Path](protocols/json/store-path.md) + - [Signature](protocols/json/signature.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation/index.md) + - [Resolved Derivation](protocols/json/derivation/resolved.md) - [Derivation Options](protocols/json/derivation/options.md) - [Deriving Path](protocols/json/deriving-path.md) - [Build Trace Entry](protocols/json/build-trace-entry.md) @@ -128,7 +131,9 @@ - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) - - [Nix Cache Info Format](protocols/nix-cache-info.md) + - [Binary Cache](protocols/binary-cache/index.md) + - [`nix-cache-info` Format](protocols/binary-cache/nix-cache-info.md) + - [`.narinfo` Format](protocols/binary-cache/narinfo.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Nix32 Encoding](protocols/nix32.md) - [`builtins.wasm` Host Interface](protocols/wasm.md) @@ -142,8 +147,9 @@ - [Debugging](development/debugging.md) - [Documentation](development/documentation.md) - [CLI guideline](development/cli-guideline.md) - - [JSON guideline](development/json-guideline.md) + - [Data Modeling Guidelines](development/data-modeling.md) - [C++ style guide](development/cxx.md) + - [Static Analysis](development/static-analysis.md) - [Experimental Features](development/experimental-features.md) - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) @@ -211,6 +217,7 @@ - [Release 3.0.0 (2025-03-04)](release-notes-determinate/rl-3.0.0.md) - [Nix Release Notes](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.35 (2026-06-22)](release-notes/rl-2.35.md) - [Release 2.34 (2026-02-27)](release-notes/rl-2.34.md) - [Release 2.33 (2025-12-09)](release-notes/rl-2.33.md) - [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 07b3130f9ce9..74815df282c8 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -27,7 +27,8 @@ /contributing/documentation /development/documentation 301! /contributing/experimental-features /development/experimental-features 301! /contributing/cli-guideline /development/cli-guideline 301! -/contributing/json-guideline /development/json-guideline 301! +/contributing/json-guideline /development/data-modeling 301! +/development/json-guideline /development/data-modeling 301! /contributing/cxx /development/cxx 301! /expressions/expression-language /language/ 301! @@ -36,6 +37,7 @@ /expressions/language-values /language/values 301! /expressions/* /language/:splat 301! /language/values /language/types 301! +/language/values.html /language/types 301! /language/constructs /language/syntax 301! /language/builtin-constants /language/builtins 301! @@ -46,6 +48,7 @@ /package-management/package-management /package-management 301! /package-management/s3-substituter /store/types/s3-binary-cache-store 301! +/protocols/nix-cache-info /protocols/binary-cache/nix-cache-info 301! /protocols/protocols /protocols 301! /json/* /protocols/json/:splat 301! diff --git a/doc/manual/source/command-ref/conf-file-prefix.md b/doc/manual/source/command-ref/conf-file-prefix.md index 627806cfbda8..53a3761fd4b1 100644 --- a/doc/manual/source/command-ref/conf-file-prefix.md +++ b/doc/manual/source/command-ref/conf-file-prefix.md @@ -8,19 +8,26 @@ Nix supports a variety of configuration settings, which are read from configurat ## Configuration file -By default Nix reads settings from the following places, in that order: +By default Nix reads settings from each of the following places. +Settings are applied on top of one another, so later settings overwrite earlier entries. -1. The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set. +1. The system-wide configuration file `nix.conf` in the configuration directory. + + The configuration directory defaults to `${sysconfdir}/nix` (i.e. `/etc/nix` on most Unix systems) or `%PROGRAMDATA%\nix\conf` on Windows, and can be overridden with [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR). Values loaded in this file are not forwarded to the Nix daemon. The client assumes that the daemon has already loaded them. -1. If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path separated by `:` will be loaded in reverse order. +1. If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path in the list will be loaded in reverse order. + The list separator is `:` on Unix and `;` on Windows. + + Otherwise it will look for `nix.conf` in: - Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` and [`XDG_CONFIG_HOME`](./env-common.md#env-XDG_CONFIG_HOME). - If unset, `XDG_CONFIG_DIRS` defaults to `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + - The [user configuration directory](./env-common.md#user-conf-dir) + - On Unix additionally, `nix/nix.conf` under each directory in [`XDG_CONFIG_DIRS`](./env-common.md#env-XDG_CONFIG_DIRS) 1. If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file. + In particular, settings are separated by newlines, just as in the configuration file. ### File format diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index 35d682949ba8..cc7fe77eae56 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -1,34 +1,25 @@ # Common Environment Variables -Most Nix commands interpret the following environment variables: +Most Nix commands interpret the following environment variables. -- [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) +## Configuration environment variables - Indicator that tells if the current environment was set up by - `nix-shell`. It can have the values `pure` or `impure`. +The following environment variables affect how Nix loads its configuration. +See the [configuration file](@docroot@/command-ref/conf-file.md#configuration-file) documentation for full details. -- [`NIX_PATH`](#env-NIX_PATH) +- [`NIX_CONF_DIR`](#env-NIX_CONF_DIR) - A colon-separated list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + Overrides the system configuration directory. - This environment variable overrides the value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path). +- [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES) - It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + Overrides the user configuration file locations. - > **Example** - > - > ```bash - > $ export NIX_PATH=`/home/eelco/Dev:nixos-config=/etc/nixos - > ``` +- [`NIX_CONFIG`](#env-NIX_CONFIG) - If `NIX_PATH` is set to an empty string, resolving search paths will always fail. + Provides configuration settings inline. - > **Example** - > - > ```bash - > $ NIX_PATH= nix-instantiate --eval '' - > error: file 'nixpkgs' was not found in the Nix search path (add it using $NIX_PATH or -I) - > ``` +## Store setting environment variables - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) @@ -55,37 +46,34 @@ Most Nix commands interpret the following environment variables: - [`NIX_STORE_DIR`](#env-NIX_STORE_DIR) - Overrides the location of the Nix store (default `prefix/store`). + Overrides the location of the Nix store. + On Unix, the default is `/nix/store`, set at compile time by the `libstore:store-dir` build option. + On Windows, there is no compile-time setting; the default is `%PROGRAMDATA%\nix\store`, determined at runtime using the [`%PROGRAMDATA%` Known Folder](#known-folders). -- [`NIX_LOG_DIR`](#env-NIX_LOG_DIR) - - Overrides the location of the Nix log directory (default - `prefix/var/log/nix`). + See the [Store Types] chapter; each store has a `store` setting linking back to this as its default. -- [`NIX_STATE_DIR`](#env-NIX_STATE_DIR) - - Overrides the location of the Nix state directory (default - `prefix/var/nix`). +- [`NIX_LOG_DIR`](#env-NIX_LOG_DIR) -- [`NIX_CONF_DIR`](#env-NIX_CONF_DIR) + Overrides the location of the Nix log directory. + On Unix, the default is `/nix/var/log/nix`, set at compile time by the `log-dir` build option. + On Windows, there is no compile-time setting; the default is `%PROGRAMDATA%\nix\log`, determined at runtime using the [`%PROGRAMDATA%` Known Folder](#known-folders). - Overrides the location of the system Nix configuration directory - (default `sysconfdir/nix`, i.e. `/etc/nix` on most systems). + The [Local Store], [Local Daemon Store], and [Experimental SSH Store with filesystem mounted] have per-store settings that override this. -- [`NIX_CONFIG`](#env-NIX_CONFIG) +- [`NIX_STATE_DIR`](#env-NIX_STATE_DIR) - Applies settings from Nix configuration from the environment. - The content is treated as if it was read from a Nix configuration file. - Settings are separated by the newline character. + Overrides the location of the Nix state directory. + On Unix, the default is `${localstatedir}/nix`, where `localstatedir` is a compile-time build option defaulting to `/nix/var`. + On Windows, there is no compile-time setting; the default is `%PROGRAMDATA%\nix\state`, determined at runtime using the [`%PROGRAMDATA%` Known Folder](#known-folders). -- [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES) + The [Local Store], [Local Daemon Store], and [Experimental SSH Store with filesystem mounted] have per-store settings that override this. - Overrides the location of the Nix user configuration files to load from. +- [`NIX_DAEMON_SOCKET_PATH`](#env-NIX_DAEMON_SOCKET_PATH) - The default are the locations according to the [XDG Base Directory Specification]. - See the [XDG Base Directories](#xdg-base-directories) sub-section for details. + Overrides the path to the Unix domain socket used to communicate with the Nix daemon. + Defaults to `daemon-socket/socket` within the state directory (see [`NIX_STATE_DIR`](#env-NIX_STATE_DIR)). - The variable is treated as a list separated by the `:` token. + See the [Local Daemon Store] documentation for details on how the socket path is resolved. - [`TMPDIR`](#env-TMPDIR) @@ -95,12 +83,15 @@ Most Nix commands interpret the following environment variables: - [`NIX_REMOTE`](#env-NIX_REMOTE) - This variable should be set to `daemon` if you want to use the Nix - daemon to execute Nix operations. This is necessary in [multi-user - Nix installations](@docroot@/installation/nix-security.md#multi-user-model). If the Nix - daemon's Unix socket is at some non-standard path, this variable - should be set to `unix://path/to/socket`. Otherwise, it should be - left unset. + Overrides the [`store`](@docroot@/command-ref/conf-file.md#conf-store) setting. + +[Store Types]: @docroot@/store/types/index.md +[store URL format]: @docroot@/store/types/index.md#store-url-format +[Local Store]: @docroot@/store/types/local-store.md +[Local Daemon Store]: @docroot@/store/types/local-daemon-store.md +[Experimental SSH Store with filesystem mounted]: @docroot@/store/types/experimental-ssh-store-with-filesystem-mounted.md + +## Nix language evaluator settings - [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS) @@ -120,32 +111,87 @@ Most Nix commands interpret the following environment variables: 384 MiB. Setting it to a low value reduces memory consumption, but will increase runtime due to the overhead of garbage collection. -## XDG Base Directories +- [`NIX_PATH`](#env-NIX_PATH) -Nix follows the [XDG Base Directory Specification]. + A colon-separated list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). -For backwards compatibility, Nix commands will follow the standard only when [`use-xdg-base-directories`] is enabled. -[New Nix commands](@docroot@/command-ref/new-cli/nix.md) (experimental) conform to the standard by default. + This environment variable overrides the value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path). -The following environment variables are used to determine locations of various state and configuration files: + It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). -- [`XDG_CONFIG_HOME`]{#env-XDG_CONFIG_HOME} (default `~/.config`) -- [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`) -- [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`) + > **Example** + > + > ```bash + > $ export NIX_PATH=/home/eelco/Dev:nixos-config=/etc/nixos + > ``` -[XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html -[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories + If `NIX_PATH` is set to an empty string, resolving search paths will always fail. + + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval '' + > error: file 'nixpkgs' was not found in the Nix search path (add it using $NIX_PATH or -I) + > ``` + +## User Directories + +Nix supports per-user **configuration**, **state**, and **cache** directories. +The following environment variables override the locations for these directories: + +- [user configuration directory]{#user-conf-dir}: [`NIX_CONFIG_HOME`]{#env-NIX_CONFIG_HOME} +- user state directory: [`NIX_STATE_HOME`]{#env-NIX_STATE_HOME} +- user cache directory: [`NIX_CACHE_HOME`]{#env-NIX_CACHE_HOME} -In addition, setting the following environment variables overrides the XDG base directories: +When these are not set, the defaults depend on the platform: -- [`NIX_CONFIG_HOME`]{#env-NIX_CONFIG_HOME} (default `$XDG_CONFIG_HOME/nix`) -- [`NIX_STATE_HOME`]{#env-NIX_STATE_HOME} (default `$XDG_STATE_HOME/nix`) -- [`NIX_CACHE_HOME`]{#env-NIX_CACHE_HOME} (default `$XDG_CACHE_HOME/nix`) +- On Unix, the [XDG base directories](#xdg-base-directories): `$XDG_CONFIG_HOME/nix`, `$XDG_STATE_HOME/nix`, `$XDG_CACHE_HOME/nix` +- On Windows, the [Windows Known Folders](#known-folders): `%APPDATA%\nix\config`, `%LOCALAPPDATA%\nix\state`, `%LOCALAPPDATA%\nix\cache` -When [`use-xdg-base-directories`] is enabled, the configuration directory is: +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories + +For backwards compatibility, legacy Nix commands (e.g. `nix-env`, `nix-channel`) use dotfiles in `$HOME` instead of these directories unless [`use-xdg-base-directories`] is enabled. +[New Nix commands](@docroot@/command-ref/new-cli/nix.md) (experimental) use the proper directories by default. + +When [`use-xdg-base-directories`] is enabled, the configuration directory is resolved as: 1. `$NIX_CONFIG_HOME`, if it is defined -2. Otherwise, `$XDG_CONFIG_HOME/nix`, if `XDG_CONFIG_HOME` is defined -3. Otherwise, `~/.config/nix`. +2. Otherwise, the platform default (e.g. `$XDG_CONFIG_HOME/nix` on Unix) Likewise for the state and cache directories. + +## Miscellaneous environment variables + +- [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) + + Indicator that tells if the current environment was set up by + `nix-shell`. It can have the values `pure` or `impure`. + +## Appendix: OS-specific conventions + +This information is not Nix-specific, but is referenced above. + +### Unix: XDG Base Directories {#xdg-base-directories} + +The [XDG Base Directory Specification] defines standard locations for user-specific configuration, state, and cache files on Unix systems. + +[XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + +The following environment variables are used: + +- [`XDG_CONFIG_HOME`]{#env-XDG_CONFIG_HOME} (default `~/.config`) +- [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`) +- [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`) +- [`XDG_CONFIG_DIRS`]{#env-XDG_CONFIG_DIRS} (default `/etc/xdg`) — colon-separated list of additional configuration base directories, searched after `XDG_CONFIG_HOME` + +### Windows: Known Folders {#known-folders} + +On Windows, [Known Folders][windows-known-folders] provide standard locations for application data on Windows. + +[windows-known-folders]: https://learn.microsoft.com/en-us/windows/win32/shell/known-folders + +The relevant folders are: + +- [`%APPDATA%`]{#env-APPDATA} — per-user roaming application data +- [`%LOCALAPPDATA%`]{#env-LOCALAPPDATA} — per-user local application data +- [`%PROGRAMDATA%`]{#env-PROGRAMDATA} — system-wide application data diff --git a/doc/manual/source/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md index 763179b8ee18..07229255e7cd 100644 --- a/doc/manual/source/command-ref/nix-collect-garbage.md +++ b/doc/manual/source/command-ref/nix-collect-garbage.md @@ -62,9 +62,9 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. - - [`--max-freed`](#opt-max-freed) *bytes* +- [`--max-freed`](#opt-max-freed) *bytes* - + Keep deleting paths until at least *bytes* bytes have been deleted, then stop. The argument *bytes* can be followed by the diff --git a/doc/manual/source/command-ref/nix-hash.md b/doc/manual/source/command-ref/nix-hash.md index 7c17ce9095b4..c1a4251b0973 100644 --- a/doc/manual/source/command-ref/nix-hash.md +++ b/doc/manual/source/command-ref/nix-hash.md @@ -45,20 +45,20 @@ md5sum`. - `--base32` - Print the hash in a base-32 representation rather than hexadecimal. - This base-32 representation is more compact and can be used in Nix + Print the hash in [Nix32](@docroot@/protocols/nix32.md) representation rather than hexadecimal. + This representation is more compact and can be used in Nix expressions (such as in calls to `fetchurl`). - `--base64` - Similar to --base32, but print the hash in a base-64 representation, - which is more compact than the base-32 one. + Similar to `--base32`, but print the hash in a [Base64](https://en.wikipedia.org/wiki/Base64) representation, + which is more compact than the Nix32 one. - `--sri` - Print the hash in SRI format with base-64 encoding. + Print the hash in [SRI](@docroot@/glossary.md#gloss-sri) format with Base64 encoding. The type of hash algorithm will be prepended to the hash string, - followed by a hyphen (-) and the base-64 hash body. + followed by a hyphen (-) and the Base64 hash body. - `--truncate` @@ -71,18 +71,18 @@ md5sum`. - `--to-base16` - Don’t hash anything, but convert the base-32 hash representation + Don’t hash anything, but convert the [Nix32](@docroot@/protocols/nix32.md) hash representation *hash* to hexadecimal. - `--to-base32` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-32. + *hash* to [Nix32](@docroot@/protocols/nix32.md). - `--to-base64` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-64. + *hash* to Base64. - `--to-sri` @@ -134,7 +134,7 @@ $ nix-hash --type sha256 --flat test/world 5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 ``` -Converting between hexadecimal, base-32, base-64, and SRI: +Converting between hexadecimal, Nix32, Base64, and SRI: ```console $ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 diff --git a/doc/manual/source/command-ref/nix-prefetch-url.md b/doc/manual/source/command-ref/nix-prefetch-url.md index 8451778ad46d..86c20b9e1de4 100644 --- a/doc/manual/source/command-ref/nix-prefetch-url.md +++ b/doc/manual/source/command-ref/nix-prefetch-url.md @@ -32,7 +32,7 @@ Otherwise, the file is downloaded, and an error is signaled if the actual hash of the file does not match the specified hash. This command prints the hash on standard output. -The hash is printed using base-32 unless `--type md5` is specified, +The hash is printed using [Nix32](@docroot@/protocols/nix32.md) unless `--type md5` is specified, in which case it's printed using base-16. Additionally, if the option `--print-path` is used, the path of the downloaded file in the Nix store is also printed. diff --git a/doc/manual/source/development/json-guideline.md b/doc/manual/source/development/data-modeling.md similarity index 86% rename from doc/manual/source/development/json-guideline.md rename to doc/manual/source/development/data-modeling.md index 309b4b3a06e4..a9d131147588 100644 --- a/doc/manual/source/development/json-guideline.md +++ b/doc/manual/source/development/data-modeling.md @@ -1,7 +1,12 @@ -# JSON guideline +# Data Modeling Guidelines -Nix consumes and produces JSON in a variety of contexts. -These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. +Nix consumes and produces JSON and attribute sets in a variety of contexts. +These guidelines ensure consistent practices for our interfaces, for ease of use, and so that experience in one part carries over to another. + +For these guidelines, we will use JSON terminology, but they apply equally well to new attribute set interfaces (primops, etc.). +Note that these are guidelines first and foremost. Exceptions include: +- Feature testing: e.g., it is OK to do `builtins?frobnicate`. +- Compatibility: we generally do not change stable interfaces just to make them comply. New replacements can be added with care. ## Extensibility diff --git a/doc/manual/source/development/debugging.md b/doc/manual/source/development/debugging.md index 6578632d991a..35e4c71ec388 100644 --- a/doc/manual/source/development/debugging.md +++ b/doc/manual/source/development/debugging.md @@ -26,7 +26,6 @@ or GCC. This is useful when debugging memory corruption issues. ```console [nix-shell]$ export mesonBuildType=debugoptimized [nix-shell]$ appendToVar mesonFlags "-Dlibexpr:gc=disabled" # Disable Boehm -[nix-shell]$ appendToVar mesonFlags "-Dbindings=false" # Disable nix-perl [nix-shell]$ appendToVar mesonFlags "-Db_sanitize=address,undefined" ``` diff --git a/doc/manual/source/development/static-analysis.md b/doc/manual/source/development/static-analysis.md new file mode 100644 index 000000000000..6bdf875befde --- /dev/null +++ b/doc/manual/source/development/static-analysis.md @@ -0,0 +1,87 @@ +# Static Analysis + +Nix uses [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) for static analysis of C++ code. +This helps catch bugs, enforce coding standards, and maintain code quality. + +## Running clang-tidy locally + +To run clang-tidy on the entire codebase in the development shell: + +```console +$ nix develop .#native-clangStdenv +$ configurePhase +$ meson compile -C build clang-tidy +``` + +This will analyze all C++ source files and report any warnings. + +To automatically apply fixes for certain warnings: + +```console +$ meson compile -C build clang-tidy-fix +``` + +> **Warning** +> +> Review the changes before committing, as automatic fixes may not always be correct. + +## CI integration + +clang-tidy runs automatically on every pull request via GitHub Actions. +The CI job builds `.#hydraJobs.clangTidy.x86_64-linux` which: + +1. Builds all components with debug mode (for faster compilation) +2. Runs clang-tidy on each component +3. Fails if any warnings are found (warnings are treated as errors) + +## Configuration + +The clang-tidy configuration is in `.clang-tidy` at the repository root (symlinked from `nix-meson-build-support/common/clang-tidy/.clang-tidy`). + +### Suppressing warnings + +If a warning is a false positive, you can suppress it in several ways: + +1. **Inline suppression** (preferred for specific cases): + ```cpp + // NOLINTBEGIN(bugprone-some-check) + ... code ... + // NOLINTEND(bugprone-some-check) + ``` + Or for a single line: + ```cpp + int x = something(); // NOLINT(bugprone-some-check) + ``` + +2. **Configuration file** (for project-wide suppression): + Add the check to the disabled list in `.clang-tidy`: + ```yaml + Checks: + - -bugprone-some-check # Reason for disabling + ``` + +3. **Check options** (for configuring check behavior): + ```yaml + CheckOptions: + bugprone-reserved-identifier.AllowedIdentifiers: '__some_identifier' + ``` + +### Adding new checks + +To enable additional checks: + +1. Edit `nix-meson-build-support/common/clang-tidy/.clang-tidy` +2. Add the check to the `Checks` list +3. Run clang-tidy locally to see the impact +4. Fix any new warnings or disable specific sub-checks if needed + +## Custom clang-tidy plugin + +The Nix project includes infrastructure for custom clang-tidy checks in `src/clang-tidy-plugin/`. +These checks can enforce Nix-specific coding patterns that aren't covered by standard clang-tidy checks. + +To add a new custom check: + +1. Add the check implementation in `src/clang-tidy-plugin/` +2. Register it in `nix-clang-tidy-checks.cc` +3. Enable it in `.clang-tidy` with the `nix-` prefix diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index 35654d163935..6ea1a233a4aa 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -9,7 +9,7 @@ You can build it yourself: ``` # nix build .#hydraJobs.coverage -# xdg-open ./result/coverage/index.html +# xdg-open ./result/index.html ``` [Extensive records of build metrics](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts), such as test coverage over time, are also available online. @@ -34,31 +34,28 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > │ ├── value/context.cc > │ … > │ -> ├── tests -> │ │ +> ├── libutil-tests +> │ ├── meson.build > │ … -> │ ├── libutil-tests -> │ │ ├── meson.build -> │ │ … -> │ │ └── data -> │ │ ├── git/tree.txt -> │ │ … -> │ │ -> │ ├── libexpr-test-support +> │ ├── data +> │ │ ├── git/tree.txt +> │ … +> │ +> ├── libexpr-test-support +> │ ├── meson.build +> │ ├── include/nix/expr > │ │ ├── meson.build -> │ │ ├── include/nix/expr -> │ │ │ ├── meson.build -> │ │ │ └── tests -> │ │ │ ├── value/context.hh -> │ │ │ … > │ │ └── tests -> │ │ ├── value/context.cc +> │ │ ├── value/context.hh > │ │ … -> │ │ -> │ ├── libexpr-tests -> │ … ├── meson.build +> │ ├── tests > │ ├── value/context.cc > │ … +> │ +> ├── libexpr-tests +> │ ├── meson.build +> │ ├── value/context.cc +> │ … > … > ``` @@ -80,7 +77,7 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. -A environment variables that Google Test accepts are also worth knowing: +The environment variables that Google Test accepts are also worth knowing: 1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) @@ -257,15 +254,6 @@ GNU gdb (GDB) 12.1 One can debug the Nix invocation in all the usual ways. For example, enter `run` to start the Nix invocation. -### Troubleshooting - -Sometimes running tests in the development shell may leave artefacts in the local repository. -To remove any traces of that: - -```console -git clean -x --force tests -``` - ### Characterisation testing { #characterisation-testing-functional } Occasionally, Nix utilizes a technique called [Characterisation Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. @@ -311,77 +299,34 @@ Generally, this build is sufficient, but in nightly or CI we also test the attri The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. -Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). +Because these tests are expensive and require more than what the standard github-actions setup provides, most of them only run on the master branch (on ). You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. ## Installer tests -After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. - -Creating a Cachix cache for your installer tests and adding its authorisation token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): +GitHub Actions CI in the Nix repository also tests the installer on PRs. It does not require additional setup and utilises [GHA Artifacts](https://docs.github.com/en/actions/tutorials/store-and-share-data) and can be run in any Nix repository fork. -- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: +- The `tests` job generates installers for the platforms below and uploads them as an artifact: - `x86_64-linux` - - `armv6l-linux` - - `armv7l-linux` - -- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-14`) will try to install Nix with the cached installer and run a trivial Nix command. + - `aarch64-darwin` -### One-time setup +- The `installer_test` job (which runs on Linux and macOS) will try to install Nix with the cached installer and run a trivial Nix command. +- Both the scripted installer and the [standalone Rust-based installer](https://github.com/NixOS/nix-installer) are tested. -1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). -2. At cachix.org: - - Create or log in to an account. - - Create a Cachix cache using the format `-nix-install-tests`. - - Navigate to the new cache > Settings > Auth Tokens. - - Generate a new Cachix auth token and copy the generated value. -3. At github.com: - - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. - - Name the secret `CACHIX_AUTH_TOKEN`. - - Paste the copied value of the Cachix cache auth token. +You can generate the installer tarball and script manually by running `nix build .#hydraJobs.installerScriptForGHA.`. ## Working on documentation ### Using the CI-generated installer for manual testing -After the CI run completes, you can check the output to extract the installer URL: +After the CI run completes, you can check the output to extract the installer artifact: 1. Click into the detailed view of the CI run. -2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). -3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) -4. Copy the value of `install_url` -5. To generate an install command, plug this `install_url` and your GitHub username into this template: +2. Scroll down to `Artifacts` section. +3. Download the corresponding installer artifact (`installer-darwin` for `aarch64-darwin` and `installer-linux` for `x86_64-linux`). +4. Unpack the downloaded `.zip` artifact. +5. To generate an install command, plug the path to the unpacked artifact into this template: ```console - curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + sh /install --tarball-url-prefix file:// ``` - - - diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index 64ca1cf5e167..c7b8931509a8 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -1,5 +1,32 @@ # Glossary +- [base directory]{#gloss-base-directory} + + The location from which relative paths are resolved. + + - For expressions in a file, the base directory is the directory containing that file. + This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). + + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory + +- [binary cache]{#gloss-binary-cache} + + A *binary cache* is a Nix store which uses a different format: its + metadata and signatures are kept in `.narinfo` files rather than in a + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). + - [build system]{#gloss-build-system} Generic term for software that facilitates the building of software by automating the invocation of compilers, linkers, and other tools. @@ -8,6 +35,26 @@ It has no knowledge of any particular programming language or toolchain. These details are specified in [derivation expressions](#gloss-derivation-expression). +- [closure]{#gloss-closure} + + The closure of a store path is the set of store paths that are + directly or indirectly “reachable” from that store path; that is, + it’s the closure of the path under the *references* relation. For + a package, the closure of its derivation is equivalent to the + build-time dependencies, while the closure of its [output path] is + equivalent to its runtime dependencies. For correct deployment it + is necessary to deploy whole closures, since otherwise at runtime + files could be missing. The command `nix-store --query --requisites ` prints out + closures of store paths. + + As an example, if the [store object] at path `P` contains a [reference] + to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` + references `R` then `R` is also in the closure of `P`. + + See [References](@docroot@/store/store-object.md#references) for details. + + [closure]: #gloss-closure + - [content address]{#gloss-content-address} A @@ -31,32 +78,37 @@ The industry term for storage and retrieval systems using [content addressing](#gloss-content-address). A Nix store also has [input addressing](#gloss-input-addressed-store-object), and metadata. +- [content-addressed store object]{#gloss-content-addressed-store-object} + + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. + This includes derivations, the outputs of [content-addressing derivations](#gloss-content-addressing-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + +- [content-addressing derivation]{#gloss-content-addressing-derivation} + + A derivation which has the + [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) + attribute set to `true`. + - [derivation]{#gloss-derivation} A derivation can be thought of as a [pure function](https://en.wikipedia.org/wiki/Pure_function) that produces new [store objects][store object] from existing store objects. - Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md#builder-execution). + Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md). This sandbox by default only allows reading from store objects specified as inputs, and only allows writing to designated [outputs][output] to be [captured as store objects](@docroot@/store/building.md#processing-outputs). A derivation is typically specified as a [derivation expression] in the [Nix language], and [instantiated][instantiate] to a [store derivation]. - There are multiple ways of obtaining store objects from store derivatons, collectively called [realisation][realise]. + There are multiple ways of obtaining store objects from store derivations, collectively called [realisation][realise]. [derivation]: #gloss-derivation -- [store derivation]{#gloss-store-derivation} - - A [derivation] represented as a [store object]. - - See [Store Derivation](@docroot@/store/derivation/index.md#store-derivation) for details. - - [store derivation]: #gloss-store-derivation - -- [directed acyclic graph]{#gloss-directed-acyclic-graph} +- [derivation expression]{#gloss-derivation-expression} - A [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAG) is graph whose edges are given a direction ("a to b" is not the same edge as "b to a"), and for which no possible path (created by joining together edges) forms a cycle. + A description of a [store derivation] using the [`derivation` primitive](./language/derivations.md) in the [Nix language]. - DAGs are very important to Nix. - In particular, the non-self-[references][reference] of [store object][store object] form a cycle. + [derivation expression]: #gloss-derivation-expression - [derivation path]{#gloss-derivation-path} @@ -68,79 +120,34 @@ [derivation path]: #gloss-derivation-path -- [derivation expression]{#gloss-derivation-expression} - - A description of a [store derivation] using the [`derivation` primitive](./language/derivations.md) in the [Nix language]. - - [derivation expression]: #gloss-derivation-expression - -- [instantiate]{#gloss-instantiate}, instantiation - - Translate a [derivation expression] into a [store derivation]. - - See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation. - - [instantiate]: #gloss-instantiate - -- [realise]{#gloss-realise}, realisation - - Ensure a [store path] is [valid][validity]. - - This can be achieved by: - - Fetching a pre-built [store object] from a [substituter] - - [Building](@docroot@/store/building.md) the corresponding [store derivation] - - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs - - See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - - See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). - - [realise]: #gloss-realise - -- [content-addressing derivation]{#gloss-content-addressing-derivation} - - A derivation which has the - [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) - attribute set to `true`. - -- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - - A [store derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. +- [deriver]{#gloss-deriver} -- [store]{#gloss-store} + The [store derivation] that produced an [output path]. - A collection of [store objects][store object], with operations to manipulate that collection. - See [Nix Store](./store/index.md) for details. + The deriver for an output path can be queried with the `--deriver` option to + [`nix-store --query`](@docroot@/command-ref/nix-store/query.md). - There are many types of stores, see [Store Types](./store/types/index.md) for details. +- [deriving path]{#gloss-deriving-path} - [store]: #gloss-store + Deriving paths are a way to refer to [store objects][store object] that might not yet be [realised][realise]. -- [Nix instance]{#gloss-nix-instance} - - 1. An installation of Nix, which includes the presence of a [store], and the Nix package manager which operates on that store. - A local Nix installation and a [remote builder](@docroot@/advanced-topics/distributed-builds.md) are two examples of Nix instances. - 2. A running Nix process, such as the `nix` command. + See [Deriving Path](./store/derivation/index.md#deriving-path) for details. -- [binary cache]{#gloss-binary-cache} + Not to be confused with [derivation path]. - A *binary cache* is a Nix store which uses a different format: its - metadata and signatures are kept in `.narinfo` files rather than in a - [Nix database]. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). +- [directed acyclic graph]{#gloss-directed-acyclic-graph} -- [store path]{#gloss-store-path} + A [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAG) is graph whose edges are given a direction ("a to b" is not the same edge as "b to a"), and for which no possible path (created by joining together edges) forms a cycle. - The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. + DAGs are very important to Nix. + In particular, the non-self-[references][reference] of [store object][store object] form a cycle. - > **Example** - > - > `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` +- [experimental feature]{#gloss-experimental-feature} - See [Store Path](@docroot@/store/store-path.md) for details. + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - [store path]: #gloss-store-path + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). - [file system object]{#gloss-file-system-object} @@ -150,21 +157,19 @@ [file system object]: #gloss-file-system-object -- [store object]{#gloss-store-object} - - Part of the contents of a [store]. - - A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. - It can be referred to by a [store path]. - - See [Store Object](@docroot@/store/store-object.md) for details. +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - [store object]: #gloss-store-object + A [store derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [IFD]{#gloss-ifd} [Import From Derivation](./language/import-from-derivation.md) +- [impure derivation]{#gloss-impure-derivation} + + [An experimental feature](@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. + - [input-addressed store object]{#gloss-input-addressed-store-object} A store object produced by building a @@ -174,42 +179,28 @@ See [input-addressing derivation outputs](store/derivation/outputs/input-address.md) for details. -- [content-addressed store object]{#gloss-content-addressed-store-object} - - A [store object] which is [content-addressed](#gloss-content-address), - i.e. whose [store path] is determined by its contents. - This includes derivations, the outputs of [content-addressing derivations](#gloss-content-addressing-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). - - See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. - -- [substitute]{#gloss-substitute} +- [installable]{#gloss-installable} - A substitute is a command invocation stored in the [Nix database] that - describes how to build a store object, bypassing the normal build - mechanism (i.e., derivations). Typically, the substitute builds the - store object by downloading a pre-built version of the store object - from some server. + Something that can be realised in the Nix store. -- [substituter]{#gloss-substituter} + See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. - An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. - Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. +- [instantiate]{#gloss-instantiate}, instantiation - See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. + Translate a [derivation expression] into a [store derivation]. - [substituter]: #gloss-substituter + See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation. -- [purity]{#gloss-purity} + [instantiate]: #gloss-instantiate - The assumption that equal Nix derivations when run always produce - the same output. This cannot be guaranteed in general (e.g., a - builder can rely on external inputs such as the network or the - system time) but the Nix model assumes it. +- [Nix Archive (NAR)]{#gloss-nar} -- [impure derivation]{#gloss-impure-derivation} + A *N*ix *AR*chive. This is a serialisation of a path in the Nix + store. It can contain regular files, directories and symbolic + links. NARs are generated and unpacked using `nix-store --dump` + and `nix-store --restore`. - [An experimental feature](@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, - so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. - [Nix database]{#gloss-nix-database} @@ -235,137 +226,136 @@ > > Building and deploying software using Nix entails writing Nix expressions to describe [packages][package] and compositions thereof. -- [reference]{#gloss-reference} +- [Nix instance]{#gloss-nix-instance} + + 1. An installation of Nix, which includes the presence of a [store], and the Nix package manager which operates on that store. + A local Nix installation and a [remote builder](@docroot@/advanced-topics/distributed-builds.md) are two examples of Nix instances. + 2. A running Nix process, such as the `nix` command. - An edge from one [store object] to another. +- [output]{#gloss-output} - See [References](@docroot@/store/store-object.md#references) for details. + A [store object] produced by a [store derivation]. + See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. - [reference]: #gloss-reference + [output]: #gloss-output - See [References](@docroot@/store/store-object.md#references) for details. +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. -- [reachable]{#gloss-reachable} +- [output path]{#gloss-output-path} - A store path `Q` is reachable from another store path `P` if `Q` - is in the *closure* of the *references* relation. + The [store path] to the [output] of a [store derivation]. - See [References](@docroot@/store/store-object.md#references) for details. + [output path]: #gloss-output-path -- [closure]{#gloss-closure} +- [package]{#package} - The closure of a store path is the set of store paths that are - directly or indirectly “reachable” from that store path; that is, - it’s the closure of the path under the *references* relation. For - a package, the closure of its derivation is equivalent to the - build-time dependencies, while the closure of its [output path] is - equivalent to its runtime dependencies. For correct deployment it - is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command `nix-store --query --requisites ` prints out - closures of store paths. + A software package; files that belong together for a particular purpose, and metadata. - As an example, if the [store object] at path `P` contains a [reference] - to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` - references `R` then `R` is also in the closure of `P`. + Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. - See [References](@docroot@/store/store-object.md#references) for details. + The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#type-attrs) containing: + - attributes that refer to the files of a package, typically in the form of [derivation outputs](#gloss-output), + - attributes with metadata, such as information about how the package is supposed to be used. - [closure]: #gloss-closure + The exact shape of these attribute sets is up to convention. -- [requisite]{#gloss-requisite} + [package]: #package - A store object [reachable] by a path (chain of references) from a given [store object]. - The [closure] is the set of requisites. +- [profile]{#gloss-profile} - See [References](@docroot@/store/store-object.md#references) for details. + A symlink to the current *user environment* of a user, e.g., + `/nix/var/nix/profiles/default`. -- [referrer]{#gloss-referrer} +- [purity]{#gloss-purity} - A reversed edge from one [store object] to another. + The assumption that equal Nix derivations when run always produce + the same output. This cannot be guaranteed in general (e.g., a + builder can rely on external inputs such as the network or the + system time) but the Nix model assumes it. -- [output]{#gloss-output} +- [reachable]{#gloss-reachable} - A [store object] produced by a [store derivation]. - See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. + A store path `Q` is reachable from another store path `P` if `Q` + is in the *closure* of the *references* relation. - [output]: #gloss-output + See [References](@docroot@/store/store-object.md#references) for details. -- [output path]{#gloss-output-path} +- [realise]{#gloss-realise}, realisation - The [store path] to the [output] of a [store derivation]. + Ensure a [store path] is [valid][validity]. - [output path]: #gloss-output-path + This can be achieved by: + - Fetching a pre-built [store object] from a [substituter] + - [Building](@docroot@/store/building.md) the corresponding [store derivation] + - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs -- [output closure]{#gloss-output-closure}\ - The [closure] of an [output path]. It only contains what is [reachable] from the output. + See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. -- [deriving path]{#gloss-deriving-path} + See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). - Deriving paths are a way to refer to [store objects][store object] that might not yet be [realised][realise]. + [realise]: #gloss-realise - See [Deriving Path](./store/derivation/index.md#deriving-path) for details. +- [reference]{#gloss-reference} - Not to be confused with [derivation path]. + An edge from one [store object] to another. -- [deriver]{#gloss-deriver} + See [References](@docroot@/store/store-object.md#references) for details. - The [store derivation] that produced an [output path]. + [reference]: #gloss-reference - The deriver for an output path can be queried with the `--deriver` option to - [`nix-store --query`](@docroot@/command-ref/nix-store/query.md). + See [References](@docroot@/store/store-object.md#references) for details. -- [validity]{#gloss-validity} +- [referrer]{#gloss-referrer} - A store path is valid if all [store object]s in its [closure] can be read from the [store]. + A reversed edge from one [store object] to another. - For a [local store], this means: - - The store path leads to an existing [store object] in that [store]. - - The store path is listed in the [Nix database] as being valid. - - All paths in the store path's [closure] are valid. +- [requisite]{#gloss-requisite} - [validity]: #gloss-validity - [local store]: @docroot@/store/types/local-store.md + A store object [reachable] by a path (chain of references) from a given [store object]. + The [closure] is the set of requisites. -- [user environment]{#gloss-user-env} + See [References](@docroot@/store/store-object.md#references) for details. - An automatically generated store object that consists of a set of - symlinks to “active” applications, i.e., other store paths. These - are generated automatically by - [`nix-env`](./command-ref/nix-env.md). See *profiles*. +- [store]{#gloss-store} -- [profile]{#gloss-profile} + A collection of [store objects][store object], with operations to manipulate that collection. + See [Nix Store](./store/index.md) for details. - A symlink to the current *user environment* of a user, e.g., - `/nix/var/nix/profiles/default`. + There are many types of stores, see [Store Types](./store/types/index.md) for details. -- [installable]{#gloss-installable} + [store]: #gloss-store - Something that can be realised in the Nix store. +- [store derivation]{#gloss-store-derivation} - See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. + A [derivation] represented as a [store object]. -- [Nix Archive (NAR)]{#gloss-nar} + See [Store Derivation](@docroot@/store/derivation/index.md#store-derivation) for details. - A *N*ix *AR*chive. This is a serialisation of a path in the Nix - store. It can contain regular files, directories and symbolic - links. NARs are generated and unpacked using `nix-store --dump` - and `nix-store --restore`. + [store derivation]: #gloss-store-derivation - See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. +- [store object]{#gloss-store-object} -- [package]{#package} + Part of the contents of a [store]. - A software package; files that belong together for a particular purpose, and metadata. + A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. + It can be referred to by a [store path]. - Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. + See [Store Object](@docroot@/store/store-object.md) for details. - The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#type-attrs) containing: - - attributes that refer to the files of a package, typically in the form of [derivation outputs](#gloss-output), - - attributes with metadata, such as information about how the package is supposed to be used. + [store object]: #gloss-store-object - The exact shape of these attribute sets is up to convention. +- [store path]{#gloss-store-path} - [package]: #package + The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. + + > **Example** + > + > `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` + + See [Store Path](@docroot@/store/store-path.md) for details. + + [store path]: #gloss-store-path - [string interpolation]{#gloss-string-interpolation} @@ -377,31 +367,47 @@ [path]: ./language/types.md#type-path [attribute name]: ./language/types.md#type-attrs -- [base directory]{#gloss-base-directory} +- [SRI]{#gloss-sri} - The location from which relative paths are resolved. + [Subresource Integrity](https://www.w3.org/TR/SRI/) (SRI) is a [W3C specification](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) for integrity metadata. + Nix uses the SRI hash format (`-`) to specify content hashes in a way that is self-describing, since the hash algorithm is part of the format. - - For expressions in a file, the base directory is the directory containing that file. - This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). - + [SRI]: #gloss-sri - - - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. +- [substitute]{#gloss-substitute} - [base directory]: #gloss-base-directory + A substitute is a command invocation stored in the [Nix database] that + describes how to build a store object, bypassing the normal build + mechanism (i.e., derivations). Typically, the substitute builds the + store object by downloading a pre-built version of the store object + from some server. -- [experimental feature]{#gloss-experimental-feature} +- [substituter]{#gloss-substituter} - Not yet stabilized functionality guarded by named experimental feature flags. - These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. + + [substituter]: #gloss-substituter + +- [user environment]{#gloss-user-env} + + An automatically generated store object that consists of a set of + symlinks to “active” applications, i.e., other store paths. These + are generated automatically by + [`nix-env`](./command-ref/nix-env.md). See *profiles*. + +- [validity]{#gloss-validity} + A store path is valid if all [store object]s in its [closure] can be read from the [store]. + + For a [local store], this means: + - The store path leads to an existing [store object] in that [store]. + - The store path is listed in the [Nix database] as being valid. + - All paths in the store path's [closure] are valid. + + [validity]: #gloss-validity + [local store]: @docroot@/store/types/local-store.md [Nix language]: ./language/index.md diff --git a/doc/manual/source/installation/prerequisites-source.md b/doc/manual/source/installation/prerequisites-source.md index 057fd444349e..e98067348ee6 100644 --- a/doc/manual/source/installation/prerequisites-source.md +++ b/doc/manual/source/installation/prerequisites-source.md @@ -1,80 +1,52 @@ # Prerequisites - - GNU Autoconf () and the - autoconf-archive macro collection - (). These are - needed to run the bootstrap script. - - - GNU Make. - - - Bash Shell. The `./configure` script relies on bashisms, so Bash is - required. - - - A version of GCC or Clang that supports C++23. - - - `pkg-config` to locate dependencies. If your distribution does not - provide it, you can get it from - . - - - The OpenSSL library to calculate cryptographic hashes. If your - distribution does not provide it, you can get it from - . - - - The `libbrotlienc` and `libbrotlidec` libraries to provide - implementation of the Brotli compression algorithm. They are - available for download from the official repository - . - - - cURL and its library. If your distribution does not provide it, you - can get it from . - - - The SQLite embedded database library, version 3.6.19 or higher. If - your distribution does not provide it, please install it from - . - - - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce - the evaluator’s memory consumption (optional). - - To enable it, install - `pkgconfig` and the Boehm garbage collector, and pass the flag - `--enable-gc` to `configure`. - - - The `boost` library of version 1.66.0 or higher. It can be obtained - from the official web site . - - - The `editline` library of version 1.14.0 or higher. It can be - obtained from the its repository - . - - - The `libsodium` library for verifying cryptographic signatures - of contents fetched from binary caches. - It can be obtained from the official web site - . - - - Recent versions of Bison and Flex to build the parser. (This is - because Nix needs GLR support in Bison and reentrancy support in - Flex.) For Bison, you need version 2.6, which can be obtained from - the [GNU FTP server](ftp://alpha.gnu.org/pub/gnu/bison). For Flex, - you need version 2.5.35, which is available on - [SourceForge](http://lex.sourceforge.net/). Slightly older versions - may also work, but ancient versions like the ubiquitous 2.5.4a - won't. - - - The `libseccomp` is used to provide syscall filtering on Linux. This - is an optional dependency and can be disabled passing a - `--disable-seccomp-sandboxing` option to the `configure` script (Not - recommended unless your system doesn't support `libseccomp`). To get - the library, visit . - - - On 64-bit x86 machines only, `libcpuid` library - is used to determine which microarchitecture levels are supported + This list and lower version bounds are maintained on best-effort basis. When in doubt, check the `meson.build` files. + + - Meson build system (). + + - Ninja (). + + - A version of GCC or Clang that supports C++23 (anything newer than Clang 19 or GCC 14 is likely to work). + + - `pkg-config` to locate dependencies. + If your distribution does not provide it, you can get it from . + + - The OpenSSL library to calculate cryptographic hashes. + If your distribution does not provide it, you can get it from . + + - The `libbrotlienc` and `libbrotlidec` libraries to provide implementation of the Brotli compression algorithm. + They are available for download from the official repository . + + - cURL library. + If your distribution does not provide it, you can get it from . + + - The SQLite embedded database library, version 3.6.19 or higher. + If your distribution does not provide it, please install it from . + + - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce the evaluator’s memory consumption (optional). + To enable it, install `pkgconfig` and the Boehm garbage collector, and pass the option `-Dlibexpr:gc=enabled` to `meson setup`. + + - The `boost` library of version 1.87.0 or higher. + It can be obtained from the official web site . + + - The `editline` library of version 1.14.0 or higher. + It can be obtained from the its repository . + + - The `libsodium` library for verifying cryptographic signatures of contents fetched from binary caches. + It can be obtained from the official web site . + + - Recent versions of Bison and Flex to build the parser. + (This is because Nix needs C++ template support in Bison and reentrancy support in Flex.) + + - The `libseccomp` is used to provide syscall filtering on Linux. + This is an optional dependency and can be disabled passing a `-Dlibstore:seccomp-sandboxing=disabled` option to the `meson setup` command + (Not recommended unless your system doesn't support `libseccomp`). + To get the library, visit . + + - On 64-bit x86 machines only, `libcpuid` library is used to determine which microarchitecture levels are supported (e.g., as whether to have `x86_64-v2-linux` among additional system types). - The library is available from its homepage - . - This is an optional dependency and can be disabled - by providing a `--disable-cpuid` to the `configure` script. - - - Unless `./configure --disable-unit-tests` is specified, GoogleTest (GTest) and - RapidCheck are required, which are available at - and - respectively. + The library is available from its homepage . + This is an optional dependency and can be disabled by providing a `-Dlibutil:cpuid=disabled` option to `meson setup` script. + + - Unless `meson setup build -Dunit-tests=false` is specified, GoogleTest (GTest) and RapidCheck are required, which are available at + and respectively. diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index 039ad6f30b1c..06518598aa10 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -25,7 +25,7 @@ as /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1/ -where `b6gvzjyb2pg0…` is a unique identifier for the package that +where `q06x3jll2yfz…` is a unique identifier for the package that captures all its dependencies (it’s a cryptographic hash of the package’s build dependency graph). This enables many powerful features. @@ -64,7 +64,7 @@ builds correctly on your system, this is because you specified the dependency explicitly. This takes care of the build-time dependencies. Once a package is built, runtime dependencies are found by scanning -binaries for the hash parts of Nix store paths (such as `r8vvq9kq…`). +binaries for the [hash parts](@docroot@/store/store-path.md#digest) of Nix store paths (such as `r8vvq9kq…`). This sounds risky, but it works extremely well. ## Multi-user support @@ -189,7 +189,7 @@ the package: ## Portability -Nix runs on Linux and macOS. +Nix runs on Linux, macOS and FreeBSD. ## NixOS diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index 67612029c8a4..cc2743dc5ca9 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -337,8 +337,8 @@ Here is more information on the `output*` attributes, and what values they may b This will specify the output hash of the single output of a [fixed-output derivation]. - The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). - The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of base-32 encoding. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](@docroot@/glossary.md#gloss-sri). + The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of Base32 encoding. > **Note** > diff --git a/doc/manual/source/language/derivations.md b/doc/manual/source/language/derivations.md index 2403183fc2d2..50aa525acbf4 100644 --- a/doc/manual/source/language/derivations.md +++ b/doc/manual/source/language/derivations.md @@ -165,7 +165,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > > for an Autoconf-style package. - The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. + The name of an output is combined with the name of the derivation to create the [name part](@docroot@/store/store-path.md#name) of the output's store path, unless it is `out`, in which case just the name of the derivation is used. > **Example** > diff --git a/doc/manual/source/package-management/binary-cache-substituter.md b/doc/manual/source/package-management/binary-cache-substituter.md index e6a772213d6d..bc2cdfb27ab2 100644 --- a/doc/manual/source/package-management/binary-cache-substituter.md +++ b/doc/manual/source/package-management/binary-cache-substituter.md @@ -19,7 +19,7 @@ whatever port you like: $ nix-serve -p 8080 ``` -To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) file on the client: +To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) file on the client: ```console $ curl http://avalon:8080/nix-cache-info @@ -28,7 +28,7 @@ WantMassQuery: 1 Priority: 30 ``` -When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) automatically if it doesn't exist. +When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) automatically if it doesn't exist. On the client side, you can tell Nix to use your binary cache using `--substituters`, e.g.: diff --git a/doc/manual/source/package-management/profiles.md b/doc/manual/source/package-management/profiles.md index 1d9e672a8def..53cf5061f834 100644 --- a/doc/manual/source/package-management/profiles.md +++ b/doc/manual/source/package-management/profiles.md @@ -11,8 +11,7 @@ in a directory another version might be stored in `/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2`. The long strings prefixed to the directory names are cryptographic hashes (to be -precise, 160-bit truncations of SHA-256 hashes encoded in a base-32 -notation) of *all* inputs involved in building the package — sources, +precise, 160-bit truncations of SHA-256 hashes encoded in [Nix32](@docroot@/protocols/nix32.md)) of *all* inputs involved in building the package — sources, dependencies, compiler flags, and so on. So if two packages differ in any way, they end up in different locations in the file system, so they don’t interfere with each other. Here is what a part of a typical Nix diff --git a/doc/manual/source/protocols/binary-cache/index.md b/doc/manual/source/protocols/binary-cache/index.md new file mode 100644 index 000000000000..d86d307ad9ee --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/index.md @@ -0,0 +1,19 @@ +# Binary Cache + +The binary cache format is an interface designed for exposing a store over HTTP. + +A binary cache consists of: + +- A [`nix-cache-info`](./nix-cache-info.md) file at the root with remote-side configuration. +- For each [store object](@docroot@/store/store-object.md): + - A [`.narinfo`](./narinfo.md) file containing the object's [metadata](@docroot@/store/store-object.md#metadata) and a (usually relative) URL to the corresponding compressed NAR. + - A possibly-compressed [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) containing the store object's file system data. +- For every entry in the [build trace](@docroot@/store/build-trace.md), a JSON file at `build-trace-v2//.doi`: + - the path encodes the [key](@docroot@/protocols/json/build-trace-entry.md#key) + - the contents are the [value](@docroot@/protocols/json/build-trace-entry.md#value). + +The following [store types](@docroot@/store/types/index.md) use the binary cache format: + +- [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md) — served over HTTP(S) +- [Local Binary Cache Store](@docroot@/store/types/local-binary-cache-store.md) — stored on the file system +- [S3 Binary Cache Store](@docroot@/store/types/s3-binary-cache-store.md) — stored in an AWS S3 bucket diff --git a/doc/manual/source/protocols/binary-cache/narinfo.md b/doc/manual/source/protocols/binary-cache/narinfo.md new file mode 100644 index 000000000000..e2e2efac0eeb --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/narinfo.md @@ -0,0 +1,42 @@ +# `.narinfo` Format + +A `.narinfo` file contains the [metadata of a store object](@docroot@/store/store-object.md#metadata) in the [binary cache](@docroot@/protocols/binary-cache/index.md) format. +It is a simple line-oriented format where each line is a `Key: Value` pair. +Some keys (e.g. `Sig`) may appear multiple times. + +The file is named `.narinfo`, where `` is the [hash part](@docroot@/store/store-path.md#digest) of the store object's [store path](@docroot@/store/store-path.md). + +The fields correspond to those documented in the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format: + +| `.narinfo` field | JSON field | Differences | +|---|---|---| +| `StorePath` | [`path`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_path) | Full [store path](@docroot@/store/store-path.md) rather than [store path base name](@docroot@/store/store-path.md#base-name) | +| `URL` | [`url`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_url) | | +| `Compression` | [`compression`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_compression) | Defaults to `bzip2` if omitted | +| `FileHash` | [`downloadHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadHash) | String-encoded hash rather than structured | +| `FileSize` | [`downloadSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadSize) | | +| `NarHash` | [`narHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narHash) | String-encoded hash rather than structured | +| `NarSize` | [`narSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narSize) | | +| `References` | [`references`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_references) | Space-separated [store path base names](@docroot@/store/store-path.md#base-name) rather than a JSON array | +| `Deriver` | [`deriver`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_deriver) | [Store path base name](@docroot@/store/store-path.md#base-name); `unknown-deriver` instead of `null` | +| `Sig` | [`signatures`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_signatures) | May appear multiple times rather than using an array | +| `CA` | [`ca`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_ca) | String-encoded [content address](@docroot@/store/store-object/content-address.md) rather than structured | + +## Example + + + +``` +StorePath: /nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +URL: nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz?sha256=1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3 +Compression: xz +FileHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +FileSize: 4029176 +NarHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +NarSize: 34878 +References: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +Deriver: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv +Sig: asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +Sig: qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +CA: fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh +``` diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/binary-cache/nix-cache-info.md similarity index 90% rename from doc/manual/source/protocols/nix-cache-info.md rename to doc/manual/source/protocols/binary-cache/nix-cache-info.md index e8351e1cebe8..3859b2a10ac4 100644 --- a/doc/manual/source/protocols/nix-cache-info.md +++ b/doc/manual/source/protocols/binary-cache/nix-cache-info.md @@ -1,6 +1,6 @@ -# Nix Cache Info Format +# `nix-cache-info` Format -The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/package-management/binary-cache-substituter.md) (e.g., `https://cache.example.com/nix-cache-info`). +The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/protocols/binary-cache/index.md) (e.g., `https://cache.example.com/nix-cache-info`). MIME type: `text/x-nix-cache-info` diff --git a/doc/manual/source/protocols/derivation-aterm.md b/doc/manual/source/protocols/derivation-aterm.md index 523678e663e3..778614eb1602 100644 --- a/doc/manual/source/protocols/derivation-aterm.md +++ b/doc/manual/source/protocols/derivation-aterm.md @@ -26,7 +26,7 @@ Derivations are serialised in one of the following formats: When derivation is encoded to a [store object] we make the following choices: -- The store path name is the derivation name with `.drv` suffixed at the end +- The store path [name](@docroot@/store/store-path.md#name) is the derivation name with `.drv` suffixed at the end Indeed, the ATerm format above does *not* contain the name of the derivation, on the assumption that a store path will also be provided out-of-band. diff --git a/doc/manual/source/protocols/json/build-trace-entry.md b/doc/manual/source/protocols/json/build-trace-entry.md index 9eea93712601..d4a3a96edb64 100644 --- a/doc/manual/source/protocols/json/build-trace-entry.md +++ b/doc/manual/source/protocols/json/build-trace-entry.md @@ -1,21 +1,21 @@ -{{#include build-trace-entry-v2-fixed.md}} +{{#include build-trace-entry-v3-fixed.md}} ## Examples ### Simple build trace entry ```json -{{#include schema/build-trace-entry-v2/simple.json}} +{{#include schema/build-trace-entry-v3/simple.json}} ``` ### Build trace entry with signature ```json -{{#include schema/build-trace-entry-v2/with-signature.json}} +{{#include schema/build-trace-entry-v3/with-signature.json}} ``` diff --git a/doc/manual/source/protocols/json/derivation/resolved.md b/doc/manual/source/protocols/json/derivation/resolved.md new file mode 100644 index 000000000000..45e18d9af047 --- /dev/null +++ b/doc/manual/source/protocols/json/derivation/resolved.md @@ -0,0 +1 @@ +{{#include ../derivation-resolved-v4-fixed.md}} diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed index 96b6f1801a57..0162036d420b 100644 --- a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -16,3 +16,6 @@ s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](@docroot@/proto s^\(./hash-v1.yaml\)^[JSON format for `Hash`](@docroot@/protocols/json/hash.html)^g s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](@docroot@/protocols/json/content-address.html#method)^g s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](@docroot@/protocols/json/content-address.html)^g +s^\(./store-path-v1.yaml\)^[JSON format for `StorePath`](@docroot@/protocols/json/store-path.html)^g +s^\(./derivation-v4.yaml\)\?#/\$defs/common/properties/[a-zA-Z]*^[JSON format for `Derivation`](@docroot@/protocols/json/derivation/index.html)^g +s^\(./derivation-v4.yaml\)^[JSON format for `Derivation`](@docroot@/protocols/json/derivation/index.html)^g diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c7c48c4ed6e2..9fcacb20387f 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -13,11 +13,13 @@ schemas = [ 'hash-v1', 'content-address-v1', 'store-path-v1', - 'store-object-info-v2', + 'signature-v2', + 'store-object-info-v3', 'derivation-v4', + 'derivation-resolved-v4', 'derivation-options-v1', 'deriving-path-v1', - 'build-trace-entry-v2', + 'build-trace-entry-v3', 'build-result-v1', 'store-v1', ] diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml index 55e55d3e5cbe..d0d8d8a0c742 100644 --- a/doc/manual/source/protocols/json/schema/build-result-v1.yaml +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -83,7 +83,7 @@ properties: description: | A mapping from output names to their build trace entries. additionalProperties: - "$ref": "build-trace-entry-v2.yaml" + "$ref": "build-trace-entry-v3.yaml#/$defs/value" failure: type: object diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml deleted file mode 100644 index 4340f82388c2..000000000000 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml +++ /dev/null @@ -1,95 +0,0 @@ -"$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v2.json" -title: Build Trace Entry -description: | - A record of a successful build outcome for a specific derivation output. - - This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md). - - > **Warning** - > - > This JSON format is currently - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) - > and subject to change. - - Verision history: - - - Version 1: Original format - - - Version 2: Remove `dependentRealisations` - -type: object -required: - - id - - outPath - - signatures -allOf: - - "$ref": "#/$defs/key" - - "$ref": "#/$defs/value" -properties: - id: {} - outPath: {} - signatures: {} -additionalProperties: - dependentRealisations: - description: deprecated field - type: object - -"$defs": - key: - title: Build Trace Key - description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. - This is the "key" part, refering to a derivation and output. - type: object - required: - - id - properties: - id: - type: string - title: Derivation Output ID - pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$" - description: | - Unique identifier for the derivation output that was built. - - Format: `{hash-quotient-drv}!{output-name}` - - - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv). - Begins with `sha256:`. - - - **output-name**: Name of the specific output (e.g., "out", "dev", "doc") - - Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"` - - value: - title: Build Trace Value - description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. - This is the "value" part, describing an output. - type: object - required: - - outPath - - signatures - properties: - outPath: - "$ref": "store-path-v1.yaml" - title: Output Store Path - description: | - The path to the store object that resulted from building this derivation for the given output name. - - patternProperties: - "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$": - "$ref": "store-path-v1.yaml" - title: Dependent Store Path - description: Store path that this dependency resolved to during the build - additionalProperties: false - - signatures: - type: array - title: Build Signatures - description: | - A set of cryptographic signatures attesting to the authenticity of this build trace entry. - items: - type: string - title: Signature - description: A single cryptographic signature diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v2 b/doc/manual/source/protocols/json/schema/build-trace-entry-v3 similarity index 100% rename from doc/manual/source/protocols/json/schema/build-trace-entry-v2 rename to doc/manual/source/protocols/json/schema/build-trace-entry-v3 diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml new file mode 100644 index 000000000000..2e825af6fb8a --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml @@ -0,0 +1,83 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v3.json" +title: Build Trace Entry +description: | + A record of a successful build outcome for a specific derivation output. + + This schema describes the JSON representation of an [entry](@docroot@/store/build-trace.md#entry) in a [build trace](@docroot@/store/build-trace.md). + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) + > and subject to change. + + ## Version History + + - Version 1: Original format + + - Version 2: + - Remove `dependentRealisations` + + - Version 3: + - Use `drvPath` not `drvHash` to refer to derivation in a more conventional way. + - Separate into `key` and `value` + - Use 2nd version of signatures format (objects, not strings) + +type: object +required: + - key + - value +properties: + key: + "$ref": "#/$defs/key" + value: + "$ref": "#/$defs/value" +additionalProperties: false + +"$defs": + key: + title: Build Trace Key + description: | + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. + This is the "key" part, referring to a derivation and output. + type: object + required: + - drvPath + - outputName + properties: + drvPath: + "$ref": "store-path-v1.yaml" + title: Derivation Path + description: | + The store path of the derivation that was built. + outputName: + type: string + title: Output Name + description: | + Name of the specific output (e.g., "out", "dev", "doc") + additionalProperties: false + + value: + title: Build Trace Value + description: | + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. + This is the "value" part, describing an output. + type: object + required: + - outPath + - signatures + properties: + outPath: + "$ref": "store-path-v1.yaml" + title: Output Store Path + description: | + The path to the store object that resulted from building this derivation for the given output name. + + signatures: + type: array + title: Build Signatures + description: | + A set of cryptographic signatures attesting to the authenticity of this build trace entry. + items: + "$ref": "signature-v2.yaml" diff --git a/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml new file mode 100644 index 000000000000..705156eb8451 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml @@ -0,0 +1,52 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-resolved-v4.json" +title: Resolved Derivation +description: | + Experimental JSON representation of a resolved Nix derivation (version 4). + + This schema describes the JSON representation of Nix's `BasicDerivation` type, + which is a derivation with all input derivation dependencies resolved to store paths. + This is the result of `Derivation::tryResolve`. + + This is called "version 4" because we wish to keep it in sync with the + [primary (not necessarily resolved) JSON schema](@docroot@/protocols/json/derivation/index.md), + but actually it is the first version for this format. + +type: object +required: + - name + - version + - outputs + - inputs + - system + - builder + - args + - env +properties: + name: { "$ref": "derivation-v4.yaml#/$defs/common/properties/name" } + version: { "$ref": "derivation-v4.yaml#/$defs/common/properties/version" } + outputs: { "$ref": "derivation-v4.yaml#/$defs/common/properties/outputs" } + system: { "$ref": "derivation-v4.yaml#/$defs/common/properties/system" } + builder: { "$ref": "derivation-v4.yaml#/$defs/common/properties/builder" } + args: { "$ref": "derivation-v4.yaml#/$defs/common/properties/args" } + env: { "$ref": "derivation-v4.yaml#/$defs/common/properties/env" } + structuredAttrs: { "$ref": "derivation-v4.yaml#/$defs/common/properties/structuredAttrs" } + + inputs: + type: array + title: Input source paths + description: | + List of store paths on which this resolved derivation depends. + Since all derivation inputs have been resolved, only source paths remain. + + > **Example** + > + > ```json + > "inputs": [ + > "b8nwz167km1yciqpwzjj24f8jcy8pq1h-separate-debug-info.sh", + > "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out" + > ] + > ``` + items: + $ref: "store-path-v1.yaml" +additionalProperties: false diff --git a/doc/manual/source/protocols/json/schema/derivation-v4.yaml b/doc/manual/source/protocols/json/schema/derivation-v4.yaml index c1884769671e..fa443d982c25 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v4.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v4.yaml @@ -23,57 +23,14 @@ required: - args - env properties: - name: - type: string - title: Derivation name - description: | - The name of the derivation. - Used when calculating store paths for the derivation’s outputs. - - version: - const: 4 - title: Format version (must be 4) - description: | - Must be `4`. - This is a guard that allows us to continue evolving this format. - The choice of `3` is fairly arbitrary, but corresponds to this informal version: - - - Version 0: ATerm format - - - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from ATerm format. - - - Version 2: Separate `method` and `hashAlgo` fields in output specs - - - Version 3: Drop store dir from store paths, just include base name. - - - Version 4: Two cleanups, batched together to lesson churn: - - - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) - - - Use canonical content address JSON format for floating content addressed derivation outputs. - - Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. - - outputs: - type: object - title: Output specifications - description: | - Information about the output paths of the derivation. - This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described. - - > **Example** - > - > ```json - > "outputs": { - > "out": { - > "method": "nar", - > "hashAlgo": "sha256", - > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" - > } - > } - > ``` - additionalProperties: - "$ref": "#/$defs/output/overall" + name: { "$ref": "#/$defs/common/properties/name" } + version: { "$ref": "#/$defs/common/properties/version" } + outputs: { "$ref": "#/$defs/common/properties/outputs" } + system: { "$ref": "#/$defs/common/properties/system" } + builder: { "$ref": "#/$defs/common/properties/builder" } + args: { "$ref": "#/$defs/common/properties/args" } + env: { "$ref": "#/$defs/common/properties/env" } + structuredAttrs: { "$ref": "#/$defs/common/properties/structuredAttrs" } inputs: type: object @@ -126,47 +83,105 @@ properties: - "$ref": "#/$defs/dynamicOutputs" additionalProperties: false additionalProperties: false +additionalProperties: false - system: - type: string - title: Build system type +"$defs": + common: + title: Common Derivation Fields description: | - The system type on which this derivation is to be built - (e.g. `x86_64-linux`). + Fields shared between `Derivation` and `BasicDerivation`. + properties: + name: + type: string + title: Derivation name + description: | + The name of the derivation. + Used when calculating store paths for the derivation's outputs. - builder: - type: string - title: Build program path - description: | - Absolute path of the program used to perform the build. - Typically this is the `bash` shell - (e.g. `/nix/store/p4xlj4imjbnm4v0x5jf4qysvyjjlgq1d-bash-4.4-p23/bin/bash`). + version: + const: 4 + title: Format version (must be 4) + description: | + Must be `4`. + This is a guard that allows us to continue evolving this format. + The choice of `3` is fairly arbitrary, but corresponds to this informal version: - args: - type: array - title: Builder arguments - description: | - Command-line arguments passed to the `builder`. - items: - type: string + - Version 0: ATerm format - env: - type: object - title: Environment variables - description: | - Environment variables passed to the `builder`. - additionalProperties: - type: string + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from ATerm format. - structuredAttrs: - title: Structured attributes - description: | - [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. - Structured attributes are JSON, and thus embedded as-is. - type: object - additionalProperties: true + - Version 2: Separate `method` and `hashAlgo` fields in output specs + + - Version 3: Drop store dir from store paths, just include base name. + + - Version 4: Two cleanups, batched together to lesson churn: + + - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) + + - Use canonical content address JSON format for floating content addressed derivation outputs. + + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. + + outputs: + type: object + title: Output specifications + description: | + Information about the output paths of the derivation. + This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described. + + > **Example** + > + > ```json + > "outputs": { + > "out": { + > "method": "nar", + > "hashAlgo": "sha256", + > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + > } + > } + > ``` + additionalProperties: + "$ref": "#/$defs/output/overall" + + system: + type: string + title: Build system type + description: | + The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + + builder: + type: string + title: Build program path + description: | + Absolute path of the program used to perform the build. + Typically this is the `bash` shell + (e.g. `/nix/store/p4xlj4imjbnm4v0x5jf4qysvyjjlgq1d-bash-4.4-p23/bin/bash`). + + args: + type: array + title: Builder arguments + description: | + Command-line arguments passed to the `builder`. + items: + type: string + + env: + type: object + title: Environment variables + description: | + Environment variables passed to the `builder`. + additionalProperties: + type: string + + structuredAttrs: + title: Structured attributes + description: | + [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. + Structured attributes are JSON, and thus embedded as-is. + type: object + additionalProperties: true -"$defs": output: overall: title: Derivation Output diff --git a/doc/manual/source/protocols/json/schema/nar-info-v3 b/doc/manual/source/protocols/json/schema/nar-info-v3 new file mode 120000 index 000000000000..378215b4156b --- /dev/null +++ b/doc/manual/source/protocols/json/schema/nar-info-v3 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/nar-info/json-3 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/signature-v2.yaml b/doc/manual/source/protocols/json/schema/signature-v2.yaml new file mode 100644 index 000000000000..d94798902f47 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/signature-v2.yaml @@ -0,0 +1,27 @@ +"$schema": "http://json-schema.org/draft-07/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/signature-v2.json" +title: Signature +description: | + A cryptographic signature along with the name of the key that produced it. + + This schema describes the JSON representation of signatures as used in various Nix JSON APIs. + + ## Version History + + - Version 1: Colon-separated string in the format `:` + + - Version 2: Structured object with `keyName` and `sig` fields + +type: object +required: + - keyName + - sig +properties: + keyName: + type: string + title: Key Name + description: The name of the key used to produce this signature + sig: + type: string + title: Signature Data + description: The raw signature bytes, Base64-encoded diff --git a/doc/manual/source/protocols/json/schema/signature-v2/simple.json b/doc/manual/source/protocols/json/schema/signature-v2/simple.json new file mode 100644 index 000000000000..abf69dfa0f92 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/signature-v2/simple.json @@ -0,0 +1,4 @@ +{ + "keyName": "cache.nixos.org-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +} diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v2 b/doc/manual/source/protocols/json/schema/store-object-info-v2 deleted file mode 120000 index 36ca7f13db94..000000000000 --- a/doc/manual/source/protocols/json/schema/store-object-info-v2 +++ /dev/null @@ -1 +0,0 @@ -../../../../../../src/libstore-tests/data/path-info/json-2 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v3 b/doc/manual/source/protocols/json/schema/store-object-info-v3 new file mode 120000 index 000000000000..d5dae48fcf69 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/path-info/json-3 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml similarity index 96% rename from doc/manual/source/protocols/json/schema/store-object-info-v2.yaml rename to doc/manual/source/protocols/json/schema/store-object-info-v3.yaml index 582b5e9eb476..bf4e4dafe144 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml @@ -1,6 +1,6 @@ "$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v2.json" -title: Store Object Info v2 +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v3.json" +title: Store Object Info v3 description: | Information about a [store object](@docroot@/store/store-object.md). @@ -44,10 +44,10 @@ $defs: properties: version: type: integer - const: 2 - title: Format version (must be 2) + const: 3 + title: Format version (must be 3) description: | - Must be `2`. + Must be `3`. This is a guard that allows us to continue evolving this format. Here is the rough version history: @@ -57,6 +57,8 @@ $defs: - Version 2: Use structured JSON type for `ca` + - Version 3: Use structured JSON type for `signatures` + path: "$ref": "./store-path-v1.yaml" title: Store Path @@ -101,7 +103,7 @@ $defs: type: string title: Store Directory description: | - The [store directory](@docroot@/store/store-path.md#store-directory) this store object belongs to (e.g. `/nix/store`). + The [path to the store directory](@docroot@/store/store-path.md#store-directory-path) this store object belongs within (e.g. `/nix/store`). additionalProperties: false impure: @@ -168,7 +170,7 @@ $defs: > This is an "impure" field that may not be included in certain contexts. items: - type: string + "$ref": "./signature-v2.yaml" # Computed closure fields closureSize: diff --git a/doc/manual/source/protocols/json/schema/store-path-v1.yaml b/doc/manual/source/protocols/json/schema/store-path-v1.yaml index 3cd7c56fcf79..31760c64a617 100644 --- a/doc/manual/source/protocols/json/schema/store-path-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-path-v1.yaml @@ -18,7 +18,7 @@ description: | The format follows this pattern: `${digest}-${name}` - - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) + - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md) (20 hash bytes become 32 ASCII characters) - **name**: The package name and optional version/suffix information type: string diff --git a/doc/manual/source/protocols/json/schema/store-v1.yaml b/doc/manual/source/protocols/json/schema/store-v1.yaml index e3e09c699621..eaf5c624b0be 100644 --- a/doc/manual/source/protocols/json/schema/store-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-v1.yaml @@ -37,7 +37,7 @@ properties: - contents properties: info: - "$ref": "./store-object-info-v2.yaml#/$defs/impure" + "$ref": "./store-object-info-v3.yaml#/$defs/impure" title: Store Object Info description: | Metadata about the [store object](@docroot@/store/store-object.md) including hash, size, references, etc. @@ -70,7 +70,7 @@ properties: "^[A-Za-z0-9+/]{43}=$": type: object additionalProperties: - "$ref": "./build-trace-entry-v2.yaml#/$defs/value" + "$ref": "./build-trace-entry-v3.yaml#/$defs/value" additionalProperties: false "$defs": diff --git a/doc/manual/source/protocols/json/signature.md b/doc/manual/source/protocols/json/signature.md new file mode 100644 index 000000000000..363b4ef11e0a --- /dev/null +++ b/doc/manual/source/protocols/json/signature.md @@ -0,0 +1,9 @@ +{{#include signature-v2-fixed.md}} + +## Examples + +### Simple signature + +```json +{{#include schema/signature-v2/simple.json}} +``` diff --git a/doc/manual/source/protocols/json/store-object-info.md b/doc/manual/source/protocols/json/store-object-info.md index 4ad83de00b3f..ea1cc3a9d223 100644 --- a/doc/manual/source/protocols/json/store-object-info.md +++ b/doc/manual/source/protocols/json/store-object-info.md @@ -1,45 +1,45 @@ -{{#include store-object-info-v2-fixed.md}} +{{#include store-object-info-v3-fixed.md}} ## Examples ### Minimal store object (content-addressed) ```json -{{#include schema/store-object-info-v2/pure.json}} +{{#include schema/store-object-info-v3/pure.json}} ``` ### Store object with impure fields ```json -{{#include schema/store-object-info-v2/impure.json}} +{{#include schema/store-object-info-v3/impure.json}} ``` ### Minimal store object (empty) ```json -{{#include schema/store-object-info-v2/empty_pure.json}} +{{#include schema/store-object-info-v3/empty_pure.json}} ``` ### Store object with all impure fields ```json -{{#include schema/store-object-info-v2/empty_impure.json}} +{{#include schema/store-object-info-v3/empty_impure.json}} ``` ### NAR info (minimal) ```json -{{#include schema/nar-info-v2/pure.json}} +{{#include schema/nar-info-v3/pure.json}} ``` ### NAR info (with binary cache fields) ```json -{{#include schema/nar-info-v2/impure.json}} +{{#include schema/nar-info-v3/impure.json}} ``` diff --git a/doc/manual/source/protocols/nix32.md b/doc/manual/source/protocols/nix32.md index 72afe893ea24..d8da1e9952cf 100644 --- a/doc/manual/source/protocols/nix32.md +++ b/doc/manual/source/protocols/nix32.md @@ -1,6 +1,6 @@ # Nix32 Encoding -Nix32 is Nix's variant of base-32 encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. +Nix32 is Nix's variant of [Base32](https://en.wikipedia.org/wiki/Base32) encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. ## Alphabet diff --git a/doc/manual/source/protocols/store-path.md b/doc/manual/source/protocols/store-path.md index 1aa79615d1c8..bdce19d1d62c 100644 --- a/doc/manual/source/protocols/store-path.md +++ b/doc/manual/source/protocols/store-path.md @@ -18,11 +18,9 @@ where - `name` = the name of the store object. -- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory) +- `store-dir` = the [path of the store directory](@docroot@/store/store-path.md#store-directory-path) -- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. - - Nix uses a custom base-32 encoding called [Nix32](@docroot@/protocols/nix32.md). +- `digest` = [Nix32](@docroot@/protocols/nix32.md) representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. For the definition of the hash compression algorithm, please refer to section 5.1 of the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf). diff --git a/doc/manual/source/release-notes/rl-2.23.md b/doc/manual/source/release-notes/rl-2.23.md index b358a0fdc3c3..92e5f4599440 100644 --- a/doc/manual/source/release-notes/rl-2.23.md +++ b/doc/manual/source/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/json-guideline.md). + The JSON format for derivations has been slightly revised to better conform to our [data modeling guidelines](@docroot@/development/data-modeling.md). In particular, the hash algorithm and content addressing method of content-addressed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. @@ -89,7 +89,7 @@ This makes records of this sort more self-describing, and easier to consume programmatically. We will follow this design principle going forward; - the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + the [data modeling guidelines](@docroot@/development/data-modeling.md) in the contributing section have been updated accordingly. - Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) diff --git a/doc/manual/source/release-notes/rl-2.30.md b/doc/manual/source/release-notes/rl-2.30.md index 34d3e5bab4c6..5a65ed99af29 100644 --- a/doc/manual/source/release-notes/rl-2.30.md +++ b/doc/manual/source/release-notes/rl-2.30.md @@ -13,7 +13,7 @@ - Deprecate manually making structured attrs using the `__json` attribute [#13220](https://github.com/NixOS/nix/pull/13220) The proper way to create a derivation using [structured attrs] in the Nix language is by using `__structuredAttrs = true` with [`builtins.derivation`]. - However, by exploiting how structured attrs are implementated, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. + However, by exploiting how structured attrs are implemented, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. This sneaky alternative method is now deprecated, and may be disallowed in future versions of Nix. [structured attrs]: @docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs diff --git a/doc/manual/source/release-notes/rl-2.32.md b/doc/manual/source/release-notes/rl-2.32.md index 5d90da0c9ebd..c59ecd6c2456 100644 --- a/doc/manual/source/release-notes/rl-2.32.md +++ b/doc/manual/source/release-notes/rl-2.32.md @@ -8,7 +8,7 @@ - Derivation JSON format now uses store path basenames only [#13570](https://github.com/NixOS/nix/issues/13570) [#13980](https://github.com/NixOS/nix/pull/13980) - Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra paramater (the store directory). + Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra parameter (the store directory). We ultimately want to rectify this issue with all JSON formats to the extent allowed by our stability promises. To start with, we are changing the JSON format for derivations because the `nix derivation` commands are — in addition to being formally unstable — less widely used than other unstable commands. diff --git a/doc/manual/source/release-notes/rl-2.33.md b/doc/manual/source/release-notes/rl-2.33.md index 810dcad00b15..0849f097cd0e 100644 --- a/doc/manual/source/release-notes/rl-2.33.md +++ b/doc/manual/source/release-notes/rl-2.33.md @@ -135,7 +135,7 @@ This is the legacy format, preserved for backwards compatibility: ### Version 2 (`--json-format 2`) -The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes: +The new structured format follows the [data modeling guidelines](@docroot@/development/data-modeling.md) with the following changes: - **Nested structure with top-level metadata**: @@ -149,9 +149,9 @@ The new structured format follows the [JSON guidelines](@docroot@/development/js } ``` - The map from store path base names to store object info is nested under the `info` field. + The map from [store path base names](@docroot@/store/store-path.md#base-name) to store object info is nested under the `info` field. -- **Store path base names instead of full paths**: +- **[Store path base names](@docroot@/store/store-path.md#base-name) instead of full paths**: Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths. Combined with `storeDir`, the full path can be reconstructed. diff --git a/doc/manual/source/release-notes/rl-2.34.md b/doc/manual/source/release-notes/rl-2.34.md index 89017f1767cf..91ee4ad5e9af 100644 --- a/doc/manual/source/release-notes/rl-2.34.md +++ b/doc/manual/source/release-notes/rl-2.34.md @@ -119,7 +119,7 @@ New command [`nix store roots-daemon`](@docroot@/command-ref/new-cli/nix3-store-roots-daemon.md) runs a daemon that serves garbage collector roots over a Unix domain socket. It enables the garbage collector to discover runtime roots when the main Nix daemon doesn't have `CAP_SYS_PTRACE` capability and therefore cannot scan `/proc`. - The garbage collector can be configured to use this daemon via the [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-experimental-option-use-roots-daemon) store setting. + The garbage collector can be configured to use this daemon via the [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-local-store-use-roots-daemon) store setting. This feature requires the [`local-overlay-store` experimental feature](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store). diff --git a/doc/manual/source/release-notes/rl-2.35.md b/doc/manual/source/release-notes/rl-2.35.md new file mode 100644 index 000000000000..a712b6385627 --- /dev/null +++ b/doc/manual/source/release-notes/rl-2.35.md @@ -0,0 +1,414 @@ +# Release 2.35.0 (2026-06-22) + +## Highlights + +- Sources are copied to the store more lazily [#3121](https://github.com/NixOS/nix/issues/3121) [#15711](https://github.com/NixOS/nix/pull/15711) [#15920](https://github.com/NixOS/nix/pull/15920) + + Historically, flakes source trees have been eagerly fetched to and evaluated from the Nix store to ensure deterministic and hermetic evaluation, even if the resulting store object is not used as a derivation input. This made the implementation simpler, yet made flakes unusable in large repositories and performed unnecessary writes to the store on each change to the source tree. + + Since Nix 2.32, all I/O (excluding `path:` and `hg+:`-style inputs) for reading sources during evaluation has been funneled to their original filesystem location (or to the `~/.cache/nix/tarball-cache-v2` bare git repository for tarball-based inputs). However, the source tree was still fetched to the store -- primarily for computing the resulting content-addressed store path. In most cases, (such as importing the `nixpkgs` package set) this is not necessary. + + Touching (and hashing the NAR serialisation of) the whole source tree is unavoidable, since: + + - In case of flake inputs, `narHash` integrity must be checked eagerly. + - The `outPath` attribute of a flake must be known in advance, and for backwards compatibility must be a content-addressed store path string with [constant string context](@docroot@/language/string-context.md#string-context-constant) representing the flake source tree. + + Even within the constraints imposed by backwards compatibility requirements, there are several improvements that are achievable. To reduce the number of copies performed, Nix now hashes the input without copying first, assuming that the `.outPath` will not end up in a derivation attribute and thus would never have to be actually fetched to the store. This comes at the slight cost of doing more work in case the assumption is wrong, but results in less work in typical use cases. The evaluator continues to behave as if the copy was performed: + + - Flakes are still evaluated from the store, from the evaluator's point of view. + - `toString ./.` continues to produce a content-addressed store path string without context. + - Path resolution crossing trees located in the filesystem and in Nix's view of it (with "virtual" overlays on top) continues to work. For example, the flake source tree can contain a relative symlink pointing outside its corresponding store object (though such usage is discouraged and makes further improvements to laziness intractable). + - Reading files from the flake's `outPath` continues to work. For example, such code is well-formed and is not considered [IFD](@docroot@/language/import-from-derivation.md): + + ```nix + builtins.readFile ( /. + (builtins.unsafeDiscardStringContext self.outPath) + "/flake.nix" ) + ``` + + Similar treatment has been applied to `builtins.fetchTarball`, which no longer eagerly copies paths to the store. + `builtins.storePath` now also short-circuits on "lazy-ish" store paths and doesn't substitute unless necessary. + + This change is expected to significantly reduce disk usage required for typical evaluations and results in ~2x speedup for fetching and unpacking a nixpkgs tarball (either via `fetchTree`/flakes or via `fetchTarball`). + +- Support FreeBSD `libjail` based sandboxing, add `x86_64-freebsd` to installer [#9968](https://github.com/NixOS/nix/pull/9968) [#13281](https://github.com/NixOS/nix/pull/13281) [#15673](https://github.com/NixOS/nix/pull/15673) + + The FreeBSD build of Nix now supports build sandboxing via FreeBSD jails and is enabled by default. + A FreeBSD build has been added to the traditional installer script. The beta rust-based installer is not yet supported. + FreeBSD support is not as well-tested as Linux or macOS, but is fully capable of building packages and performing other tasks expected of Nix on Linux. + +## Improvements + +- HTTP/3 (QUIC) support [#15961](https://github.com/NixOS/nix/pull/15961) + + Nix can now fetch from binary caches and other HTTP(S) sources over HTTP/3 (QUIC), controlled by a new [`http3`](@docroot@/command-ref/conf-file.md#conf-http3) setting (disabled by default). + When enabled, Nix requests HTTP/3 and transparently falls back to HTTP/2 or HTTP/1.1 for servers that do not advertise QUIC. + The setting only takes effect when linked against a `libcurl` built with HTTP/3 support, otherwise it is ignored and Nix keeps using HTTP/2 without warning or error. + + Enable it with: + + ``` + nix.conf: http3 = true + CLI: --http3 + ``` + + Or disable with: + + ``` + nix.conf: http3 = false + CLI: --no-http3 + ``` + +- Link mimalloc for faster evaluation [#15596](https://github.com/NixOS/nix/pull/15596) + + The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) by default, replacing glibc's malloc for all non-GC allocations. + This yields a **5–12% wall-clock improvement** on evaluation workloads, ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS configurations. + The allocator can be disabled at build time with `-Dmimalloc=disabled`. + +- The `revCount` attribute of the Git fetchers is now lazily computed and passed-through as-is when explicitly specified [#15772](https://github.com/NixOS/nix/pull/15772) [#14596](https://github.com/NixOS/nix/pull/14596) + + `revCount` and `lastModified` attributes passed to the Git fetcher are no longer eagerly validated when explicitly specified. + + When not explicitly specified, `revCount` is now also a thunk value and not computed eagerly. This delays this (potentially) expensive computation until the value is actually required. + +- Configurable file-transfer retry backoff with full jitter and `Retry-After` support [#15023](https://github.com/NixOS/nix/issues/15023) [#15419](https://github.com/NixOS/nix/issues/15419) [#15449](https://github.com/NixOS/nix/pull/15449) + + File transfer retries (downloads and uploads) now use AWS-style "full jitter" exponential backoff, treat HTTP 503 as rate-limited (same longer delay as 429), + and honor the `Retry-After` response header. + + Retry timing is configurable via new `nix.conf` settings: + + - [`filetransfer-retry-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay): base delay for transient errors + - [`filetransfer-retry-delay-rate-limited`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay-rate-limited): base delay for 429/503 + - [`filetransfer-retry-max-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-max-delay): per-attempt delay ceiling + - [`filetransfer-retry-jitter`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-jitter): enable full jitter + + The existing `download-attempts` setting has been renamed to [`filetransfer-retry-attempts`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-attempts) to reflect that it applies to uploads as well as downloads. + The old name remains as an alias for backwards compatibility. + + Per-substituter overrides are available as store reference parameters ([`retry-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay), [`retry-delay-rate-limited`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay-rate-limited), [`retry-max-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-max-delay), [`retry-attempts`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-attempts)), e.g. `s3://my-cache?retry-attempts=8`. + +- Improve daemon socket path logic for chroot stores [#15190](https://github.com/NixOS/nix/pull/15190) + + The default daemon socket path now uses the per-store [`state`](@docroot@/store/types/local-store.md#store-local-store-state) directory whenever one is defined, rather than always using the global [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR). + This means [local chroot stores](@docroot@/store/types/local-store.md#chroot) each get their own socket path automatically. + + Example: + + ```bash + nix-daemon --store /foo/bar + ``` + + will now use a socket at: + ``` + /foo/bar/nix/var/nix/daemon-socket/socket + ``` + instead of + ``` + $NIX_STATE_DIR/daemon-socket/socket + ``` + + Users who wish to serve or connect to a chroot store at the old location will have to force the socket location: + + - When serving (running a daemon), use the new [`--socket-path`](@docroot@/command-ref/new-cli/nix3-daemon.md#opt-socket-path) flag: + + ```bash + nix daemon --socket-path "$NIX_STATE_DIR/daemon-socket/socket" + ``` + + - When connecting as a client, put the path in the [store URL](@docroot@/store/types/local-daemon-store.md): + + ``` + unix://$NIX_STATE_DIR/daemon-socket/socket + ``` + +- Linux sandbox: also block `listxattr` syscalls [#15743](https://github.com/NixOS/nix/pull/15743) + + The Linux sandbox now also returns `ENOTSUP` for `listxattr`, `llistxattr` and `flistxattr`, matching the existing treatment of `getxattr`/`setxattr`/`removexattr`. + This prevents host xattrs (e.g. `security.selinux`) from leaking into builds and fixes tools such as `mkfs.ubifs` that probe xattr support via `listxattr`. + +- Support SCP-like URLs in fetchGit and type = "git" flake inputs [#14852](https://github.com/NixOS/nix/issues/14852) [#14867](https://github.com/NixOS/nix/issues/14867) [#14863](https://github.com/NixOS/nix/pull/14863) + + Nix now (once again) recognizes [SCP-like syntax for Git URLs](https://git-scm.com/docs/git-clone#_git_urls). This partially + restores compatibility with Nix 2.3 for `fetchGit`. The following syntax is once again supported: + + ```nix + builtins.fetchGit "host:/absolute/path/to/repo" + ``` + + Nix also passes through the tilde (for home directories) verbatim: + + ```nix + builtins.fetchGit "host:~/relative/to/home" + ``` + + IPv6 addresses also supported when bracketed: + + ```nix + builtins.fetchGit "user@[::1]:~/relative/to/home" + ``` + + `builtins.fetchTree` also supports this syntax now: + + ```nix + builtins.fetchTree { type = "git"; url = "host:/path/to/repo"; } + ``` + +- `nix flake check` now supports `--print-out-paths` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) and `--out-link` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) defaulting to not creating out links if the flag is not specified. + +- Added `--skip-alive` (and `--skip-live` alias for compatibility with Lix users) option to `nix store delete` for collecting garbage within a closure [#7239](https://github.com/NixOS/nix/issues/7239) [#15236](https://github.com/NixOS/nix/pull/15236) [#15727](https://github.com/NixOS/nix/pull/15727) + + `nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. + The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. + +- `builtins.getFlake` now supports path values [#15290](https://github.com/NixOS/nix/pull/15290) + + `builtins.getFlake` now accepts path values in addition to flakerefs. This improves the usability of relative flakes, allowing you to write `builtins.getFlake ./subflake`. + This change does not allow specifying paths that are not already in the store (though they do not have valid store objects, i.e. this will not force a copy if the flake has only been hashed -- and not copied to the store). This may change in a future release. + +- `nix-profile.fish` and `nix-profile-daemon.fish` now use `$NIX_LINK` for computing the value of `NIX_PROFILE` instead of `$HOME/.nix-profile` [#14293](https://github.com/NixOS/nix/pull/14293) + +- `nix` binary now exports symbols from C bindings [#15696](https://github.com/NixOS/nix/pull/15696) + + This allows Nix plugins written against the C API to look up symbols dynamically without linking to corresponding `libnix*c.so` libraries. + +- The computed Git LFS endpoint URLs have been fixed to follow the spec [#15891](https://github.com/NixOS/nix/pull/15891) and memory usage of LFS fetches has been decreased [#15912](https://github.com/NixOS/nix/pull/15912) + +- We now verify that fetched Git LFS objects have the same OID as requested [#15845](https://github.com/NixOS/nix/pull/15845) + +- Primop documentation now includes time complexity information [#14554](https://github.com/NixOS/nix/pull/14554) + +- Improved documentation on store paths and derivation building [#14699](https://github.com/NixOS/nix/pull/14699) + +- The [build hook](@docroot@/command-ref/conf-file.md#conf-build-hook) is now killed with `SIGTERM` instead of `SIGKILL` [#15105](https://github.com/NixOS/nix/pull/15105) + +- Download/upload logs strip `userinfo` URL components [#15715](https://github.com/NixOS/nix/pull/15715) + +## Content-addressed derivations changes + + The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. + This changes the binary cache endpoints for realisations and the daemon/nix-serve protocol (gated behind a daemon protocol feature flag). + +- Realisations keyed by store path instead of hash modulo [#11897](https://github.com/NixOS/nix/issues/11897) [#12464](https://github.com/NixOS/nix/pull/12464) + + Previously, a build trace entry (realisation) was keyed by the hash modulo of the derivation. In simpler terms, derivations transitively depending on distinct fixed-output derivations with the same `outPath` would share a realisation. + + Now, build trace entries are keyed by the regular derivation store path (`.drvPath`) plus the output name. For example, instead of: + + ``` + sha256:ba7816bf8f01...!out + ``` + + The key is now: + + ``` + /nix/store/abc...-foo.drv^out + ``` + +- Removed support for "deep" realisations [#15289](https://github.com/NixOS/nix/pull/15289) + + Previously the build trace (set of "realisations") contained entries for both unresolved and [resolved](@docroot@/store/resolution.md) derivations. + Now, it contains entries exclusively for resolved derivations. + For now, unresolved derivations will be resolved from these underlying build trace entries. + This is slower, but has the benefit of making build trace entries stateless and self-describing --- making sharing realisations easier between stores. + + This change necessitates changes to the binary cache format: + + - The directory for build traces moved from `realisations/` to `build-trace-v2/`. + - File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. + - The JSON format of build trace entries is now split into `key` and `value` objects: + ```json + { + "key": { + "drvPath": "abc...-foo.drv", + "outputName": "out" + }, + "value": { + "outPath": "xyz...-foo", + "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] + } + } + ``` + Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. + - The deprecated `dependentRealisations` field has been removed. + + The build trace entries stored in the local SQLite database no longer have any foreign key references to store objects. + This is because the build trace entries for resolved derivations that may have been deleted need to be preserved, otherwise the outputs of other unresolved derivations will be effectively forgotten. + GC for the build trace is not yet implemented due to the lack of a clear default policy. + +- Structured signature for realisations and `path-info` [#15009](https://github.com/NixOS/nix/pull/15009) + + [Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. + `nix path-info --json --json-format 3` opts into the new version for this command. + JSON parsing accepts both the old string format and new structured format for backwards compatibility. + + This format is also used for the build trace entries in binary caches. + +- `nix realisation` command has been renamed to `nix store build-trace` [#16000](https://github.com/NixOS/nix/pull/16000) [#15948](https://github.com/NixOS/nix/pull/15948) + +## Build performance improvements + +- Make post-build-hook asynchronous [#15406](https://github.com/NixOS/nix/issues/15406) [#15451](https://github.com/NixOS/nix/pull/15451) + + The [`post-build-hook`](@docroot@/command-ref/conf-file.md#conf-post-build-hook) now runs asynchronously, without blocking the build event loop. + Dependent builds are not started until the hook finishes, but multiple hook instances are now launched concurrently -- up to the [`max-jobs`](@docroot@/command-ref/conf-file.md#conf-max-jobs) limit. + +- zstd compression now emits multi-frame output and uses less memory [#15550](https://github.com/NixOS/nix/pull/15550) + + zstd-compressed NARs are now written as a sequence of independent 16 MiB frames instead of a single large frame. + This lays the groundwork for parallel decompression in a future release without requiring caches to be repopulated, and significantly lowers peak memory use during compression + (e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). + + The output remains standard zstd and is decoded unchanged by existing Nix binaries and the `zstd` CLI; compression ratio is effectively unchanged. + + Per-frame compression now uses up to 4 worker threads. For zstd this is the new default: the [`parallel-compression`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-parallel-compression) store setting defaults to `true` when `compression=zstd` (it remains `false` for other compression algorithms like `xz`). + Set `?parallel-compression=false` to opt out. + +- More parallelism for binary cache uploads [#15957](https://github.com/NixOS/nix/pull/15957) + + Uploads of NARs now start without waiting for all references to be uploaded. + Also, NARs are now uploaded in order of descending (decompressed) NAR size. + The closure invariant is still maintained by copying `.narinfo` in a topologically sorted order. + +- The derivation build scheduler memory usage reduction and performance improvements [#15611](https://github.com/NixOS/nix/pull/15611) [#15695](https://github.com/NixOS/nix/pull/15695) + + Memory usage of the derivation build scheduler has been improved to allow more state sharing. + Inefficiencies leading to quadratic complexity of scheduling build/substitution jobs have been addressed. + Scheduling resources are allocated more sparingly and freed earlier to reduce peak consumption. + + These improvements amount to ~2-8x less `nix-daemon` memory usage for typical workloads and more in larger derivation graphs, not accounting for short-lived allocations used during substitution. + + Notably, the current architecture of the build scheduler gets proportionally slower on Linux with larger heaps as derivation "builder" processes are `fork`-ed directly from the Nix process, which blocks the builder event loop for the duration of the `fork`. Thus, smaller heap of `nix-daemon` translates into faster build startups. + +- Concurrent path substitutions and eval-time fetches of the same inputs now run only once [#15555](https://github.com/NixOS/nix/pull/15555) [#15644](https://github.com/NixOS/nix/pull/15644) + + This avoids redundant work in case multiple Nix processes try to substitute/download the same resource concurrently. + +- `.narinfo` lookups in binary caches are more concurrent + + Querying the existence and path metadata in binary caches is now more asynchronous. Operations like `nix path-info` on large closures are faster and more efficient. + The build scheduler event loop now doesn't block on `.narinfo` queries, which improves performance with passthru binary caches. + +## Bug fixes + +- Fix hash collision between store paths with self-references and their zeroed-out equivalents [#15837](https://github.com/NixOS/nix/issues/15837) [#15931](https://github.com/NixOS/nix/pull/15931) + + When computing the hash of a NAR with self-references, Nix zeroes out the self-references but also hashes their positions. + The latter was accidentally lost in Nix 2.17.0, which meant a NAR with self-references could hash to the same store path as an otherwise-identical NAR in which some of the self-references had been zeroed out. + + This release restores hashing the positions of self-references. + As a consequence, content-addressed store paths derived from self-referential NARs will differ from those produced by Nix 2.17 through 2.34. + This affects users of the experimental `ca-derivations` features, as well as users of `nix store make-content-addressed`. + +- C API: Fix `EvalState` pointer passed to primop callbacks [#15300](https://github.com/NixOS/nix/pull/15300) [#15383](https://github.com/NixOS/nix/pull/15383) + + The `EvalState *` passed to C API primop callbacks was incorrectly pointing to the internal `nix::EvalState` rather than the C API wrapper struct. + This caused a segfault when the callback used the pointer with C API functions such as `nix_alloc_value()`. + The same issue affected `printValueAsJSON` and `printValueAsXML` callbacks on external values. + +- GitHub fetcher now validates URL parameters [#15304](https://github.com/NixOS/nix/issues/15304) [#15331](https://github.com/NixOS/nix/pull/15331) + + The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. + +- Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands [#15776](https://github.com/NixOS/nix/pull/15776) + + Setting [`keep-derivations`](@docroot@/command-ref/conf-file.md#conf-keep-derivations) to `true` and trying to delete a derivation with realised outputs would previously fail. + Same with [`keep-outputs`](@docroot@/command-ref/conf-file.md#conf-keep-outputs) and trying to delete an output that still has derivers. + These options no longer affect the deletion commands, and are now documented as such. + +- S3 substituters fall back to the URL's region for STS WebIdentity auth [#15594](https://github.com/NixOS/nix/pull/15594) + + When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL as a fallback for the STS endpoint region if neither `AWS_REGION` nor `AWS_DEFAULT_REGION` is set. + Previously, IRSA setups that exported `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail with a misleading "IMDS provider" error. + +- S3: restore STS WebIdentity and ECS container credential providers [#15507](https://github.com/NixOS/nix/pull/15507) + + Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a custom chain built on `aws-c-auth`. + That chain omitted two providers, breaking S3 binary cache access in container workloads: + + - **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, `AWS_ROLE_SESSION_NAME`) -- used by EKS IRSA, GitHub Actions OIDC, and any `sts:AssumeRoleWithWebIdentity` federation. + - **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, `AWS_CONTAINER_CREDENTIALS_FULL_URI`) -- used by ECS tasks and EKS Pod Identity. + + The typical symptom was a misleading IMDS error (`Valid credentials could not be sourced by the IMDS provider`), because IMDS is the last provider tried after the correct one was skipped. + + Both providers are now part of the chain, ordered to match the pre-2.33 behaviour. + As in both the old and new AWS SDK default chains, ECS and IMDS are mutually exclusive: when container credential environment variables are set, IMDS is skipped. + +- HTTP 401 and 407 responses from binary caches are no longer treated as missing files [#15877](https://github.com/NixOS/nix/pull/15877) + + Nix no longer treats `Unauthorized` and `Proxy Authentication Required` HTTP codes as an indication of a missing file. This used to be the case because AWS S3 returns 403 `Forbidden` for missing objects in unlistable buckets. 401/407 were accidentally included and this workaround is now tightly scoped to 403 responses. + +- Fixed `nixbld` gid in `/etc/group` in the Linux build sandbox when user namespaces are not supported [#15131](https://github.com/NixOS/nix/pull/15131) + +- Store garbage collection is now more robust [#15992](https://github.com/NixOS/nix/pull/15992) [#15720](https://github.com/NixOS/nix/pull/15720) [#15616](https://github.com/NixOS/nix/pull/15616) + +- Fixed deadlock for hash-mismatching fixed-output derivations [#15874](https://github.com/NixOS/nix/pull/15874) + +- `nix-copy-closure` no longer ignores `--include-outputs` flag [#15896](https://github.com/NixOS/nix/pull/15896) + +- Fixes to `recursive-nix` experimental feature + + Prior to this release, internal datastructures used to implement this feature were not used in a thread-safe manner. + Threads handling daemon connections are now reaped promptly, fixing resource leaks. + +## Contributors + +This release was made possible by the following 59 contributors: + +- Michael Wang [**(@zwang20)**](https://github.com/zwang20) +- Amaan Qureshi [**(@amaanq)**](https://github.com/amaanq) +- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium) +- Reuben Gardos Reid [**(@ReubenJ)**](https://github.com/ReubenJ) +- StepBroBD [**(@stepbrobd)**](https://github.com/stepbrobd) +- dram [**(@dramforever)**](https://github.com/dramforever) +- Tom [**(@thunze)**](https://github.com/thunze) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- steveoliphant [**(@steveoliphant)**](https://github.com/steveoliphant) +- espes [**(@espes)**](https://github.com/espes) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Artemis Tosini [**(@artemist)**](https://github.com/artemist) +- sander [**(@sandydoo)**](https://github.com/sandydoo) +- Erik Jensen [**(@rkjnsn)**](https://github.com/rkjnsn) +- Cameron Will [**(@cwill747)**](https://github.com/cwill747) +- Maciej Krüger [**(@mkg20001)**](https://github.com/mkg20001) +- Dror Speiser [**(@drorspei)**](https://github.com/drorspei) +- Eveeifyeve [**(@Eveeifyeve)**](https://github.com/Eveeifyeve) +- Audrey Dutcher [**(@rhelmot)**](https://github.com/rhelmot) +- Lisanna Dettwyler [**(@lisanna-dettwyler)**](https://github.com/lisanna-dettwyler) +- TyIsI [**(@TyIsI)**](https://github.com/TyIsI) +- Adam Kliś [**(@BonusPlay)**](https://github.com/BonusPlay) +- Domen Kožar [**(@domenkozar)**](https://github.com/domenkozar) +- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii) +- ryota2357 [**(@ryota2357)**](https://github.com/ryota2357) +- LIN, Jian [**(@jian-lin)**](https://github.com/jian-lin) +- znmz [**(@znmz)**](https://github.com/znmz) +- Felix Stupp [**(@Zocker1999NET)**](https://github.com/Zocker1999NET) +- Johannes Kirschbauer [**(@hsjobeki)**](https://github.com/hsjobeki) +- Antonio Nuno Monteiro [**(@anmonteiro)**](https://github.com/anmonteiro) +- tomberek [**(@tomberek)**](https://github.com/tomberek) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- adisbladis [**(@adisbladis)**](https://github.com/adisbladis) +- Luna Nova [**(@LunNova)**](https://github.com/LunNova) +- Riccardo Mazzarini [**(@noib3)**](https://github.com/noib3) +- Bouke van der Bijl [**(@bouk)**](https://github.com/bouk) +- Dario [**(@dve00)**](https://github.com/dve00) +- Michael Hoang [**(@Enzime)**](https://github.com/Enzime) +- Paul Sbarra [**(@tones111)**](https://github.com/tones111) +- edef [**(@edef1c)**](https://github.com/edef1c) +- Adam Dinwoodie [**(@me-and)**](https://github.com/me-and) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Jeremy Fleischman [**(@jfly)**](https://github.com/jfly) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Alex Ionescu [**(@aionescu)**](https://github.com/aionescu) +- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy) +- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault) +- Pierre Penninckx [**(@ibizaman)**](https://github.com/ibizaman) +- Leonard Sheng Sheng Lee [**(@sheeeng)**](https://github.com/sheeeng) +- rszyma [**(@rszyma)**](https://github.com/rszyma) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Lennart Kolmodin [**(@kolmodin)**](https://github.com/kolmodin) +- zowoq [**(@zowoq)**](https://github.com/zowoq) +- Peter Collingbourne [**(@pcc)**](https://github.com/pcc) +- Simon Žlender [**(@szlend)**](https://github.com/szlend) +- Lily Foster [**(@lilyinstarlight)**](https://github.com/lilyinstarlight) +- randomizedcoder [**(@randomizedcoder)**](https://github.com/randomizedcoder) +- Krish Jaiswal diff --git a/doc/manual/source/store/build-trace.md b/doc/manual/source/store/build-trace.md index a879d37d208d..cb9cb3099680 100644 --- a/doc/manual/source/store/build-trace.md +++ b/doc/manual/source/store/build-trace.md @@ -8,7 +8,7 @@ The *build trace* is a [memoization table](https://en.wikipedia.org/wiki/Memoization) for builds. It maps the inputs of builds to the outputs of builds. -Concretely, that means it maps [derivations][derivation] to maps of [output] names to [store objects][store object]. +Each *[entry]{#entry}* in the build trace maps a [derivation][derivation] to a map of [output] names to [store objects][store object]. In general the derivations used as a key should be [*resolved*](./resolution.md). A build trace with all-resolved-derivation keys is also called a *base build trace* for extra clarity. diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index 32e800129342..087413406487 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -1,72 +1,178 @@ # Building -## Normalizing derivation inputs +As discussed in the [main page on derivations](./derivation/index.md): -- Each input must be [realised] prior to building the derivation in question. +> A derivation is a specification for running an executable on precisely defined input to produce one or more [store objects][store object]. + +This page describes *building* a derivation, which is to say following the instructions in the derivation to actually run the executable. +Some elements of derivations are self-explanatory. +For example, the arguments specified in the derivation really are the arguments passed to the executable. +In other cases, however, there is additional common steps performed by Nix for all derivations --- mostly for setting up the build environment and collecting the built outputs. + +The chief design consideration for the building process is *determinism*. +Conventional operating systems are typically not designed with determinism in mind. +But determinism is needed to make Nix's build caching a transparent abstraction. + +> **Explanation** +> +> For example, no one wants to slightly modify a derivation, and then find that it no longer builds for an unrelated reason, because the original derivation *also* doesn't build anymore, but the cache hit on the original derivation was hiding this. +> We want builds that succeed once to continue succeeding, to encourage fearless modification of old build recipes. +> Determinism is what enables things that once worked to keep working. + +The life cycle of a build can be broken down into 3 parts: + +1. Spawn the builder process with the proper environment, including the correct process arguments, environment variables, and file system state. + +2. Wait for the builder process to exit and collect its exit status. + Exit code 0 means success; anything else is a build failure. + (Strictly speaking, Nix detects process exit by waiting for the standard output and error streams to close. + If a builder explicitly closes these streams without exiting, Nix will kill it, and deem the build a failure. + Processes should therefore exit *without* explicitly closing those standard streams, and let the exiting of the process close them implicitly.) + + Nix also logs the standard output and error of the process, but this is just for human convenience and does not influence the behavior of the system. + (Builder processes have no idea what the consumer of their standard output and error does with the pseudo-terminal master, only that they are indeed consumed so buffers do not fill up etc. and writes to each output standard stream will continue to succeed. + In practice, Nix will store the log in `/nix/var/log/nix`) + +3. Processing the outputs after the builder has exited. + + The builder process on exit should have left behind files for each output the derivation is supposed to produce. + The files must be processed to turn them into bona fide store objects. + If the processing succeeds, those store objects are associated with the derivation as (the results of) a successful build. + +Step (3) is done by Nix externally to the build itself, which is just steps (1) and (2). +In step (3), just inert data is processed, since the builder process has exited or been killed by then. +Step (1) however is best described not from Nix's perspective, but from the build process's perspective. + +> **Explanation** +> +> Ultimately, what matters for determinism is what the build process can observe: what resources (files, networking, etc.) it can see, what syscalls succeed or fail, etc. +> Nix can achieve this through many different sandboxing strategies (namespaces, VMs, chroots, ...), but the process shouldn't be able to tell them apart. +> We therefore specify building from the process's perspective, not Nix's perspective, to focus on *what*, not *how*. + +## What derivations can be built + +Actually only some derivations are ready to be built. +In particular, only [*resolved*](./resolution.md) derivations can be built. +That is to say, a derivation that depends on other derivations is not ready yet to be built, because some of those other derivations might not have yet been built. +If the other derivations are indeed all built, we can witness this fact by resolving the derivation, and converting all the derivation's input references into plain store paths. + +> **Note** +> +> Note that [input-addressing](derivation/outputs/input-address.md) derivations are improperly resolved. +> As discussed on the linked page, the current input-addressing algorithm does not respect resolution-equivalence of derivations (\\(\\sim_\mathrm{Drv}\\)). +> That means that if Nix properly resolved an input-addressed derivation, the resolved derivation would have different input addresses, violating expectations. +> Nix therefore improperly resolves the derivation, keeping its original input-addressed output paths, creating an invalid derivation that is both resolved and instructed to create the outputs at the originally expected paths. + +## Environment of the builder process + +This section describes how the [`builder`](./derivation/index.md#builder) is executed. + +> **Implementation detail** +> +> Nix prevents multiple [Nix instances][Nix instance] from performing the same build at the same time, for example by acquiring exclusive file locks. + +### File system + +The builder should have access to a limited file system where only certain objects are available. +The most important exposed files are the inputs (other store objects) of the (resolved) derivation. +Additionally, some other files are exposed. + +#### Store inputs + +The builder will be run against a file system in which the [store directory][store directory path] contains the [closure] of the inputs. +In particular, consider a store that just contains this closure. +That store is exposed to the file system according to the rules specified in the [Exposing Store Objects in OS File Systems](./store-path.md#exposing) documentation. +This precisely defines the file system layout of the store that should be visible to the builder process. + +> **Note** +> +> Historically, Nix exposed *at least* the following store contents to the builder, but also arbitrarily other store objects, due to limitations around operating systems' file system virtualization capabilities, and wanting to avoid copying or moving files. +> It still can do this in so-called *unsandboxed* builds. +> +> Such builds should be considered discouraged, but one that works less badly against non-mischievous derivations than might be expected. +> This is because store paths are relatively unpredictable, so a well-behaved program is unlikely to stumble upon a store object it wasn't supposed to know about. +> +> As operating systems developed better file system primitives, the need for disabling sandboxing has lessened greatly over the years, and this trend should continue into the future. + +The outputs are expected to be created in that store directory as if they were valid store objects. +(They are just files during builder execution, but during [processing outputs](#processing-outputs) they will be turned into proper store objects.) +The [environment variables](#env-vars) for each output indicate where the builder should write them; +Nix ensures that those paths do not yet exist when the builder is run. + +> **Note** +> +> In sandboxed builds, ensuring that the outputs do not exist in the store directory is trivial. +> In unsandboxed builds, it is harder in general. +> In the worst case, the derivation is in fact rewritten so different output paths are used instead, and then the outputs are rewritten back to the intended output paths after. +> In the content-addressing case rewriting would be needed either way, but in the input-addressing case, this is a significant degradation, as the point of input addressing is to avoid rewrites by knowing output paths in advance. [realised]: @docroot@/glossary.md#gloss-realise +[closure]: @docroot@/glossary.md#gloss-closure +[store directory path]: ./store-path.md#store-directory-path + +### Other file system state + +- The current working directory of the builder process will be a fresh temporary directory. + It is initially empty when the process starts except for a few input files: -- Once this is done, the derivation is *normalized*, replacing each input deriving path with its store path, which we now know from realising the input. + - If [`__structuredAttrs`](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs) is enabled: `.attrs.json` (the derivation attributes as JSON) and `.attrs.sh` (a Bash-compatible rendering of the same). + The environment variables `NIX_ATTRS_JSON_FILE` and `NIX_ATTRS_SH_FILE` point to these files, respectively. -## Builder Execution {#builder-execution} + - If [`passAsFile`](@docroot@/language/advanced-attributes.md#adv-attr-passAsFile) is used (only without `__structuredAttrs`): for each attribute name listed, a file `.attr-` where `` is the [Nix32](@docroot@/protocols/nix32.md)-encoded SHA-256 hash of the attribute name. + The environment variable `Path` points to the file containing the attribute's value. -The [`builder`](./derivation/index.md#builder) is executed as follows: + In sandboxed builds, this directory is at a deterministic path inside the sandbox (controlled by the [`sandbox-build-dir`](@docroot@/command-ref/conf-file.md#conf-sandbox-build-dir) setting, default `/build`). + See also the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for the host-side location. -- A temporary directory is created where the build will take place. The - current directory is changed to this directory. +- Basic device nodes for essential operations (null device, random number generation, standard streams as a pseudo terminal) - See the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for more information. + (A pseudo terminal would not be strictly necessary since the standard streams are passively logging, not there to facilitate interaction. + But it is still useful to entice programs to do nicer logging with e.g. colors etc.) -- The environment is cleared and set to the derivation attributes, as - specified above. +- On Linux: Process information via `/proc` -- In addition, the following variables are set: +- Minimal user and group identity information - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. +- A loopback-only network configuration with hostname set to `localhost` - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. +> **Note** +> +> Fixed-output derivations have access to additional operating system state to facilitate communication with the outside world, such as network name resolution and TLS certificate verification. +> This is necessary because these derivations are allowed to access the network, unlike regular derivations which are fully sandboxed. - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. +### Environment variables {#env-vars} - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. +The environment is cleared and set to the derivation attributes, as +specified above. - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). +For most derivations types this must contain at least: - - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` - is set to `true` for the derivation. A detailed explanation of this - behavior can be found in the - [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). +- For each output declared in `outputs`, the corresponding environment variable is set to point to the intended path in the Nix store for that output. + Each output path is a concatenation of the cryptographic hash of all build inputs, the `name` attribute and the output name. + (The output name is omitted if it's `out`.) - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) +In addition, the following variables are set: -- If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple [Nix instances][Nix instance] from performing the same - build at the same time. +- `NIX_BUILD_TOP` contains the path of the temporary directory for this build. -- A log of the combined standard output and error is written to - `/nix/var/log/nix`. +- Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the temporary directory. + This is to prevent the builder from accidentally writing temporary files anywhere else. + Doing so might cause interference by other processes. -- The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. +- `PATH` is set to `/path-not-set` to prevent shells from initialising it to their built-in default value. -- The temporary directory is removed (unless the `-K` option was - specified). +- `HOME` is set to `/homeless-shelter`. + (Without sandboxing, this discourages programs from using `/etc/passwd` or the like to find the user's home directory, which could cause impurity.) + Usually, when `HOME` is set, it is used as the location of the home directory, even if it points to a non-existent path. + +- `NIX_STORE` is set to the path of the top-level Nix [store directory path] (typically, `/nix/store`). + +- `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` is set to `true` for the derivation. + A detailed explanation of this behavior can be found in the [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). + +### Arguments + +The builder is passed the arguments specified by the derivation attribute `args`. ## Processing outputs @@ -74,28 +180,40 @@ If the builder exited successfully, the following steps happen in order to turn - **Normalize the file permissions** - Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Any possible `setuid` and `setgid` - bits are cleared. - - > **Note** - > - > Setuid and setgid programs are not currently supported by Nix. - > This is because the Nix archives used in deployment have no concept of ownership information, - > and because it makes the build result dependent on the user performing the build. + The files must conform to the model described in the [Exposing in OS file systems](./file-system-object/os-file-system.md) section. + For example, timestamps and permissions are canonicalised. - **Calculate the references** - Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. + Nix scans each output path for [references] to input store objects by looking for the [digest][store path digest] of each input. + (The name part and the [store directory path] are ignored when scanning; an input's hash part that is neither followed by a `-` nor proceeded by a `/` still scans as a reference.) + Since these are potential runtime dependencies, Nix will register them as references of the output store object they occur in. + + Nix also scans for references from one output to another in the same way, because outputs are allowed to refer to each other. + + The outputs' references must form a [directed acyclic graph](@docroot@/glossary.md#gloss-directed-acyclic-graph). + (This is not a special restriction for outputs; it is true for the references of all store objects in general.) + + In the case of derivations with output paths that are fixed in advance (i.e. [input-addressing] derivations, or [fixed content-addressing] derivations), the actual final store path to each output is used during the build if possible. + For [floating content-addressing] derivations, however, the final store path is not known in advance by definition. + Scratch store paths must therefore be used instead. + Scanning will use those scratch paths, but then any output-to-be that contains such a scanned scratch path must be rewritten to instead use the final (content-addressed) path of the output in question. - Nix also scans for references to other outputs' paths in the same way, because outputs are allowed to refer to each other. - If the outputs' references to each other form a cycle, this is an error, because the references of store objects much be acyclic. +At this point, the file system data is in the proper form, and the valid acyclic reference data for each output is also calculated, so the outputs are added to the store as proper store objects. +Additionally, those store objects (at least in the case that they are [content-addressed][content-addressing]) can be associated with the derivation in the [build trace] in the record for a successful build. +> **Implementation detail** +> +> Nix will normally clean up and remove the temporary build directory after every build, successful or unsuccessful. +> The builder doesn't know whether Nix does or not, however, as it will have exited before the build directory is cleaned up, and it will not see any old build directory if (after a failed build) it is run again. +> The [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) option can be specified to keep the build directory in the case of a failing build. +[references]: ./store-object.md#references +[store path digest]: ./store-path.md#digest +[store object]: ./store-object.md [Nix instance]: @docroot@/glossary.md#gloss-nix-instance +[content-addressing]: ./derivation/outputs/content-address.md +[input-addressing]: ./derivation/outputs/input-address.md +[fixed content-addressing]: ./derivation/outputs/content-address.md#fixed +[floating content-addressing]: ./derivation/outputs/content-address.md#floating +[build trace]: ./build-trace.md diff --git a/doc/manual/source/store/derivation/outputs/content-address.md b/doc/manual/source/store/derivation/outputs/content-address.md index aa65fbe4932a..9fadcff3c6ca 100644 --- a/doc/manual/source/store/derivation/outputs/content-address.md +++ b/doc/manual/source/store/derivation/outputs/content-address.md @@ -13,7 +13,7 @@ Given the method, the output's name (computed from the derivation name and outpu ## Fixed-output content-addressing {#fixed} In this case the content address of the *fixed* in advanced by the derivation itself. -In other words, when the derivation has finished [building](@docroot@/store/building.md), and the provisional output' content-address is computed as part of the process to turn it into a *bona fide* store object, the calculated content address must much that given in the derivation, or the build of that derivation will be deemed a failure. +In other words, when the derivation has finished [building](@docroot@/store/building.md), and the provisional output' content-address is computed as part of the process to turn it into a *bona fide* store object, the calculated content address must match that given in the derivation, or the build of that derivation will be deemed a failure. The output spec for an output with a fixed content addresses additionally contains: diff --git a/doc/manual/source/store/derivation/outputs/index.md b/doc/manual/source/store/derivation/outputs/index.md index ca2ce6665b04..9b46405cdf58 100644 --- a/doc/manual/source/store/derivation/outputs/index.md +++ b/doc/manual/source/store/derivation/outputs/index.md @@ -9,7 +9,7 @@ The outputs specification is a map, from names to specifications for individual ## Output Names {#outputs} -Output names can be any string which is also a valid [store path](@docroot@/store/store-path.md) name. +Output names can be any string which is also a valid [store path name](@docroot@/store/store-path.md#name). The name mapped to each output specification is not actually the name of the output. In the general case, the output store object has name `derivationName + "-" + outputSpecName`, not any other metadata about it. However, an output spec named "out" describes and output store object whose name is just the derivation name. diff --git a/doc/manual/source/store/derivation/outputs/input-address.md b/doc/manual/source/store/derivation/outputs/input-address.md index 3fd20f17d724..6df9b94961e8 100644 --- a/doc/manual/source/store/derivation/outputs/input-address.md +++ b/doc/manual/source/store/derivation/outputs/input-address.md @@ -16,7 +16,7 @@ Concretely, this would cause a "mass rebuild" whenever any fetching detail chang To solve this problem, we compute output hashes differently, so that certain output hashes become identical. We call this concept quotient hashing, in reference to quotient types or sets. -So how do we compute the hash part of the output paths of an input-addressed derivation? +So how do we compute the [hash part](@docroot@/store/store-path.md#digest) of the output paths of an input-addressed derivation? This is done by the function `hashQuotientDerivation`, shown below. First, a word on inputs. diff --git a/doc/manual/source/store/file-system-object.md b/doc/manual/source/store/file-system-object.md index 60cb3e572063..2ffad5f0ec94 100644 --- a/doc/manual/source/store/file-system-object.md +++ b/doc/manual/source/store/file-system-object.md @@ -19,7 +19,7 @@ Every file system object is one of the following: In general, Nix does not assign any semantics to symbolic links. Certain operations however, may make additional assumptions and attempt to use the target to find another file system object. - > See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. + > See [the Wikipedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. File system objects and their children form a tree. A bare file or symlink can be a root file system object. diff --git a/doc/manual/source/store/file-system-object/os-file-system.md b/doc/manual/source/store/file-system-object/os-file-system.md new file mode 100644 index 000000000000..6ce21f7788c9 --- /dev/null +++ b/doc/manual/source/store/file-system-object/os-file-system.md @@ -0,0 +1,40 @@ +# Exposing File System Objects in real operating system file systems + +Nix's [file system object] data model is minimal. +All the various other bits and pieces of real world filesystem interfaces, such as [extended file attributes](https://en.wikipedia.org/wiki/Extended_file_attributes), are specifically ignored to reduce our interface surface and the reproducibility issues associated with a larger interface. +In the view of Nix's developers, the types of simple, fine-grained batch jobs (typically, building software) that Nix specializes in simply don't benefit enough from that extra complexity for it to be worth the costs of supporting it. + +But to actually be used by software, file system objects need to be made available through the operating system's file system. +This is sometimes called "mounting" or "exposing" the file system object, though do note it may or may not be implemented with what the operating system calls "mounting". + +[file system object]: ../file-system-object.md + +## Metadata normalization + +File systems typically contain other metadata that is outside Nix's data model. +To avoid this other metadata being a side channel and source of nondeterminism, Nix is careful to normalize to fixed values. +For example, on Unix, the following metadata normalization occurs: + +- The creation and last modification timestamps on all files are set to Unix Epoch 1s (00:00:01 1/1/1970 UTC) + +- The group is set to the [default group](@docroot@/command-ref/conf-file.md#conf-build-users-group) + +- The Unix mode of the file to 0444 or 0555 (i.e., read-only, with execute permission enabled if the file was originally executable). + +- Any possible `setuid` and `setgid` bits are cleared. + + > **Note** + > + > `setuid` and `setgid` programs are not currently supported by Nix. + > These special file system permissions are in general a security footgun, and with data owned by different users in different stores, it would especially be a hazard when copying store objects between stores. + > + > This restriction has not proved to be onerous in practice. + > For example, NixOS uses so called setuid-wrappers which are outside the store. + +> **Explanation** +> +> As discussed before, Nix essentially shares its file system object data model with other tools like Git. +> But those tools tend to ignore this metadata in both directions --- when reading files, like Nix, but when writing files, timestamps are set organically, and the user is free to set other special permissions (`setuid`, `setgid`, sticky, etc.) however they like. +> Normalizing, and not just ignoring, this metadata is therefore what distinguishes Nix from these other tools more than the file system object data model itself. +> +> Nix's approach is motivated by deterministic building. Whereas Git can assume that humans running commands will simply ignore timestamps etc. as appropriate, understanding they are local and ephemeral, Nix aims to run software that was not necessarily designed with Nix in mind, and is unaware of whatever sandboxing/virtualization is in place. diff --git a/doc/manual/source/store/index.md b/doc/manual/source/store/index.md index f1e8f1402988..d063fc4fdc05 100644 --- a/doc/manual/source/store/index.md +++ b/doc/manual/source/store/index.md @@ -2,4 +2,33 @@ The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. -There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). +Concretely, albeit using concepts that are only defined in the rest of the chapter, a store consists of: + +- A set of [store objects][store object], the immutable file system data. + + This can also be looked at as a map from [store paths][store path] to store objects. + +- A set of [derivations][derivation], instructions for building store objects. + + This can also be looked at as a map from [store paths][store path] to derivations. + Since store paths to derivations always end in `.drv`, and store paths to other store objects never do, the two maps can also be combined into one. + Derivations can also be encoded as store objects too. + +- A [build trace], a record of which derivations have been built and what they produced. + + > **Warning** + > + > The concept of a build trace is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) + > and subject to change. + +There are [multiple types of Nix stores][store type] with different capabilities, such as the default one on the [local file system][local store] (`/nix/store`) or [binary caches][binary cache]. + +[store object]: ./store-object.md +[store path]: ./store-path.md +[derivation]: ./derivation/index.md +[build trace]: ./build-trace.md + +[store type]: ./types/index.md +[local store]: ./types/local-store.md +[binary cache]: ./types/http-binary-cache-store.md diff --git a/doc/manual/source/store/store-object.md b/doc/manual/source/store/store-object.md index 71ec772fb528..eb84ab84370d 100644 --- a/doc/manual/source/store/store-object.md +++ b/doc/manual/source/store/store-object.md @@ -42,9 +42,9 @@ The *referrers closure* of a store object are the store objects that can reach t > - The references of a store object --- the set of store paths called the references --- is a field of a store object, and thus intrinsic by definition. Regardless of what store contains the store object in question, and what else that store may or may not contain, the references are the same. > -> - The requisites of a store object are almost intrinsic --- some store paths due not precisely refer to a unique single store object. +> - The requisites of a store object are almost intrinsic --- some store paths do not precisely refer to a unique single store object. > Exactly what store object is being referenced, and what in turn *its* references are, depends on the store in question. -> Different stores that disagree. +> Different stores may disagree on what a given store path refers to. > > - The referrers of a store object are completely extrinsic, and depends solely on the store which contains that store object, not the store object itself. > Other store objects which refer to the store object in question may be added or removed from the store. @@ -66,3 +66,9 @@ A store can only contain a store object if it also contains all the store object > > The "closure property" isn't meant to prohibit, for example, [lazy loading](https://en.wikipedia.org/wiki/Lazy_loading) of store objects. > However, the "closure property" and immutability in conjunction imply that any such lazy loading ought to be deterministic. + +### Store Object Metadata {#metadata} + +[Store implementations](@docroot@/store/types/index.md) currently associate more information than described above with a store object. +Quite arguably some of this information doesn't belong here, because it conflates concerns. +For details see the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format or the [narinfo](@docroot@/protocols/binary-cache/narinfo.md) format. diff --git a/doc/manual/source/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md index 7834ac510271..282b7545a231 100644 --- a/doc/manual/source/store/store-object/content-address.md +++ b/doc/manual/source/store/store-object/content-address.md @@ -9,7 +9,7 @@ In particular, the content-addressing scheme will ensure that the digest of the - file system object graph (the root one and its children, if it has any) - references -- [store directory](../store-path.md#store-directory) +- [store directory path](../store-path.md#store-directory-path) - name of the store object, and not any other information, which would not be an intrinsic property of that store object. @@ -63,9 +63,9 @@ That provisional store path is instead what is replaced with the sentinel value, > If, conversely, the data after this normalization process is still different, we'll compute a different content-address. > The method of preparing the provisional self-referenced data has *failed* to be deterministic in the sense of not *leaking* the choice of provisional store path --- a choice which is supposed to be arbitrary --- into the final store object. > -> This property is informal because at this stage, we are just described store objects, which have no formal notion of their origin. -> Without such a formal notion, there is nothing to formally accuse of being insufficiently deterministic. -> Where we cover [derivations](@docroot@/store/derivation/index.md), we will have a chance to make this a formal property, not of content-addressed store objects themselves, but of derivations that *produce* content-addressed store objects. +> This property is informal at this stage, because we are just describing store objects, which have no formal notion of their origin. +> Without that, we cannot rigorously define what it would mean for the process to be insufficiently deterministic. +> When we cover [derivations](@docroot@/store/derivation/index.md), we will be able to make this a formal property — not of content-addressed store objects themselves, but of the derivations that *produce* them. ### Name and Store Directory diff --git a/doc/manual/source/store/store-path.md b/doc/manual/source/store/store-path.md index 04bdfec004c2..43037f7baac6 100644 --- a/doc/manual/source/store/store-path.md +++ b/doc/manual/source/store/store-path.md @@ -1,72 +1,136 @@ -# Store Path +# Store Path and Store Directory -> **Example** -> -> `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` -> -> A rendered store path +Nix's [store object] and [file system object] data models are minimal and abstract. +But to actually be used by software, store objects need to be made available through the operating system's file system. + +This is done by exposing all the store objects in a single *[store directory][store directory path]*. +Every entry in that directory is a *[store path base name]* pointing to a store object. +Store objects exposed in this way can then be referenced by *[store paths][store path]*. + +[store object]: ./store-object.md +[file system object]: ./file-system-object.md +[store path]: #store-path +[store path base name]: #base-name +[store directory path]: #store-directory-path + +## Store Path Base Name {#base-name} -Nix implements references to [store objects](./store-object.md) as *store paths*. +Nix implements references to store objects as *store path base names*. -Think of a store path as an [opaque], [unique identifier]: -The only way to obtain store path is by adding or building store objects. -A store path will always reference exactly one store object. +Think of a store path base name as an [opaque], [unique identifier]: +The only way to obtain a store path base name is by adding or building store objects. +A store path base name will always reference exactly one store object. [opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type [unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier -Store paths are pairs of +Store path base names are pairs of -- A 20-byte digest for identification -- A symbolic name for people to read +- A 20-byte [digest]{#digest} for identification +- A symbolic [name]{#name} for people to read > **Example** > > - Digest: `q06x3jll2yfzckz2bzqak089p43ixkkq` > - Name: `firefox-33.1` -To make store objects accessible to operating system processes, stores have to expose store objects through the file system. +A store path base name is rendered to a string as the concatenation of -A store path is rendered to a file system path as the concatenation of - -- [Store directory](#store-directory) (typically `/nix/store`) -- Path separator (`/`) -- Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) +- Digest rendered in [Nix32], a variant of [Base32] (20 hash bytes become 32 ASCII characters) - Hyphen (`-`) - Name > **Example** > > ``` -> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 -> |--------| |------------------------------| |----------| -> store directory digest name +> q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |------------------------------| |----------| +> digest name > ``` -Exactly how the digest is calculated depends on the type of store path. +[Nix32]: @docroot@/protocols/nix32.md +[Base32]: https://en.wikipedia.org/wiki/Base32 + +Exactly how the digest is calculated depends on the type of store object being referenced. Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). -## Store Directory - -Every [Nix store](./index.md) has a store directory. +## Store Directory Path -Not every store can be accessed through the file system. -But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). +Every [Nix store] has a store directory path. +This is an absolute, lexically canonical (not containing any `..`, `.`, or similar) path which points to the directory where all store objects are to be found. -[file system objects]: ./file-system-object.md - -This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +[Nix store]: ./index.md > **Note** > > The store directory defaults to `/nix/store`, but is in principle arbitrary. -It is important which store a given store object belongs to: +## Store Path + +A store path is the pair of a store directory path and a [store path base name]. +It is rendered to a file system path as the concatenation of + +- [Store directory path] (typically `/nix/store`) +- Path separator (`/`) +- The [store path base name] + +> **Example** +> +> ``` +> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +When we have fixed a given store, or given store directory path (that all the stores in use share), the abstract syntax for store paths and the abstract syntax for store path base names coincide: the store directory path is known from context, so only the other two fields vary from one store path to the next. + +## Exposing Store Objects in OS File Systems {#exposing} + +Not every store can be accessed through the file system. +But if the store has a file system representation, the following should be true: + +- The store directory path is canonical: no prefix of the path (i.e. path of the first *n* path segments) points to a symlink. + In other words, the store directory can be looked up from the store directory path without following any symlinks. + (This condition is a separate condition in addition to the "lexical canonicity" described above, which is a property of just the path itself. + This (regular) "canonicity" is a property about the path and the filesystem it navigates jointly.) + + > **Note** + > + > The [`allow-symlinked-store`](@docroot@/command-ref/conf-file.md#conf-allow-symlinked-store) setting can be used to relax this requirement. + +- The store directory path in fact points to a directory. + +- The store directory contains, for every store object in the store, the [file system object] of that store object at the (rendered) [store path base name]. + The permissions and other metadata for these files in the store directory is in the normal form described in [Exposing in OS file systems](./file-system-object/os-file-system.md). + +The above properties mean that the following file accesses will work. +Suppose we have a store available on the file system per the above rules, and `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` is the store path base name of a store object in that store. + +- Suppose that the store directory (path) is `/foo/bar`. + Then, `/foo/bar/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` exists and is the file system object of that store object. + +- Suppose that we don't know what the store directory path of the store is, but we do have a capability `storeDir` to the store directory on the file system. + (This would be a "file descriptor" on Unix, or a "file handle" on Windows.) + Then (using the Unix notation for this): + ``` + openat(storeDir, "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1", O_NOFOLLOW) + ``` + will succeed (so long as the file system object is not a symlink), and the yielded capability will point to the file system object of that store object. + + (The behavior for symlinks is harder to specify because of limitations in POSIX.) + +## Relocating store objects + +The inclusion of the store directory path in the full rendered store path means that the full rendered store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +(And actually, all of the currently-supported ways of computing the digest of a store path also depend on the store directory path, as described in the [specification of store path digests](@docroot@/protocols/store-path.md). +So this is also true even just for store path base names, in general.) + +It is therefore important to consider which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. Nix can only guarantee referential integrity if store paths do not cross store boundaries. -Therefore one can only copy store objects to a different store if +One can only copy store objects to a different store if - The source and target stores' directories match diff --git a/doc/manual/source/store/types/index.md.in b/doc/manual/source/store/types/index.md.in index b211ac98fe3a..fbdcf0111f59 100644 --- a/doc/manual/source/store/types/index.md.in +++ b/doc/manual/source/store/types/index.md.in @@ -26,6 +26,8 @@ tells Nix to access the store on a remote machine via the SSH protocol, using `/path/to/my/key` as the SSH private key. The supported settings for each store type are documented below. +### `auto` special URL {#auto} + The special store URL `auto` causes Nix to automatically select a store as follows: @@ -35,7 +37,7 @@ store as follows: * Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect to the Nix daemon listening on that socket](./local-daemon-store.md). -* Otherwise, on Linux only, use the [local chroot store](./local-store.md) +* Otherwise, on Linux only, use the [local chroot store](@docroot@/store/types/local-store.md#chroot) `~/.local/share/nix/root`, which will be created automatically if it does not exist. diff --git a/doc/manual/theme/head.hbs b/doc/manual/theme/head.hbs index e514a99777f7..40bfef7d2f8f 100644 --- a/doc/manual/theme/head.hbs +++ b/doc/manual/theme/head.hbs @@ -11,5 +11,5 @@ MathJax = { } }; - + diff --git a/flake.lock b/flake.lock index a437e1959056..99e0a8ec1cce 100644 --- a/flake.lock +++ b/flake.lock @@ -3,15 +3,15 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", "type": "github" }, "original": { - "owner": "edolstra", + "owner": "NixOS", "repo": "flake-compat", "type": "github" } @@ -23,12 +23,12 @@ ] }, "locked": { - "lastModified": 1748821116, - "narHash": "sha256-F82+gS044J1APL0n4hH50GYdPRv/5JWm34oCJYmVKdE=", - "rev": "49f0870db23e8c1ca0b5259734a02cd9e1e371a1", - "revCount": 377, + "lastModified": 1782949081, + "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=", + "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e", + "revCount": 480, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/hercules-ci/flake-parts/0.1.377%2Brev-49f0870db23e8c1ca0b5259734a02cd9e1e371a1/01972f28-554a-73f8-91f4-d488cc502f08/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/hercules-ci/flake-parts/0.1.480%2Brev-17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e/019f2195-dee5-7233-9747-eca0c27f7406/source.tar.gz" }, "original": { "type": "tarball", @@ -38,18 +38,17 @@ "git-hooks-nix": { "inputs": { "flake-compat": "flake-compat", - "gitignore": [], "nixpkgs": [ "nixpkgs" ] }, "locked": { - "lastModified": 1747372754, - "narHash": "sha256-2Y53NGIX2vxfie1rOW0Qb86vjRZ7ngizoo+bnXU9D9k=", - "rev": "80479b6ec16fefd9c1db3ea13aeb038c60530f46", - "revCount": 1026, + "lastModified": 1784288435, + "narHash": "sha256-ReRHaLgr/uVqdD8afFSn+myXIfpHeOhP0yYe0TJqAA8=", + "rev": "43b3c1ab9d40fb1dbb008f451988a91e375825e9", + "revCount": 1231, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/cachix/git-hooks.nix/0.1.1026%2Brev-80479b6ec16fefd9c1db3ea13aeb038c60530f46/0196d79a-1b35-7b8e-a021-c894fb62163d/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/cachix/git-hooks.nix/0.1.1231%2Brev-43b3c1ab9d40fb1dbb008f451988a91e375825e9/019f7135-8fdf-76f0-b1a1-d2c67e91af8d/source.tar.gz" }, "original": { "type": "tarball", @@ -58,16 +57,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1782767157, - "narHash": "sha256-db/8NgfehQlPNt8rMY0J1gvwzTaURU/foM7y/AQimIM=", - "rev": "1d4e0f865d68258aada31e68e6d79c8c463f3b34", - "revCount": 914302, + "lastModified": 1784160687, + "narHash": "sha256-iYL/bixrb6FlHFu/gIuBYzq6c6lM5AAXsXNSWXtIgQc=", + "rev": "4382ed2b7a6839d4280a9b386db49cbc5907414d", + "revCount": 1009383, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2511.914302%2Brev-1d4e0f865d68258aada31e68e6d79c8c463f3b34/019f1c78-b5ab-7af2-8516-c0d5406b0646/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2605.1009383%2Brev-4382ed2b7a6839d4280a9b386db49cbc5907414d/019f6c11-ad78-7500-a194-d18a5bad0fbe/source.tar.gz" }, "original": { "type": "tarball", - "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2511" + "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2605" } }, "nixpkgs-23-11": { diff --git a/flake.nix b/flake.nix index caf7eb11f4b8..d46b4683146a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2511"; + inputs.nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2605"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; @@ -12,8 +12,6 @@ # work around https://github.com/NixOS/nix/issues/7730 inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; - # work around 7730 and https://github.com/NixOS/nix/issues/7807 - inputs.git-hooks-nix.inputs.gitignore.follows = ""; outputs = inputs@{ @@ -105,11 +103,8 @@ { config = crossSystem; } - // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") { - useLLVM = true; - } // lib.optionalAttrs (crossSystem == "x86_64-w64-mingw32") { - emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable_11}/bin/wine"; + emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable}/bin/wine"; }; overlays = [ (overlayFor (pkgs: pkgs.${stdenv})) @@ -323,12 +318,6 @@ // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; } - // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { - # Some perl dependencies are broken on i686-linux. - # Since the support is only best-effort there, disable the perl - # bindings - perlBindings = self.hydraJobs.perlBindings.${system}; - } # Add "passthru" tests // flatMapAttrs @@ -451,7 +440,7 @@ supportsCross = false; }; - "nix-perl-bindings" = { + "nix-clang-tidy-plugin" = { supportsCross = false; }; } @@ -544,6 +533,16 @@ devShells = let makeShell = import ./packaging/dev-shell.nix { inherit lib devFlake; }; + makeShell' = + { pkgs }: + makeShell { + inherit pkgs; + nixComponents = pkgs.nixComponents2.overrideScope ( + finalScope: prevScope: { + withUnityBuild = false; + } + ); + }; prefixAttrs = prefix: lib.concatMapAttrs (k: v: { "${prefix}-${k}" = v; }); in forAllSystems ( @@ -551,7 +550,7 @@ prefixAttrs "native" ( forAllStdenvs ( stdenvName: - makeShell { + makeShell' { pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}; } ) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 333fe03344d1..84207a398a5d 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -47,30 +47,17 @@ fi ''}"; }; - meson-format = - let - meson = pkgs.meson.overrideAttrs { - doCheck = false; - doInstallCheck = false; - patches = [ - (pkgs.fetchpatch { - url = "https://github.com/mesonbuild/meson/commit/38d29b4dd19698d5cad7b599add2a69b243fd88a.patch"; - hash = "sha256-PgPBvGtCISKn1qQQhzBW5XfknUe91i5XGGBcaUK4yeE="; - }) - ]; - }; - in - { - enable = true; - files = "(meson.build|meson.options)$"; - entry = "${pkgs.writeScript "format-meson" '' - #!${pkgs.runtimeShell} - for file in "$@"; do - ${lib.getExe meson} format -ic ${../meson.format} "$file" - done - ''}"; - }; - nixfmt-rfc-style = { + meson-format = { + enable = true; + files = "(meson.build|meson.options)$"; + entry = "${pkgs.writeScript "format-meson" '' + #!${pkgs.runtimeShell} + for file in "$@"; do + ${lib.getExe pkgs.meson} format -ic ${../meson.format} "$file" + done + ''}"; + }; + nixfmt = { enable = true; excludes = [ # Invalid diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 068c10ee9cb6..b618bd900d4e 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -285,6 +285,10 @@ sub downloadFile { downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); }; warn "$@" if $@; + eval { + downloadFile("binaryTarballCross.x86_64-linux.x86_64-unknown-freebsd", "1"); + }; + warn "$@" if $@; downloadFile("installerScript", "1"); # Upload nix-fallback-paths.nix. @@ -296,6 +300,7 @@ sub downloadFile { " riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . " x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" . " aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" . + " x86_64-freebsd = \"" . getStorePath("buildCross.nix-everything.x86_64-unknown-freebsd.x86_64-linux") . "\";\n" . "}\n"); for my $fn (glob "$tmpDir/*") { diff --git a/meson.build b/meson.build index 3e2d29cd5495..c01297fb5477 100644 --- a/meson.build +++ b/meson.build @@ -1,15 +1,12 @@ -# This is just a stub project to include all the others as subprojects -# for development shell purposes +# This is just a top-level project to include all the others as subprojects +# for development shell purposes (when building via Nix) or for distro packaging purposes. project( - 'nix-dev-shell', + 'Nix', 'cpp', version : files('.version'), subproject_dir : 'src', - default_options : [ - 'localstatedir=/nix/var', - ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', ) # Internal Libraries @@ -21,6 +18,14 @@ subproject('libflake') subproject('libmain') subproject('libcmd') +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libfetchers-c') +subproject('libexpr-c') +subproject('libflake-c') +subproject('libmain-c') + # Executables subproject('nix') @@ -37,21 +42,6 @@ if get_option('doc-gen') endif endif -# External C wrapper libraries -subproject('libutil-c') -subproject('libstore-c') -subproject('libfetchers-c') -subproject('libexpr-c') -subproject('libflake-c') -subproject('libmain-c') - -asan_enabled = 'address' in get_option('b_sanitize') - -# Language Bindings -if get_option('bindings') and not meson.is_cross_build() and not asan_enabled - subproject('perl') -endif - # Testing if get_option('unit-tests') subproject('libutil-test-support') @@ -63,7 +53,17 @@ if get_option('unit-tests') subproject('libexpr-tests') subproject('libflake-tests') endif -subproject('nix-functional-tests') + +if get_option('functional-tests') + subproject('nix-functional-tests') +endif + if get_option('json-schema-checks') subproject('json-schema-checks') endif + +# Static Analysis +# Build the custom clang-tidy plugin if LLVM is available. +# required: false → gracefully skipped if dependency('LLVM') fails. +clang_tidy_plugin_sp = subproject('clang-tidy-plugin', required : false) +subdir('nix-meson-build-support/common/clang-tidy') diff --git a/meson.options b/meson.options index a306a84252ea..2e0d873fae08 100644 --- a/meson.options +++ b/meson.options @@ -15,10 +15,10 @@ option( ) option( - 'bindings', + 'functional-tests', type : 'boolean', value : true, - description : 'Build language bindings (e.g. Perl)', + description : 'Build functional (E2E) tests', ) option( diff --git a/nix-meson-build-support/common/asan-options/asan-options.cc b/nix-meson-build-support/common/asan-options/asan-options.cc index c9782fea03b5..62db7ddd20e0 100644 --- a/nix-meson-build-support/common/asan-options/asan-options.cc +++ b/nix-meson-build-support/common/asan-options/asan-options.cc @@ -1,4 +1,17 @@ -extern "C" [[gnu::retain, gnu::weak]] const char * __asan_default_options() +#if defined(__has_attribute) +# if __has_attribute(no_profile_instrument_function) +# define NIX_NO_PROFILE_INSTRUMENT_FUNCTION __attribute__((no_profile_instrument_function)) +# endif +#endif + +#ifndef NIX_NO_PROFILE_INSTRUMENT_FUNCTION +# define NIX_NO_PROFILE_INSTRUMENT_FUNCTION +#endif + +// This ASan hook is linked into many instrumented binaries and libraries. Do +// not emit coverage counters for it because repeated weak definitions of the +// hook can produce profile data that llvm-profdata treats as corrupted. +extern "C" [[gnu::retain, gnu::weak]] NIX_NO_PROFILE_INSTRUMENT_FUNCTION const char * __asan_default_options() { // We leak a bunch of memory knowingly on purpose. It's not worthwhile to // diagnose that memory being leaked for now. diff --git a/nix-meson-build-support/common/asan-options/meson.build b/nix-meson-build-support/common/asan-options/meson.build index 56e6a6a56a7f..80527b5a9884 100644 --- a/nix-meson-build-support/common/asan-options/meson.build +++ b/nix-meson-build-support/common/asan-options/meson.build @@ -1,7 +1,7 @@ # Clang gets grumpy about missing libasan symbols if -shared-libasan is not # passed when building shared libs, at least on Linux if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option( - 'b_sanitize', + 'b_sanitize', )) add_project_link_arguments('-shared-libasan', language : 'cpp') endif diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy new file mode 100644 index 000000000000..b0cc92429c4e --- /dev/null +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -0,0 +1,100 @@ +UseColor: true +Checks: + - -* + - bugprone-* + # Too many warnings + - -bugprone-assignment-in-if-condition + # Too many warnings + - -bugprone-narrowing-conversions + # Kind of nonsense + - -bugprone-easily-swappable-parameters + # Too many warnings for now + - -bugprone-implicit-widening-of-multiplication-result + # Exception handling patterns in Nix + - -bugprone-empty-catch + # Many warnings + - -bugprone-unchecked-optional-access + # Many warnings, questionable lint + - -bugprone-branch-clone + # Extremely noisy before clang 19: https://github.com/llvm/llvm-project/issues/93959 + - -bugprone-multi-level-implicit-pointer-conversion + # We don't compile out our asserts + - -bugprone-assert-side-effect + # TODO: figure out if this warning is useful + - -bugprone-exception-escape + # We use pointers to aggregates intentionally; void * would look weird + - -bugprone-sizeof-expression + # + # Checks disabled to pass on current codebase (can be progressively enabled): + # + # 11 warnings - optional value conversions in various places + - -bugprone-optional-value-conversion + # 4 warnings - switches without default cases + - -bugprone-switch-missing-default-case + # 4 warnings - string_view::data() usage patterns + - -bugprone-suspicious-stringview-data-usage + # 4 warnings - .cc files included in other files (intentional pattern) + - -bugprone-suspicious-include + # 2 warnings - unused return values (AllowCastToVoid helps but some remain) + - -bugprone-unused-return-value + # 2 warnings - unused local RAII-style variables + - -bugprone-unused-local-non-trivial-variable + # 2 warnings - returning const& from parameter + - -bugprone-return-const-ref-from-parameter + # 1 warning - signed char misuse + - -bugprone-signed-char-misuse + # 1 warning - calling parent virtual instead of override + - -bugprone-parent-virtual-call + # 1 warning - null termination issues + - -bugprone-not-null-terminated-result + # 1 warning - macro parentheses + - -bugprone-macro-parentheses + # 1 warning - increment/decrement in conditions + - -bugprone-inc-dec-in-conditions + # 2 warnings - sorts Value* by ->string_view(), not by pointer value (false positive) + - -bugprone-nondeterministic-pointer-iteration-order + # 9 warnings - intentional std::bit_cast/memcpy on Value* arrays (evaluator hot path) + - -bugprone-bitwise-pointer-cast + # 1 warning (header) - value.hh mkFailed: GC alloc in noexcept. Boehm's + # gc_cleanup::operator new isn't marked noexcept but aborts on OOM rather + # than throws; std::terminate here is the intended behavior anyway. + - -bugprone-unhandled-exception-at-new + # 1 warning (header) - fmt.hh Magenta::operator<<: generic colorizer + # template; fires when T=unsigned char but that instantiation is correct. + - -bugprone-unintended-char-ostream-output + # + # Non-bugprone checks (some disabled to pass on current codebase): + # + # 4 warnings - exceptions not derived from std::exception + # All thrown exceptions must derive from std::exception + # - hicpp-exception-baseclass + # 88 warnings - C-style casts should be explicit about intent + # - cppcoreguidelines-pro-type-cstyle-cast + # 11 warnings - coroutine lambdas with captures (intentional pattern in async goal/store code) + # - cppcoreguidelines-avoid-capturing-lambda-coroutines + - performance-noexcept-swap + - performance-noexcept-move-constructor + - performance-noexcept-destructor + - performance-use-std-move + - misc-throw-by-value-catch-by-reference + - cppcoreguidelines-missing-std-forward + - android-cloexec-open + - android-cloexec-pipe2 + # Custom nix checks (when added) + - nix-* + +CheckOptions: + # __asan_default_options: ASAN runtime configuration function (see nix-meson-build-support/common/asan-options/) + # __wrap___assert_fail: Linker-wrapped assert handler for better stack traces (see nix-meson-build-support/common/assert-fail/) + # _SingleDerivedPathRaw, _DerivedPathRaw: Internal type aliases in derived-path.hh (leading underscore pattern) + # _SingleBuiltPathRaw, _BuiltPathRaw: Internal type aliases in built-path.hh (leading underscore pattern) + bugprone-reserved-identifier.AllowedIdentifiers: '__asan_default_options;__wrap___assert_fail;_SingleDerivedPathRaw;_DerivedPathRaw;_SingleBuiltPathRaw;_BuiltPathRaw' + # Allow explicitly discarding return values with (void) cast + bugprone-unused-return-value.AllowCastToVoid: true + bugprone-unsafe-functions.ReportDefaultFunctions: false + # Repurpose bugprone-unsafe-functions to lint functions that we'd want to wrap. + bugprone-unsafe-functions.CustomFunctions: > + ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; + ::std::filesystem::remove_all, nix::deletePath, "Use nix::deletePath (remove_all is not TOCTOU safe)"; + +ExtraArgs: ["-Werror=unnecessary-virtual-specifier"] diff --git a/nix-meson-build-support/common/clang-tidy/build_required_targets.py b/nix-meson-build-support/common/clang-tidy/build_required_targets.py new file mode 100755 index 000000000000..24e4f290c607 --- /dev/null +++ b/nix-meson-build-support/common/clang-tidy/build_required_targets.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +Builds all generated files before running clang-tidy. + +clang-tidy needs all source files and headers to exist before it can +analyze the code. This script queries Ninja for all custom command +targets that generate files needed for analysis and builds them. + +Generated files include: +- .gen.hh: Embedded file headers (SQL schemas, Nix expressions, etc.) +- .gen.inc: Generated include files +- lexer-tab.cc, parser-tab.cc: Flex/Bison generated parsers +- Perl XS bindings, and other generated sources + +See: https://github.com/mesonbuild/meson/issues/12817 +""" + +import subprocess +from pathlib import Path + + +def get_targets_of_rule(build_root: str, rule_name: str) -> list[str]: + return ( + subprocess.check_output( + ["ninja", "-C", build_root, "-t", "targets", "rule", rule_name] + ) + .decode() + .strip() + .splitlines() + ) + + +def ninja_build(build_root: str, targets: list[str]): + if targets: + subprocess.check_call(["ninja", "-C", build_root, "--", *targets]) + + +def main(): + import argparse + + ap = argparse.ArgumentParser(description="Build required targets for clang-tidy") + ap.add_argument("build_root", help="Ninja build root", type=str) + ap.add_argument("--stamp", type=Path, default=None, help="Stamp file to touch on success") + + args = ap.parse_args() + + custom_commands = get_targets_of_rule(args.build_root, "CUSTOM_COMMAND") + + targets = ( + # Generated headers from embedded files + [t for t in custom_commands if t.endswith(".gen.hh")] + # Generated include files + + [t for t in custom_commands if t.endswith(".gen.inc")] + # Flex/Bison generated parsers + + [t for t in custom_commands if t.endswith("-tab.cc")] + ) + ninja_build(args.build_root, targets) + + if args.stamp: + args.stamp.touch() + + +if __name__ == "__main__": + main() diff --git a/nix-meson-build-support/common/clang-tidy/clang-tidy-runner.py b/nix-meson-build-support/common/clang-tidy/clang-tidy-runner.py new file mode 100755 index 000000000000..8d7b99c7d438 --- /dev/null +++ b/nix-meson-build-support/common/clang-tidy/clang-tidy-runner.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +""" +Wrapper around run-clang-tidy with better UX. + +This script handles: +- Loading the custom nix-clang-tidy plugin +- Setting up header filters to only check project headers +- Managing parallelism based on available CPUs +- Working around run-clang-tidy's unpatched shebang (NixOS/nixpkgs#214945) +""" + +import multiprocessing +import os +import sys +from pathlib import Path + + +def default_concurrency(): + return min( + multiprocessing.cpu_count(), int(os.environ.get("NIX_BUILD_CORES", "16")) + ) + + +def go( + exe: str, + plugin_path: Path | None, + config_file: Path | None, + compile_commands_json_dir: Path, + jobs: int, + paths: list[Path], + werror: bool, + fix: bool, +): + args = [ + # clang-unwrapped ships run-clang-tidy with an unpatched '#!/usr/bin/env python3' + # shebang (see NixOS/nixpkgs#214945), which fails in sandboxed builds where + # /usr/bin/env doesn't exist. Invoke through sys.executable to bypass the shebang. + sys.executable, + exe, + "-quiet", + ] + + if plugin_path is not None: + args += ["-load", str(plugin_path)] + + if config_file is not None: + args += ["-config-file", str(config_file)] + + args += [ + "-p", + str(compile_commands_json_dir), + "-j", + str(jobs), + # Lint all project headers (public include/nix/*/ and private *.hh). + # Project headers are reached via relative -I flags; system/installed + # headers are absolute /nix/store/... paths. Match any non-absolute path + # so the filter works in both monolithic (../src/libfoo/include/...) + # and per-component (../include/...) build topologies. + "-header-filter", + r"^[^/].*\.hh", + # Exclude generated Bison/Flex headers (parser-tab.hh, lexer-tab.hh) + "-exclude-header-filter", + r".-tab\.hh$", + ] + + if werror: + args += ["-warnings-as-errors", "*"] + if fix: + args += ["-fix"] + + args += ["--"] + args += [str(p) for p in paths] + + os.execvp(sys.executable, args) + + +def main(): + import argparse + + ap = argparse.ArgumentParser(description="Run clang-tidy on the Nix codebase") + ap.add_argument( + "--jobs", + "-j", + type=int, + default=default_concurrency(), + help="Parallel linting jobs to run", + ) + ap.add_argument( + "--plugin-path", + type=Path, + default=None, + help="Path to the nix-clang-tidy plugin", + ) + ap.add_argument( + "--config-file", + type=Path, + default=None, + help="Path to the .clang-tidy config file", + ) + ap.add_argument( + "--compdb-path", + type=Path, + required=True, + help="Path to the directory containing the cleaned compilation database", + ) + ap.add_argument( + "--werror", + action="store_true", + help="Treat warnings as errors", + ) + ap.add_argument( + "--fix", + action="store_true", + help="Apply fixes for warnings", + ) + ap.add_argument( + "--run-clang-tidy-path", + default="run-clang-tidy", + help="Path to run-clang-tidy", + ) + ap.add_argument( + "paths", + nargs="*", + help="Source paths to check", + ) + args = ap.parse_args() + + go( + args.run_clang_tidy_path, + args.plugin_path, + args.config_file, + args.compdb_path, + args.jobs, + args.paths, + args.werror, + args.fix, + ) + + +if __name__ == "__main__": + main() diff --git a/nix-meson-build-support/common/clang-tidy/clean_compdb.py b/nix-meson-build-support/common/clang-tidy/clean_compdb.py new file mode 100755 index 000000000000..8087b0bf9b6e --- /dev/null +++ b/nix-meson-build-support/common/clang-tidy/clean_compdb.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +""" +Strips PCH (precompiled header) arguments from a compilation database. + +This is needed because clang-tidy cannot process PCH flags when using +nixpkgs' cc-wrapper. The cc-wrapper makes assumptions that don't hold +when the compiler is being used indirectly by clang-tidy. + +See: https://github.com/mesonbuild/meson/issues/13499 +""" + +import json +import shlex + + +def process_compdb(compdb: list[dict]) -> list[dict]: + def munch_command(args: list[str]) -> list[str]: + out = [] + eat_next = False + for i, arg in enumerate(args): + if arg in ["-fpch-preprocess", "-fpch-instantiate-templates"]: + # -fpch-preprocess as used with gcc + # -fpch-instantiate-templates as used by clang + continue + elif arg == "-include-pch" or ( + arg == "-include" + and i + 1 < len(args) + and args[i + 1] == "precompiled-headers.hh" + ): + # -include-pch some-pch (clang), or -include some-pch (gcc) + eat_next = True + continue + if not eat_next: + out.append(arg) + eat_next = False + return out + + def chomp(item: dict) -> dict: + item = item.copy() + item["command"] = shlex.join(munch_command(shlex.split(item["command"]))) + return item + + def cmdfilter(item: dict) -> bool: + file = item["file"] + # Filter out precompiled header files + if file.endswith("precompiled-headers.hh"): + return False + # Filter out Flex/Bison generated parsers (generated code) + if file.endswith("-tab.cc"): + return False + return True + + return [chomp(x) for x in compdb if cmdfilter(x)] + + +def main(): + import argparse + + ap = argparse.ArgumentParser( + description="Strip PCH arguments from compilation database" + ) + ap.add_argument("input", type=argparse.FileType("r"), help="Input json file") + ap.add_argument("output", type=argparse.FileType("w"), help="Output json file") + args = ap.parse_args() + + input_json = json.load(args.input) + json.dump(process_compdb(input_json), args.output, indent=2) + + +if __name__ == "__main__": + main() diff --git a/nix-meson-build-support/common/clang-tidy/meson.build b/nix-meson-build-support/common/clang-tidy/meson.build new file mode 100644 index 000000000000..9247c30b885a --- /dev/null +++ b/nix-meson-build-support/common/clang-tidy/meson.build @@ -0,0 +1,86 @@ +# clang-tidy integration for the Nix project +# +# This provides the 'clang-tidy' run_target for per-component builds. +# The custom plugin (nix-clang-tidy) is built as a separate component +# and loaded automatically when available. + +python = find_program('python3') +run_clang_tidy = find_program('run-clang-tidy', required : false) + +# Resolves via override_dependency (monolithic) or pkg-config (per-component Nix) +nix_clang_tidy_plugin = dependency('nix-clang-tidy', required : false) + +# Clean the compilation database by stripping PCH arguments. +# This is needed because clang-tidy cannot process PCH flags from cc-wrapper. +# See: https://github.com/mesonbuild/meson/issues/13499 +meson.add_postconf_script( + python, + meson.current_source_dir() / 'clean_compdb.py', + meson.global_build_root() / 'compile_commands.json', + meson.current_build_dir() / 'compile_commands.json', +) + +if run_clang_tidy.found() + # Pre-build generated headers/sources (.gen.hh, -tab.cc, etc.) so clang-tidy + # can run on a fresh build dir without a full `ninja` first. + # See: https://github.com/mesonbuild/meson/issues/12817 + build_gen_targets = custom_target( + 'build-gen-for-tidy', + command : [ + python, + files('build_required_targets.py'), + meson.global_build_root(), + '--stamp', '@OUTPUT@', + ], + output : 'gen-targets.stamp', + build_always_stale : true, + build_by_default : false, + ) + + # In monolithic builds the plugin .so is built from source — ensure it's + # built before we try to -load it. In per-component builds the plugin is + # already installed (buildInputs), so no extra dep is needed. + clang_tidy_deps = [ build_gen_targets ] + if is_variable('clang_tidy_plugin_sp') and clang_tidy_plugin_sp.found() + clang_tidy_deps += [ + clang_tidy_plugin_sp.get_variable('nix_clang_tidy_plugin'), + ] + endif + + run_clang_tidy_args = [ + meson.current_source_dir() / 'clang-tidy-runner.py', + '--run-clang-tidy-path', + run_clang_tidy, + '--compdb-path', + meson.current_build_dir(), + '--config-file', + meson.current_source_dir() / '.clang-tidy', + ] + + if nix_clang_tidy_plugin.found() + run_clang_tidy_args += [ + '--plugin-path', + nix_clang_tidy_plugin.get_variable('plugin_path'), + ] + endif + + run_target( + 'clang-tidy', + command : [ + python, + run_clang_tidy_args, + '--werror', + ], + depends : clang_tidy_deps, + ) + + run_target( + 'clang-tidy-fix', + command : [ + python, + run_clang_tidy_args, + '--fix', + ], + depends : clang_tidy_deps, + ) +endif diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index e13c7df72e48..f7cca818057e 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -14,7 +14,7 @@ if host_machine.system() == 'cygwin' ) endif -add_project_arguments( +warning_flags = [ '-Wdeprecated-copy', '-Werror=suggest-override', '-Werror=switch', @@ -27,19 +27,40 @@ add_project_arguments( '-Wignored-qualifiers', '-Wimplicit-fallthrough', '-Wno-deprecated-declarations', + '-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings. + '-Wno-subobject-linkage', # GCC doesn't like unity builds. + # Catch brace elision bugs: when WorkerProto::Version changed from `unsigned int` + # to `struct { unsigned int major; uint8_t minor; }`, `.version = 16` silently + # became `.version = {16, 0}` instead of failing, breaking protocol compatibility + # in a subtle way + '-Werror=c99-designator', +] + +if meson.project_name() in [ + 'nix-util', + 'nix-store', + 'nix-util-c', + 'nix-store-c', + 'nix-expr', + 'nix-expr-c', +] + # Catch vtables with vague linkage. All vtables must have a "key" function to make + # sure they are emitted as strong symbols (as required by Itanium ABI). This is needed + # in certain cases of dynamic linking, where weak symbols are not always coalesced. Otherwise + # typeinfo symbols get duplicated and dynamic_cast/exception handling ends up broken on Darwin's + # libc++. + # Search for 'virtual void anchor' in the repo to see how this is done. + warning_flags += [ '-Werror=weak-vtables' ] +endif + +add_project_arguments( + cxx.get_supported_arguments(warning_flags), language : 'cpp', ) # GCC doesn't benefit much from precompiled headers. do_pch = cxx.get_id() == 'clang' -if cxx.get_id() == 'gcc' - add_project_arguments( - '-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings. - language : 'cpp', - ) -endif - # This is a clang-only option for improving build times. # It forces the instantiation of templates in the PCH itself and # not every translation unit it's included in. @@ -49,11 +70,6 @@ endif # instantiations in libutil and libstore. if cxx.get_id() == 'clang' add_project_arguments('-fpch-instantiate-templates', language : 'cpp') - # Catch brace elision bugs: when WorkerProto::Version changed from `unsigned int` - # to `struct { unsigned int major; uint8_t minor; }`, `.version = 16` silently - # became `.version = {16, 0}` instead of failing, breaking protocol compatibility - # in a subtle way - add_project_arguments('-Werror=c99-designator', language : 'cpp') endif # Detect if we're using libstdc++ (GCC's standard library) @@ -88,3 +104,6 @@ endif subdir('assert-fail') subdir('asan-options') subdir('cxa-throw') +if not meson.is_subproject() + subdir('clang-tidy') +endif diff --git a/nix-meson-build-support/export/meson.build b/nix-meson-build-support/export/meson.build index 62a27bd48c37..d5cb39661d06 100644 --- a/nix-meson-build-support/export/meson.build +++ b/nix-meson-build-support/export/meson.build @@ -11,12 +11,34 @@ endforeach requires_public += deps_public extra_pkg_config_variables = get_variable('extra_pkg_config_variables', {}) +fs = import('fs') +plugin_c_api_enabled = get_variable('plugin_c_api_enabled', false) + +if plugin_c_api_enabled + this_libraries = get_variable('this_libraries') + this_library = this_libraries.get_shared_lib() + this_static_library = this_libraries.get_static_lib() +else + this_library = get_variable('this_library') +endif extra_cflags = [] if not meson.project_name().endswith('-c') extra_cflags += [ '-std=c++23' ] endif +whole_archive_dep_name = meson.project_name() + '-whole-archive' +installed_whole_archive_link_arg = '' +if plugin_c_api_enabled and host_machine.system() == 'darwin' + installed_whole_archive_link_arg = '-Wl,-force_load,${libdir}/' + fs.name( + this_static_library.full_path(), + ) +elif plugin_c_api_enabled and not host_machine.system().startswith('windows') + installed_whole_archive_link_arg = '-Wl,--whole-archive,${libdir}/' + fs.name( + this_static_library.full_path(), + ) + ',--no-whole-archive' +endif + import('pkgconfig').generate( this_library, filebase : meson.project_name(), @@ -29,6 +51,19 @@ import('pkgconfig').generate( variables : extra_pkg_config_variables, ) +if installed_whole_archive_link_arg != '' + import('pkgconfig').generate( + filebase : whole_archive_dep_name, + name : 'Nix whole-archive static dependency', + description : 'Force-load static Nix C API archive into the host executable', + extra_cflags : extra_cflags, + requires : requires_public, + requires_private : requires_private, + libraries : [ installed_whole_archive_link_arg ], + variables : extra_pkg_config_variables, + ) +endif + meson.override_dependency( meson.project_name(), declare_dependency( @@ -39,3 +74,16 @@ meson.override_dependency( variables : extra_pkg_config_variables, ), ) + +if installed_whole_archive_link_arg != '' + meson.override_dependency( + whole_archive_dep_name, + declare_dependency( + include_directories : include_dirs, + link_whole : [ this_static_library ], + compile_args : [ '-std=c++23' ], + dependencies : deps_public_subproject + deps_public, + variables : extra_pkg_config_variables, + ), + ) +endif diff --git a/packaging/components.nix b/packaging/components.nix index 962f0ba05b2e..50629ef1bd73 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -45,10 +45,13 @@ let filesetToSource = lib.fileset.toSource; /** - Given a set of layers, create a mkDerivation-like function + Given a stdenv and set of layers, create a mkDerivation-like function */ - mkPackageBuilder = - exts: userFn: stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn); + mkPackageBuilderFor = + stdenv': exts: userFn: + stdenv'.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn); + + mkPackageBuilder = mkPackageBuilderFor stdenv; setVersionLayer = finalAttrs: prevAttrs: { }; @@ -127,8 +130,6 @@ let !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isCygwin) # build failure && !stdenv.hostPlatform.isStatic - # LTO breaks exception handling on x86-64-darwin. - && stdenv.system != "x86_64-darwin" ) '' case "$mesonBuildType" in @@ -141,6 +142,13 @@ let ninja ] ++ prevAttrs.nativeBuildInputs or [ ]; + mesonFlags = + prevAttrs.mesonFlags or [ ] + ++ (lib.optionals scope.withUnityBuild [ + "-Dunity=on" + "-Dunity_size=8192" + "-Db_pch=false" + ]); mesonCheckFlags = prevAttrs.mesonCheckFlags or [ ] ++ [ "--print-errorlogs" ]; @@ -200,8 +208,13 @@ let enableSanitizersLayer = finalAttrs: prevAttrs: let - sanitizers = lib.optional scope.withASan "address" ++ lib.optional scope.withUBSan "undefined"; + sanitizers = + lib.optional scope.withASan "address" + ++ lib.optional scope.withUBSan "undefined" + ++ lib.optional scope.withTSan "thread"; in + # Thread sanitizer can't be used with ASan or UBSan + assert scope.withTSan -> !(scope.withASan || scope.withUBSan); { mesonFlags = (prevAttrs.mesonFlags or [ ]) @@ -216,6 +229,30 @@ let ); }; + enableClangTidyLayer = + finalAttrs: prevAttrs: + lib.optionalAttrs scope.withClangTidy { + nativeBuildInputs = (prevAttrs.nativeBuildInputs or [ ]) ++ [ + pkgs.buildPackages.llvmPackages.clang-tools # provides run-clang-tidy + ]; + + buildInputs = (prevAttrs.buildInputs or [ ]) ++ [ + scope.nix-clang-tidy-plugin # provides nix-clang-tidy.pc with plugin_path + ]; + + # Use debug build for faster compilation (no optimizations) + mesonBuildType = "debug"; + + # Skip tests - we only care about clang-tidy results + doCheck = false; + + # Run clang-tidy after the normal build + postBuild = (prevAttrs.postBuild or "") + '' + echo "Running clang-tidy on ${finalAttrs.pname}..." + ninja clang-tidy + ''; + }; + nixDefaultsLayer = finalAttrs: prevAttrs: { strictDeps = prevAttrs.strictDeps or true; enableParallelBuilding = true; @@ -268,6 +305,26 @@ in */ withUBSan = false; + /** + Whether meson components are built with [ThreadSanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html). + */ + withTSan = false; + + /** + Whether meson components are checked with [clang-tidy](https://clang.llvm.org/extra/clang-tidy/). + */ + withClangTidy = false; + + /** + Whether to use [unity builds](https://mesonbuild.com/Unity-builds.html#unity-builds). + */ + withUnityBuild = true; + + /** + Whether to embed the public C API into nix-cli so plugins can resolve those symbols from the executable. + */ + withPluginCAPI = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); + /** A user-provided extension function to apply to each component derivation. */ @@ -366,6 +423,7 @@ in mesonBuildLayer fixupStaticLayer enableSanitizersLayer + enableClangTidyLayer scope.mesonComponentOverrides ]; mkMesonLibrary = mkPackageBuilder [ @@ -378,6 +436,7 @@ in mesonLibraryLayer fixupStaticLayer enableSanitizersLayer + enableClangTidyLayer scope.mesonComponentOverrides ]; @@ -414,7 +473,9 @@ in /** The Nix command line interface. Note that this does not include its tests, whereas `nix-everything` does. */ - nix-cli = callPackage ../src/nix/package.nix { version = fineVersion; }; + nix-cli = callPackage ../src/nix/package.nix { + version = fineVersion; + }; nix-functional-tests = callPackage ../tests/functional/package.nix { version = fineVersion; @@ -447,7 +508,21 @@ in */ nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { }; - nix-perl-bindings = callPackage ../src/perl/package.nix { }; + # The clang-tidy plugin is a build-time tool loaded into clang-tidy itself, + # so it must be built with a clang stdenv for ABI compatibility with the + # clang-tidy binary from the same llvmPackages set, regardless of the + # scope's stdenv (which may be GCC). + nix-clang-tidy-plugin = callPackage ../src/clang-tidy-plugin/package.nix { + llvmPackages = pkgs.buildPackages.llvmPackages; + mkMesonDerivation = mkPackageBuilderFor pkgs.buildPackages.llvmPackages.stdenv [ + nixDefaultsLayer + scope.sourceLayer + setVersionLayer + mesonLayer + fixupStaticLayer + scope.mesonComponentOverrides + ]; + }; /** Combined package that has the CLI, libraries, and (assuming non-cross, no overrides) it requires that all tests succeed. diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 71731b540176..7f6a94346dec 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -16,6 +16,20 @@ in scope: { inherit stdenv; + mimalloc = + if lib.versionAtLeast pkgs.mimalloc.version "3.3.2" then + pkgs.mimalloc + else + pkgs.mimalloc.overrideAttrs rec { + version = "3.3.2"; + src = pkgs.fetchFromGitHub { + owner = "microsoft"; + repo = "mimalloc"; + tag = "v${version}"; + hash = "sha256-GZ37qQVDe9jgMb4Coe5oKvgaLTspZDlSkS5rdy1MfUU="; + }; + }; + boehmgc = (pkgs.boehmgc.override { enableLargeConfig = true; @@ -86,28 +100,28 @@ scope: { (prevAttrs.postInstall or ""); }); - curl = - (pkgs.curl.override { - http3Support = !pkgs.stdenv.hostPlatform.isWindows; - # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. - zstdSupport = true; - brotliSupport = true; - zlibSupport = true; - # libpsl uses a data file needed at runtime, not useful for nix. - pslSupport = !stdenv.hostPlatform.isStatic; - idnSupport = !stdenv.hostPlatform.isStatic; - }).overrideAttrs - { - # TODO: Fix in nixpkgs. Static build with brotli is marked as broken, but it's not the case. - # Remove once https://github.com/NixOS/nixpkgs/pull/494111 lands in the 25.11 channel. - meta.broken = false; - }; + curl = pkgs.curl.override { + http3Support = !pkgs.stdenv.hostPlatform.isWindows; + # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. + zstdSupport = true; + brotliSupport = true; + zlibSupport = true; + # libpsl uses a data file needed at runtime, not useful for nix. + pslSupport = !stdenv.hostPlatform.isStatic; + idnSupport = !stdenv.hostPlatform.isStatic; + }; libblake3 = (pkgs.libblake3.override { inherit stdenv; # Nixpkgs disables tbb on static - useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); + useTBB = + !( + stdenv.hostPlatform.isWindows + || stdenv.hostPlatform.isStatic + # Some tbb tests fail with libc++. + || (stdenv.cc.libcxx != null && stdenv.cc.libcxx.isLLVM) + ); }) # For some reason that is not clear, it is wanting to use libgcc_eh which is not available. # Force this to be built with compiler-rt & libunwind over libgcc_eh works. @@ -134,6 +148,45 @@ scope: { } ); + sqlite = + if !stdenv.hostPlatform.isWindows then + pkgs.sqlite + else + pkgs.sqlite.overrideAttrs (prevAttrs: { + nativeBuildInputs = lib.filter (x: !(x.pname == "tcl")) prevAttrs.nativeBuildInputs or [ ]; + configureFlags = (lib.filter (x: !(lib.hasPrefix "--with-tcl" x)) prevAttrs.configureFlags) ++ [ + "--disable-tcl" + ]; + }); + + libgit2 = + ( + if lib.versionAtLeast pkgs.libgit2.version "1.9.4" then + pkgs.libgit2 + else + # Grab newer libgit2. + pkgs.libgit2.overrideAttrs rec { + version = "1.9.4"; + src = pkgs.fetchFromGitHub { + owner = "libgit2"; + repo = "libgit2"; + tag = "v${version}"; + hash = "sha256-ZKUiz3pdFE2SKxh53X2oyr7hs32Njj5YVA0OXDXz7h0="; + }; + } + ).overrideAttrs + (old: { + separateDebugInfo = true; + + patches = old.patches or [ ] ++ [ + # Fix a use-after-free crash when `git_thread_create` fails during + # pack building (e.g. with EAGAIN under thread pressure), leaving + # orphaned delta-search worker threads running while the + # packbuilder is freed. + ./patches/libgit2-packbuilder-dont-fail-on-thread-create-error.patch + ]; + }); + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. boost = (pkgs.boost.override { @@ -168,16 +221,4 @@ scope: { # Required for configuration detection for getsockname (for automatic port allocation for `nix serve`) __darwinAllowLocalNetworking = true; }); - - libgit2 = pkgs.libgit2.overrideAttrs (old: { - separateDebugInfo = true; - - patches = old.patches or [ ] ++ [ - # Fix a use-after-free crash when `git_thread_create` fails during - # pack building (e.g. with EAGAIN under thread pressure), leaving - # orphaned delta-search worker threads running while the - # packbuilder is freed. - ./patches/libgit2-packbuilder-dont-fail-on-thread-create-error.patch - ]; - }); } diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 479229b06d0f..ad634d475d19 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -110,246 +110,242 @@ let in -{ pkgs }: - -# TODO: don't use nix-util for this? -pkgs.nixComponents2.nix-util.overrideAttrs ( - finalAttrs: prevAttrs: - - let - stdenv = pkgs.nixDependencies2.stdenv; - buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; - modular = devFlake.getSystem stdenv.buildPlatform.system; - transformFlag = - prefix: flag: - assert builtins.isString flag; - let - rest = builtins.substring 2 (builtins.stringLength flag) flag; - in - "-D${prefix}:${rest}"; - havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; - ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; - - availableComponents = lib.filterAttrs ( - k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v - ) allComponents; - - activeComponents = buildInputsClosureCond isInternal ( - lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents) - ); +{ pkgs, nixComponents }: - allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2; - internalDrvs = byDrvPath ( - # Drop the attr names (not present in buildInputs anyway) - lib.attrValues availableComponents - ++ lib.concatMap (c: lib.filter (v: !v.meta.broken) (lib.attrValues (c.tests or { }))) ( - lib.attrValues availableComponents - ) - ); +nixComponents.callPackage ( + { stdenv }: + (stdenv.mkDerivation ( + finalAttrs: - isInternal = - dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"}; + let + buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = + prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + + availableComponents = lib.filterAttrs ( + k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v + ) allComponents; + + activeComponents = buildInputsClosureCond isInternal ( + lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents) + ); - activeComponentNames = lib.listToAttrs ( - map (c: { - name = c.pname or c.name; - value = null; - }) activeComponents - ); + allComponents = lib.filterAttrs (k: v: lib.isDerivation v) nixComponents; + internalDrvs = byDrvPath ( + # Drop the attr names (not present in buildInputs anyway) + lib.attrValues availableComponents + ++ lib.concatMap (c: lib.filter (v: !v.meta.broken) (lib.attrValues (c.tests or { }))) ( + lib.attrValues availableComponents + ) + ); - isActiveComponent = name: activeComponentNames ? ${name}; + isInternal = + dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"}; - in - { - pname = "shell-for-nix"; + activeComponentNames = lib.listToAttrs ( + map (c: { + name = c.pname or c.name; + value = null; + }) activeComponents + ); - passthru = { - inherit activeComponents; + isActiveComponent = name: activeComponentNames ? ${name}; - # We use this attribute to store non-derivation values like functions and - # perhaps other things that are primarily for overriding and not the shell. - config = { - # Default getComponents - getComponents = - c: - builtins.removeAttrs c ( - lib.optionals (!havePerl) [ "nix-perl-bindings" ] - ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ] + in + { + pname = "shell-for-nix"; + + passthru = { + inherit activeComponents; + + # We use this attribute to store non-derivation values like functions and + # perhaps other things that are primarily for overriding and not the shell. + config = { + # Default getComponents + getComponents = c: builtins.removeAttrs c (lib.optionals (!buildCanExecuteHost) [ "nix-manual" ]); + }; + + /** + Produce a devShell for a given set of nix components + + Example: + + ```nix + shell.withActiveComponents (c: { + inherit (c) nix-util; + }) + ``` + */ + withActiveComponents = + f2: + finalAttrs.finalPackage.overrideAttrs ( + finalAttrs: prevAttrs: { + passthru = prevAttrs.passthru // { + config = prevAttrs.passthru.config // { + getComponents = f2; + }; + }; + } ); - }; - /** - Produce a devShell for a given set of nix components - - Example: - - ```nix - shell.withActiveComponents (c: { - inherit (c) nix-util; - }) - ``` - */ - withActiveComponents = - f2: - finalAttrs.finalPackage.overrideAttrs ( - finalAttrs: prevAttrs: { - passthru = prevAttrs.passthru // { - config = prevAttrs.passthru.config // { - getComponents = f2; - }; - }; - } + small = finalAttrs.finalPackage.withActiveComponents ( + c: + lib.intersectAttrs (lib.genAttrs [ + "nix-cli" + "nix-util-tests" + "nix-store-tests" + "nix-expr-tests" + "nix-fetchers-tests" + "nix-flake-tests" + "nix-functional-tests" + ] (_: null)) c ); + }; - small = finalAttrs.finalPackage.withActiveComponents ( - c: - lib.intersectAttrs (lib.genAttrs [ - "nix-cli" - "nix-util-tests" - "nix-store-tests" - "nix-expr-tests" - "nix-fetchers-tests" - "nix-flake-tests" - "nix-functional-tests" - "nix-perl-bindings" - ] (_: null)) c - ); - }; - - # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop - version = lib.fileContents ../.version-determinate; - name = finalAttrs.pname; - - installFlags = "sysconfdir=$(out)/etc"; - shellHook = '' - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH - - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share - - # Make the default phases do the right thing. - # FIXME: this wouldn't be needed if the ninja package set buildPhase() instead of $buildPhase. - # FIXME: mesonConfigurePhase shouldn't cd to the build directory. It would be better to pass '-C ' to ninja. - - cdToBuildDir() { - if [[ ! -e build.ninja ]]; then - cd build - fi - } - - configurePhase() { - mesonConfigurePhase - } - - buildPhase() { - cdToBuildDir - ninjaBuildPhase - } - - checkPhase() { - cdToBuildDir - mesonCheckPhase - } - - installPhase() { - cdToBuildDir - ninjaInstallPhase + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ../.version-determinate; + name = finalAttrs.pname; + + installFlags = "sysconfdir=$(out)/etc"; + shellHook = '' + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH + + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share + + # Make the default phases do the right thing. + # FIXME: this wouldn't be needed if the ninja package set buildPhase() instead of $buildPhase. + # FIXME: mesonConfigurePhase shouldn't cd to the build directory. It would be better to pass '-C ' to ninja. + + cdToBuildDir() { + if [[ ! -e build.ninja ]]; then + cd build + fi + } + + configurePhase() { + mesonConfigurePhase + } + + buildPhase() { + cdToBuildDir + ninjaBuildPhase + } + + checkPhase() { + cdToBuildDir + mesonCheckPhase + } + + installPhase() { + cdToBuildDir + ninjaInstallPhase + } + ''; + + # We use this shell with the local checkout, not unpackPhase. + src = null; + + # Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html + # Remove when gdb fix is rolled out everywhere. + separateDebugInfo = false; + + mesonBuildType = "debugoptimized"; + + env = { + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" + modular.pre-commit.settings.rawConfig + }"; } - ''; - - # We use this shell with the local checkout, not unpackPhase. - src = null; - - # Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html - # Remove when gdb fix is rolled out everywhere. - separateDebugInfo = false; - - mesonBuildType = "debugoptimized"; + // lib.optionalAttrs stdenv.hostPlatform.isLinux { + CC_LD = "mold"; + CXX_LD = "mold"; + }; - env = { - # For `make format`, to work without installing pre-commit - _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" - modular.pre-commit.settings.rawConfig - }"; - } - // lib.optionalAttrs stdenv.hostPlatform.isLinux { - CC_LD = "mold"; - CXX_LD = "mold"; - }; - - dontUseCmakeConfigure = true; - - mesonFlags = [ - (lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks")) - ] - ++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags) - ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags) - ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags) - ++ lib.optionals havePerl ( - map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags) - ) - ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags) - ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags) - ++ map (transformFlag "nix") (ignoreCrossFile pkgs.nixComponents2.nix-cli.mesonFlags); - - nativeBuildInputs = - let - inputs = - dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) ( - lib.lists.concatMap ( - # Nix manual has a build-time dependency on nix, but we - # don't want to do a native build just to enter the cross - # dev shell. - # - # TODO: think of a more principled fix for this. - c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs - ) activeComponents + dontUseCmakeConfigure = true; + + mesonFlags = [ + (lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks")) + ] + ++ map (transformFlag "libutil") (ignoreCrossFile nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile nixComponents.nix-fetchers.mesonFlags) + ++ map (transformFlag "libexpr") (ignoreCrossFile nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags) + ++ map (transformFlag "libutil-c") (ignoreCrossFile nixComponents.nix-util-c.mesonFlags) + ++ map (transformFlag "libstore-c") (ignoreCrossFile nixComponents.nix-store-c.mesonFlags) + ++ map (transformFlag "libfetchers-c") (ignoreCrossFile nixComponents.nix-fetchers-c.mesonFlags) + ++ map (transformFlag "libexpr-c") (ignoreCrossFile nixComponents.nix-expr-c.mesonFlags) + ++ map (transformFlag "libflake-c") (ignoreCrossFile nixComponents.nix-flake-c.mesonFlags) + ++ map (transformFlag "libmain-c") (ignoreCrossFile nixComponents.nix-main-c.mesonFlags) + ++ map (transformFlag "nix") (ignoreCrossFile nixComponents.nix-cli.mesonFlags); + + nativeBuildInputs = + let + inputs = + dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) ( + lib.lists.concatMap ( + # Nix manual has a build-time dependency on nix, but we + # don't want to do a native build just to enter the cross + # dev shell. + # + # TODO: think of a more principled fix for this. + c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs + ) activeComponents + ) ) - ) - ++ lib.optional ( - !buildCanExecuteHost - # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 - && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) - && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages - && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages) - ) pkgs.buildPackages.mesonEmulatorHook - ++ [ - pkgs.buildPackages.gnused - modular.pre-commit.settings.package - (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) - pkgs.buildPackages.nixfmt-rfc-style - pkgs.buildPackages.shellcheck - pkgs.buildPackages.include-what-you-use - ] - ++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb - ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( - lib.hiPrio pkgs.buildPackages.clang-tools - ) - ++ lib.optional stdenv.hostPlatform.isLinux pkgs.buildPackages.mold-wrapped; - in - # FIXME: separateDebugInfo = false doesn't actually prevent -Wa,--compress-debug-sections - # from making its way into NIX_CFLAGS_COMPILE. - lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs; - - propagatedNativeBuildInputs = dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) ( - lib.lists.concatMap (c: c.propagatedNativeBuildInputs) activeComponents - ) - ); + ++ lib.optional ( + !buildCanExecuteHost + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages) + ) pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.gnused + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) + pkgs.buildPackages.nixfmt + pkgs.buildPackages.shellcheck + pkgs.buildPackages.include-what-you-use + ] + ++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb + ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( + lib.hiPrio pkgs.buildPackages.clang-tools + ) + ++ lib.optional stdenv.hostPlatform.isLinux pkgs.buildPackages.mold; + in + # FIXME: separateDebugInfo = false doesn't actually prevent -Wa,--compress-debug-sections + # from making its way into NIX_CFLAGS_COMPILE. + lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs; + + propagatedNativeBuildInputs = dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) ( + lib.lists.concatMap (c: c.propagatedNativeBuildInputs) activeComponents + ) + ); - buildInputs = - # TODO change Nixpkgs to mark gbenchmark as building on Windows - lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark - ++ dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) - ) - ++ lib.optional havePerl pkgs.perl; + buildInputs = + # TODO change Nixpkgs to mark gbenchmark as building on Windows + lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark + ++ dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) + ); - propagatedBuildInputs = dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) - ); - } -) + propagatedBuildInputs = dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) + ); + } + )) +) { } diff --git a/packaging/everything.nix b/packaging/everything.nix index 1d07dd4da09a..fdb72663fcca 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -41,8 +41,6 @@ nix-internal-api-docs, nix-external-api-docs, - nix-perl-bindings, - testers, patchedSrc ? null, @@ -69,16 +67,7 @@ let nix-main-c nix-cmd ; - } - // - lib.optionalAttrs - (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - { - # Currently fails in static build - inherit - nix-perl-bindings - ; - }; + }; devdoc = buildEnv { name = "determinate-nix-${nix-cli.version}-devdoc"; @@ -121,7 +110,7 @@ stdenv.mkDerivation (finalAttrs: { dontBuild = true; /** - `doCheck` controles whether tests are added as build gate for the combined package. + `doCheck` controls whether tests are added as build gate for the combined package. This includes both the unit tests and the functional tests, but not the integration tests that run in CI (the flake's `hydraJobs` and some of the `checks`). */ @@ -146,11 +135,9 @@ stdenv.mkDerivation (finalAttrs: { nix-functional-tests ] ++ - lib.optionals (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) + lib.optionals (stdenv.hostPlatform.isLinux && stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ - # Perl currently fails in static build - # TODO: Split out tests into a separate derivation? - nix-perl-bindings + nix-util-tests.tests.run-without-new-syscalls ]; nativeBuildInputs = [ diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 8a9e525be869..5600d394ebb5 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -33,7 +33,6 @@ let forAllPackages = forAllPackages' { }; forAllPackages' = { - enableBindings ? false, enableDocs ? false, # already have separate attrs for these }: lib.genAttrs ( @@ -64,9 +63,7 @@ let "nix-cli" "nix-functional-tests" "nix-json-schema-checks" - ] - ++ lib.optionals enableBindings [ - "nix-perl-bindings" + "nix-clang-tidy-plugin" ] ++ lib.optionals enableDocs [ "nix-manual" @@ -84,7 +81,6 @@ rec { let arbitrarySystem = "x86_64-linux"; listedPkgs = forAllPackages' { - enableBindings = true; enableDocs = true; } (_: null); actualPkgs = lib.concatMapAttrs ( @@ -126,8 +122,25 @@ rec { system: self.devShells.${system}.default.inputDerivation )) [ "i686-linux" ]; - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-perl-bindings); + # Static analysis with clang-tidy + clangTidy = lib.genAttrs linux64BitSystems ( + system: + let + pkgs = nixpkgsFor.${system}.nativeForStdenv.clangStdenv; + tidyScope = pkgs.nixComponents2.overrideScope ( + self: super: { + withClangTidy = true; + # clang-tidy doesn't seem to like unity builds. + withUnityBuild = false; + # nix-everything is built via callPackage (not the layer system), so + # enableClangTidyLayer's doCheck=false doesn't reach it. Set it here + # so checkInputs (the *-tests.tests.run derivations) aren't pulled in. + nix-everything = super.nix-everything.overrideAttrs { doCheck = false; }; + } + ); + in + tidyScope.nix-everything + ); # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of @@ -241,6 +254,13 @@ rec { "NIX_CONFIG" = "lazy-trees = true"; }) ); + + filetransfer-retry-backoff = forAllSystems ( + system: + nixpkgsFor.${system}.native.callPackage ../tests/filetransfer-retry-backoff { + nix = nixpkgsFor.${system}.native.nixComponents2.nix-cli; + } + ); }; metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { diff --git a/packaging/installer/install.in b/packaging/installer/install.in index b4e808d8e941..af61a433ddf5 100755 --- a/packaging/installer/install.in +++ b/packaging/installer/install.in @@ -65,6 +65,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_aarch64-darwin@ system=aarch64-darwin ;; + FreeBSD.amd64|FreeBSD.x86_64) + hash=@tarballHash_x86_64-freebsd@ + path=@tarballPath_x86_64-freebsd@ + system=x86_64-freebsd + ;; *) oops "sorry, there is no binary distribution of Nix for your platform";; esac diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index 17339ce02171..28c2d1df6772 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -3,15 +3,15 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", "type": "github" }, "original": { - "owner": "edolstra", + "owner": "NixOS", "repo": "flake-compat", "type": "github" } @@ -24,12 +24,12 @@ ] }, "locked": { - "lastModified": 1748821116, - "narHash": "sha256-F82+gS044J1APL0n4hH50GYdPRv/5JWm34oCJYmVKdE=", - "rev": "49f0870db23e8c1ca0b5259734a02cd9e1e371a1", - "revCount": 377, + "lastModified": 1782949081, + "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=", + "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e", + "revCount": 480, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/hercules-ci/flake-parts/0.1.377%2Brev-49f0870db23e8c1ca0b5259734a02cd9e1e371a1/01972f28-554a-73f8-91f4-d488cc502f08/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/hercules-ci/flake-parts/0.1.480%2Brev-17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e/019f2195-dee5-7233-9747-eca0c27f7406/source.tar.gz" }, "original": { "type": "tarball", @@ -39,21 +39,18 @@ "git-hooks-nix": { "inputs": { "flake-compat": "flake-compat", - "gitignore": [ - "nix" - ], "nixpkgs": [ "nix", "nixpkgs" ] }, "locked": { - "lastModified": 1747372754, - "narHash": "sha256-2Y53NGIX2vxfie1rOW0Qb86vjRZ7ngizoo+bnXU9D9k=", - "rev": "80479b6ec16fefd9c1db3ea13aeb038c60530f46", - "revCount": 1026, + "lastModified": 1784288435, + "narHash": "sha256-ReRHaLgr/uVqdD8afFSn+myXIfpHeOhP0yYe0TJqAA8=", + "rev": "43b3c1ab9d40fb1dbb008f451988a91e375825e9", + "revCount": 1231, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/cachix/git-hooks.nix/0.1.1026%2Brev-80479b6ec16fefd9c1db3ea13aeb038c60530f46/0196d79a-1b35-7b8e-a021-c894fb62163d/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/cachix/git-hooks.nix/0.1.1231%2Brev-43b3c1ab9d40fb1dbb008f451988a91e375825e9/019f7135-8fdf-76f0-b1a1-d2c67e91af8d/source.tar.gz" }, "original": { "type": "tarball", @@ -80,16 +77,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1784330119, - "narHash": "sha256-0T9a4ybg+hskhl0DjG2wYYn7D9sHABXAbefZE3GSKeI=", - "rev": "e0a8747e95bf57d9bf61fc802e563675f7bf6d42", - "revCount": 915341, + "lastModified": 1784208194, + "narHash": "sha256-w4h9y1vJiAbDehtU80kRZhzgCv/kzIeTnediZQ7L7QU=", + "rev": "f0e42401b6814ba07b607df2122fe288e5ac4caf", + "revCount": 1006619, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-25.11/0.1.915341%2Brev-e0a8747e95bf57d9bf61fc802e563675f7bf6d42/019f726f-f9de-7dc2-aa15-ca490d995c1c/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1006619%2Brev-f0e42401b6814ba07b607df2122fe288e5ac4caf/019f6b37-08e3-7322-a4e3-04085e3df190/source.tar.gz" }, "original": { "type": "tarball", - "url": "https://flakehub.com/f/DeterminateSystems/secure-packages-25.11/0" + "url": "https://flakehub.com/f/DeterminateSystems/secure-packages-26.05/0" } }, "nixpkgs-23-11": { diff --git a/packaging/secure-packages/flake.nix b/packaging/secure-packages/flake.nix index 2e511a59fac5..723eb14fe414 100644 --- a/packaging/secure-packages/flake.nix +++ b/packaging/secure-packages/flake.nix @@ -1,6 +1,6 @@ { inputs.nix.url = "../.."; - inputs.nix.inputs.nixpkgs.url = "https://flakehub.com/f/DeterminateSystems/secure-packages-25.11/0"; + inputs.nix.inputs.nixpkgs.url = "https://flakehub.com/f/DeterminateSystems/secure-packages-26.05/0"; outputs = { self, nix }: nix.outputs; } diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 7a61764d4f33..538a5e74d5ee 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -832,8 +832,8 @@ EOF # TODO: should probably alert the user if this is disabled? _sudo "to launch the Nix volume mounter" \ launchctl bootstrap system "$NIX_VOLUME_MOUNTD_DEST" || true - # TODO: confirm whether kickstart is necessesary? - # I feel a little superstitous, but it can guard + # TODO: confirm whether kickstart is necessary? + # I feel a little superstitious, but it can guard # against multiple problems (doesn't start, old # version still running for some reason...) _sudo "to launch the Nix volume mounter" \ diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index d4ea88b5ea6c..ae6625e1bc41 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -270,7 +270,7 @@ _diff() { printf -v CHANGED_GROUP_FORMAT "%b" "${GREEN}%>${RED}%<${ESC}" diff --changed-group-format="$CHANGED_GROUP_FORMAT" "$@" else - # simple colorized diff comatible w/ pre `--color` versions + # simple colorized diff compatible w/ pre `--color` versions diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" fi } @@ -961,7 +961,7 @@ configure_shell_profile() { cert_in_store() { # in a subshell # - change into the cert-file dir - # - get the phyiscal pwd + # - get the physical pwd # and test if this path is in the Nix store [[ "$(cd -- "$(dirname "$NIX_SSL_CERT_FILE")" && exec pwd -P)" == "$NIX_ROOT/store/"* ]] } diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 1a20dffd2459..93cb3c45a55b 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -53,7 +53,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index abf716cec6fc..201a56438950 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -58,7 +58,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" diff --git a/src/perl/.version b/src/clang-tidy-plugin/.version similarity index 100% rename from src/perl/.version rename to src/clang-tidy-plugin/.version diff --git a/src/clang-tidy-plugin/meson.build b/src/clang-tidy-plugin/meson.build new file mode 100644 index 000000000000..7896bd085479 --- /dev/null +++ b/src/clang-tidy-plugin/meson.build @@ -0,0 +1,50 @@ +project( + 'nix-clang-tidy-plugin', + 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++23', + 'warning_level=2', + ], + meson_version : '>= 1.8', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('nix-meson-build-support/deps-lists') +subdir('nix-meson-build-support/common') + +# clang-tidy plugins require LLVM (clang headers provided via buildInputs) +llvm_dep = dependency('LLVM', version : '>= 16', required : true) + +sources = files( + 'nix-clang-tidy-checks.cc', +) + +# Build as a shared module (plugin) that can be loaded by clang-tidy --load +nix_clang_tidy_plugin = shared_module( + 'nix-clang-tidy', + sources, + dependencies : [ llvm_dep ], + install : true, +) + +# Export plugin path for consumers (nix-meson-build-support/common/clang-tidy/meson.build). +# In monolithic builds: override_dependency provides the build-dir path. +# In per-component Nix builds: the .pc file provides the installed path. +meson.override_dependency( + 'nix-clang-tidy', + declare_dependency( + variables : {'plugin_path' : nix_clang_tidy_plugin.full_path()}, + ), +) +import('pkgconfig').generate( + nix_clang_tidy_plugin, + filebase : 'nix-clang-tidy', + name : 'nix-clang-tidy', + description : 'Custom clang-tidy checks for Nix', + variables : { + 'plugin_path' : get_option('prefix') / get_option('libdir') / 'libnix-clang-tidy.so', + }, +) diff --git a/src/clang-tidy-plugin/nix-clang-tidy-checks.cc b/src/clang-tidy-plugin/nix-clang-tidy-checks.cc new file mode 100644 index 000000000000..64ec63c2d319 --- /dev/null +++ b/src/clang-tidy-plugin/nix-clang-tidy-checks.cc @@ -0,0 +1,34 @@ +/** + * @brief Custom clang-tidy checks for the Nix project. + * + * This module registers custom clang-tidy checks specific to the Nix codebase. + * To add a new check: + * 1. Create check-name.hh and check-name.cc in this directory + * 2. Include the header here + * 3. Register the check in addCheckFactories() + * 4. Add the source file to meson.build + * 5. Enable the check in .clang-tidy (e.g., nix-checkname) + */ + +#include +#include + +namespace nix::clang_tidy { + +using namespace clang; +using namespace clang::tidy; + +class NixClangTidyChecks : public ClangTidyModule +{ +public: + void addCheckFactories([[maybe_unused]] ClangTidyCheckFactories & CheckFactories) override + { + // Custom checks will be registered here. + // Example: + // CheckFactories.registerCheck("nix-my-custom-check"); + } +}; + +static ClangTidyModuleRegistry::Add X("nix-module", "Adds Nix-specific checks"); + +} // namespace nix::clang_tidy diff --git a/src/clang-tidy-plugin/nix-meson-build-support b/src/clang-tidy-plugin/nix-meson-build-support new file mode 120000 index 000000000000..0b140f56bdee --- /dev/null +++ b/src/clang-tidy-plugin/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/clang-tidy-plugin/package.nix b/src/clang-tidy-plugin/package.nix new file mode 100644 index 000000000000..d55a4b0b4ec0 --- /dev/null +++ b/src/clang-tidy-plugin/package.nix @@ -0,0 +1,46 @@ +{ + lib, + mkMesonDerivation, + + pkg-config, + llvmPackages, + + # Configuration Options + + version, +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-clang-tidy-plugin"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../nix-meson-build-support + ./nix-meson-build-support + ../../.version + ./.version + ./meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + pkg-config + llvmPackages.llvm.dev + ]; + + buildInputs = [ + llvmPackages.libclang + llvmPackages.clang-unwrapped.dev # for ClangTidyModule.h + ]; + + meta = { + description = "Custom clang-tidy checks for the Nix codebase"; + platforms = lib.platforms.unix; + }; +}) diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in index 3af2f5b813fa..ea8c069e9545 100644 --- a/src/external-api-docs/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -42,6 +42,8 @@ INPUT = \ @src@/src/libexpr-c \ @src@/src/libflake-c \ @src@/src/libstore-c \ + @src@/src/libfetchers-c \ + @src@/src/libmain-c \ @src@/src/external-api-docs/README.md FILE_PATTERNS = nix_api_*.h *.md @@ -61,3 +63,4 @@ USE_MDFILE_AS_MAINPAGE = @src@/src/external-api-docs/README.md WARN_IF_UNDOCUMENTED = NO WARN_IF_INCOMPLETE_DOC = NO QUIET = YES +WARNINGS = @WARNINGS@ diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build index cba6f646b93b..d96da3863e6f 100644 --- a/src/external-api-docs/meson.build +++ b/src/external-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-external-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -14,6 +14,7 @@ doxygen_cfg = configure_file( 'PROJECT_NUMBER' : meson.project_version(), 'OUTPUT_DIRECTORY' : meson.current_build_dir(), 'src' : fs.parent(fs.parent(meson.project_source_root())), + 'WARNINGS' : meson.is_subproject() ? 'NO' : 'YES', }, ) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index 28cde8c09e69..79fe692fa572 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -31,7 +31,9 @@ mkMesonDerivation (finalAttrs: { # Source is not compiled, but still must be available for Doxygen # to gather comments. (cpp ../libexpr-c) + (cpp ../libfetchers-c) (cpp ../libflake-c) + (cpp ../libmain-c) (cpp ../libstore-c) (cpp ../libutil-c) ]; diff --git a/src/internal-api-docs/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in index 2769edd9f727..aa81d77c5684 100644 --- a/src/internal-api-docs/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -112,3 +112,4 @@ PREDEFINED = DOXYGEN_SKIP WARN_IF_UNDOCUMENTED = NO WARN_IF_INCOMPLETE_DOC = NO QUIET = YES +WARNINGS = @WARNINGS@ diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build index daab4c93c309..3976e4c3f81a 100644 --- a/src/internal-api-docs/meson.build +++ b/src/internal-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-internal-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -15,6 +15,7 @@ doxygen_cfg = configure_file( 'OUTPUT_DIRECTORY' : meson.current_build_dir(), 'BUILD_ROOT' : meson.build_root(), 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + 'WARNINGS' : meson.is_subproject() ? 'NO' : 'YES', }, ) diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index a1b525bc51e4..66dc9b758a44 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -6,7 +6,7 @@ project( 'nix-json-schema-checks', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -51,6 +51,13 @@ schemas = [ 'simple.json', ], }, + { + 'stem' : 'signature', + 'schema' : schema_dir / 'signature-v2.yaml', + 'files' : [ + 'simple.json', + ], + }, { 'stem' : 'deriving-path', 'schema' : schema_dir / 'deriving-path-v1.yaml', @@ -62,12 +69,9 @@ schemas = [ }, { 'stem' : 'build-trace-entry', - 'schema' : schema_dir / 'build-trace-entry-v2.yaml', + 'schema' : schema_dir / 'build-trace-entry-v3.yaml', 'files' : [ 'simple.json', - # The field is no longer supported, but we want to show that we - # ignore it during parsing. - 'with-dependent-realisations.json', 'with-signature.json', ], }, @@ -94,6 +98,17 @@ schemas += [ 'files' : [ 'dyn-dep-derivation.json', 'simple-derivation.json', + 'try-resolve' / 'no-inputs-before.json', + 'try-resolve' / 'with-inputs-before.json', + 'try-resolve' / 'resolution-failure-before.json', + ], + }, + { + 'stem' : 'derivation', + 'schema' : schema_dir / 'derivation-resolved-v4.yaml', + 'files' : [ + 'try-resolve' / 'no-inputs-after.json', + 'try-resolve' / 'with-inputs-after.json', ], }, { @@ -154,20 +169,20 @@ schemas += [ # Match overall { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml', + 'schema' : schema_dir / 'store-object-info-v3.yaml', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'impure.json', - 'json-2' / 'empty_pure.json', - 'json-2' / 'empty_impure.json', + 'json-3' / 'pure.json', + 'json-3' / 'impure.json', + 'json-3' / 'empty_pure.json', + 'json-3' / 'empty_impure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml', + 'schema' : schema_dir / 'store-object-info-v3.yaml', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'impure.json', + 'json-3' / 'pure.json', + 'json-3' / 'impure.json', ], }, { @@ -182,32 +197,32 @@ schemas += [ # Match exact variant { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/base', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'empty_pure.json', + 'json-3' / 'pure.json', + 'json-3' / 'empty_pure.json', ], }, { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/impure', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/impure', 'files' : [ - 'json-2' / 'impure.json', - 'json-2' / 'empty_impure.json', + 'json-3' / 'impure.json', + 'json-3' / 'empty_impure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/base', 'files' : [ - 'json-2' / 'pure.json', + 'json-3' / 'pure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/narInfo', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/narInfo', 'files' : [ - 'json-2' / 'impure.json', + 'json-3' / 'impure.json', ], }, ] diff --git a/src/json-schema-checks/signature/simple.json b/src/json-schema-checks/signature/simple.json new file mode 100644 index 000000000000..abf69dfa0f92 --- /dev/null +++ b/src/json-schema-checks/signature/simple.json @@ -0,0 +1,4 @@ +{ + "keyName": "cache.nixos.org-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +} diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index fccd3997e16b..dd8d8a0f7e36 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,12 +1,12 @@ #include "nix/cmd/built-path.hh" +#include "nix/store/build-result.hh" #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" +#include "nix/store/outputs-query.hh" #include "nix/util/comparator.hh" #include -#include - namespace nix { // Custom implementation to avoid `ref` ptr equality @@ -108,24 +108,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const overloaded{ [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { - auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto & [outputName, outputPath] : p.outputs) { - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath->outPath()), - outputName); - DrvOutput key{*drvOutput, outputName}; - auto thisRealisation = store.queryRealisation(key); - if (thisRealisation) - res.insert(Realisation{*thisRealisation, std::move(key)}); - else - res.insert(outputPath); - } else { - res.insert(outputPath); - } + /* Use a custom callback to collect realisations as they're queried. */ + deepQueryPartialDerivationOutput( + store, p.drvPath->outPath(), outputName, nullptr, [&](const DrvOutput & drvOutput) { + auto realisation = store.queryRealisation(drvOutput); + if (realisation) + res.insert(Realisation{*realisation, drvOutput}); + return realisation; + }); + res.insert(outputPath); } }, }, @@ -133,4 +125,50 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const return res; } +SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { return SingleBuiltPath::Opaque{bo.path}; }, + [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { + auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); + // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. + SingleDerivedPath::Built truncatedBfd{ + .drvPath = makeConstantStorePathRef(drvPath.outPath()), + .output = bfd.output, + }; + auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); + return SingleBuiltPath::Built{ + .drvPath = make_ref(std::move(drvPath)), + .output = {bfd.output, outputPath}, + }; + }, + }, + b.raw()); +} + +BuiltPath toBuiltPath(KeyedBuildResult & result, ref evalStore, ref store) +{ + auto success = result.tryGetSuccess(); + assert(success); + return std::visit( + overloaded{ + [&](const DerivedPath::Built & bfd) { + std::map outputs; + for (auto & [outputName, realisation] : success->builtOutputs) + outputs.emplace(outputName, realisation.outPath); + BuiltPath bp = BuiltPath::Built{ + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }; + return bp; + }, + [&](const DerivedPath::Opaque & bo) { + BuiltPath bp = BuiltPath::Opaque{bo.path}; + return bp; + }, + }, + result.path.raw()); +} + } // namespace nix diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index f61d38e4a3ea..045665ebf2fa 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -447,9 +447,34 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu void MixOutLinkBase::createOutLinksMaybe(const std::vector & buildables, ref & store) { - if (outLink != "") + createOutLinksMaybe(toBuiltPaths(buildables), store); +} + +void MixOutLinkBase::createOutLinksMaybe(const BuiltPaths & paths, ref & store) +{ + if (outLink) if (auto store2 = store.dynamic_pointer_cast()) - createOutLinks(outLink, toBuiltPaths(buildables), *store2); + createOutLinks(*outLink, paths, *store2); +} + +void MixPrintOutPaths::printOutPathsMaybe(const BuiltPaths & paths, ref store) +{ + if (!printOutputPaths) + return; + + logger->stop(); + for (auto & path : paths) { + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & bo) { logger->cout(store->printStorePath(bo.path)); }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + logger->cout(store->printStorePath(output.second)); + } + }, + }, + path); + } } } // namespace nix diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 3b0e07b2b1fa..a396a172816e 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,9 +1,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/expr/eval-settings.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/main/shared.hh" #include "nix/util/config-global.hh" -#include "nix/store/filetransfer.hh" #include "nix/expr/eval.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/registry.hh" diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index 95fdf95ad00c..83aba78d8f4b 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,22 +1,56 @@ #include "nix/cmd/editor-for.hh" #include "nix/util/environment-variables.hh" #include "nix/util/source-path.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" namespace nix { -Strings editorFor(const SourcePath & file, uint32_t line) +std::tuple editorFor(const SourcePath & file, uint32_t line, bool readOnly) { auto path = file.getPhysicalPath(); - if (!path) - throw Error("cannot open '%s' in an editor because it has no physical path", file); - auto editor = getEnv("EDITOR").value_or("cat"); - auto args = tokenizeString(editor); + OsString editor = getEnvOsNonEmpty(OS_STR("EDITOR")).value_or(OS_STR("cat")); + auto args = tokenizeString(editor); if (line > 0 - && (editor.find("emacs") != std::string::npos || editor.find("nano") != std::string::npos - || editor.find("vim") != std::string::npos || editor.find("kak") != std::string::npos)) - args.push_back(fmt("+%d", line)); - args.push_back(path->string()); - return args; + && (editor.contains(OS_STR("emacs")) || editor.contains(OS_STR("nano")) || editor.contains(OS_STR("vim")) + || editor.contains(OS_STR("kak")))) + args.push_back(string_to_os_string(fmt("+%d", line))); + + if (path) { + args.push_back(path->native()); + return {std::move(args), AutoCloseFD{}, AutoDelete{}}; + } + + /* Resolve symlinks when creating a temporary. That's how a regular editor + would behave. Also GitSourceAccessor behaves poorly with symlinks in the + path and fails with "«...» does not exist". */ + auto file2 = file.resolveSymlinks(); + auto stat = file2.lstat(); + /* TODO: Maybe we should print a directory listing and open that instead? */ + if (stat.type != SourceAccessor::tRegular) + throw Error("can't open a file %s of type '%s'", file2.to_string(), stat.typeString()); + + auto tempDir = createTempDir(defaultTempDir(), "nix-edit", 0700); + AutoDelete autoDel(tempDir, /*recursive=*/true); + auto tempPath = tempDir / file2.path.baseName().value_or("nix-edit"); + + /* Create the file with the same name, so editors that recognise file + extensions spin up syntax highlighting and LSPs. */ + auto tempFd = openNewFileForWrite( + tempPath, + readOnly ? 0400 : 0600, + {.truncateExisting = false, .followSymlinksOnTruncate = false, .writeOnly = false}); + + if (!tempFd) + throw NativeSysError("failed to create temporary file %s", PathFmt(tempPath)); + + /* Copy the contents into the created copy. */ + FdSink fileSink(tempFd.get()); + file2.readFile(fileSink); + fileSink.flush(); + args.push_back(tempPath); + + return {std::move(args), std::move(tempFd), std::move(autoDel)}; } } // namespace nix diff --git a/src/libcmd/include/nix/cmd/built-path.hh b/src/libcmd/include/nix/cmd/built-path.hh index d41529e5ac4f..34761ee02dc8 100644 --- a/src/libcmd/include/nix/cmd/built-path.hh +++ b/src/libcmd/include/nix/cmd/built-path.hh @@ -105,4 +105,10 @@ struct BuiltPath : _BuiltPathRaw typedef std::vector BuiltPaths; +SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b); + +struct KeyedBuildResult; + +BuiltPath toBuiltPath(KeyedBuildResult & result, ref evalStore, ref store); + } // namespace nix diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index cfbd9eeec96e..c43b4f87ed43 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -421,13 +421,16 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu struct MixOutLinkBase : virtual Args { /** Prefix for any output symlinks. Empty means do not write an output symlink. */ - std::filesystem::path outLink; + std::optional outLink = std::nullopt; - MixOutLinkBase(const std::string & defaultOutLink) + MixOutLinkBase(const std::optional & defaultOutLink) : outLink(defaultOutLink) { } + /** underlying function */ + void createOutLinksMaybe(const BuiltPaths & paths, ref & store); + /** smaller wrapper for convenience (historically this was the only one) */ void createOutLinksMaybe(const std::vector & buildables, ref & store); }; @@ -449,9 +452,25 @@ struct MixOutLinkByDefault : MixOutLinkBase, virtual Args addFlag({ .longName = "no-link", .description = "Do not create symlinks to the build results.", - .handler = {&outLink, std::filesystem::path{}}, + .handler = {[&] { outLink = std::nullopt; }}, }); } }; +struct MixPrintOutPaths : virtual Args +{ + bool printOutputPaths = false; + + MixPrintOutPaths() + { + addFlag({ + .longName = "print-out-paths", + .description = "Print the resulting output paths", + .handler = {&printOutputPaths, true}, + }); + } + + void printOutPathsMaybe(const BuiltPaths & paths, ref store); +}; + } // namespace nix diff --git a/src/libcmd/include/nix/cmd/editor-for.hh b/src/libcmd/include/nix/cmd/editor-for.hh index 3fb8a072e732..998b60f4ee25 100644 --- a/src/libcmd/include/nix/cmd/editor-for.hh +++ b/src/libcmd/include/nix/cmd/editor-for.hh @@ -1,15 +1,24 @@ #pragma once ///@file -#include "nix/util/types.hh" #include "nix/util/source-path.hh" +#include "nix/util/os-string.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" namespace nix { /** * Helper function to generate args that invoke $EDITOR on * filename:lineno. + * + * When file doesn't have a physical path, the contents get copied into a + * temporary file, and a file descriptor and RAII cleanup guard for it are + * returned. + * + * @param readOnly make the temporary file readonly if the file has no physical + * path. Ignored otherwise. */ -Strings editorFor(const SourcePath & file, uint32_t line); +std::tuple editorFor(const SourcePath & file, uint32_t line, bool readOnly = true); } // namespace nix diff --git a/src/libcmd/include/nix/cmd/installable-attr-path.hh b/src/libcmd/include/nix/cmd/installable-attr-path.hh index b563508a95b6..c33b119a5fa8 100644 --- a/src/libcmd/include/nix/cmd/installable-attr-path.hh +++ b/src/libcmd/include/nix/cmd/installable-attr-path.hh @@ -17,10 +17,10 @@ #include "nix/util/url.hh" #include "nix/fetchers/registry.hh" #include "nix/store/build-result.hh" +#include "nix/expr/root-value.hh" #include #include -#include "nix/expr/root-value.hh" namespace nix { diff --git a/src/libcmd/include/nix/cmd/misc-store-flags.hh b/src/libcmd/include/nix/cmd/misc-store-flags.hh index 27e139076802..5c38a2593427 100644 --- a/src/libcmd/include/nix/cmd/misc-store-flags.hh +++ b/src/libcmd/include/nix/cmd/misc-store-flags.hh @@ -1,3 +1,6 @@ +#pragma once +/// @file + #include "nix/util/args.hh" #include "nix/store/content-address.hh" diff --git a/src/libcmd/include/nix/cmd/repl-interacter.hh b/src/libcmd/include/nix/cmd/repl-interacter.hh index 7cba481059c9..9b9a03c3e423 100644 --- a/src/libcmd/include/nix/cmd/repl-interacter.hh +++ b/src/libcmd/include/nix/cmd/repl-interacter.hh @@ -1,8 +1,10 @@ #pragma once /// @file +#include "nix/util/file-descriptor.hh" #include "nix/util/finally.hh" #include "nix/util/fun.hh" +#include "nix/util/terminal.hh" #include "nix/util/types.hh" #include #include @@ -39,6 +41,8 @@ public: class ReadlineLikeInteracter : public virtual ReplInteracter { std::filesystem::path historyFile; + bool isInteractive = nix::isTTY(getStandardInput()); + public: ReadlineLikeInteracter(std::filesystem::path historyFile) : historyFile(std::move(historyFile)) diff --git a/src/libcmd/include/nix/cmd/repl.hh b/src/libcmd/include/nix/cmd/repl.hh index d46aa94b6b50..81c7b8df5a2d 100644 --- a/src/libcmd/include/nix/cmd/repl.hh +++ b/src/libcmd/include/nix/cmd/repl.hh @@ -28,7 +28,7 @@ struct AbstractNixRepl * @param programName Name of the command, e.g. `nix` or `nix-env`. * @param args arguments to the command. */ - using RunNix = void(const std::string & programName, OsStrings args, const std::optional & input); + using RunNix = void(const std::string & programName, OsStrings args); /** * @param runNix Function to run the nix CLI to support various diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh index 1e5661d7abb9..e01475104b5a 100644 --- a/src/libcmd/include/nix/cmd/unix-socket-server.hh +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -53,6 +53,26 @@ struct ServeUnixSocketOptions * Mode for the created socket file. */ mode_t socketMode = 0666; + +#ifndef _WIN32 + /** + * Name of the socket for socket activation, as included in `LISTEN_FDNAMES` + * Ordinarily the name of the socket unit, e.g. `nix-daemon.socket` + * If this field is empty, no name filtering will be performed. + */ + std::string activationName = ""; + + /** + * Additional file descriptor to poll. Useful for doing a self-pipe trick + * https://cr.yp.to/docs/selfpipe.html. + */ + Descriptor auxiliaryFd = INVALID_DESCRIPTOR; + + /** + * Optional callback invoked on POLLIN event for auxiliaryFd. + */ + std::function onAuxiliaryFdPollin = nullptr; +#endif }; MakeError(AbortServeSocket, BaseError); diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 3b6fb336f2b2..524dc799f02f 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -1,24 +1,13 @@ -#include "nix/store/globals.hh" #include "nix/cmd/installable-attr-path.hh" #include "nix/store/outputs-spec.hh" #include "nix/util/util.hh" #include "nix/cmd/command.hh" #include "nix/expr/attr-path.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/store/derivations.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" #include "nix/expr/get-drvs.hh" -#include "nix/store/store-api.hh" -#include "nix/main/shared.hh" #include "nix/flake/flake.hh" -#include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" -#include "nix/fetchers/registry.hh" -#include "nix/store/build-result.hh" - -#include -#include #include diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 8318ba2c55d8..d2022aeb09d0 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -1,29 +1,18 @@ #include "nix/store/globals.hh" #include "nix/cmd/installable-flake.hh" -#include "nix/cmd/installable-derived-path.hh" #include "nix/store/outputs-spec.hh" #include "nix/util/util.hh" #include "nix/cmd/command.hh" #include "nix/expr/attr-path.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/store/derivations.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-error.hh" -#include "nix/expr/get-drvs.hh" -#include "nix/store/store-api.hh" -#include "nix/main/shared.hh" #include "nix/flake/flake.hh" #include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" -#include "nix/fetchers/registry.hh" -#include "nix/store/build-result.hh" #include "nix/flake/provenance.hh" #include "nix/cmd/flake-schemas.hh" -#include -#include - #include namespace nix { diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index b297685250b5..2285e0c3cf51 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,3 +1,4 @@ +#include "nix/cmd/built-path.hh" #include "nix/store/globals.hh" #include "nix/cmd/installables.hh" #include "nix/cmd/installable-derived-path.hh" @@ -13,19 +14,14 @@ #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" -#include "nix/expr/get-drvs.hh" #include "nix/store/store-api.hh" #include "nix/main/shared.hh" #include "nix/flake/flake.hh" #include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" #include "nix/fetchers/registry.hh" #include "nix/store/build-result.hh" #include "nix/util/exit.hh" -#include -#include - #include #include "nix/util/strings-inline.hh" @@ -471,28 +467,6 @@ ref SourceExprCommand::parseInstallable(ref store, const std return installables.front(); } -static SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) -{ - return std::visit( - overloaded{ - [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { return SingleBuiltPath::Opaque{bo.path}; }, - [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { - auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); - // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. - SingleDerivedPath::Built truncatedBfd{ - .drvPath = makeConstantStorePathRef(drvPath.outPath()), - .output = bfd.output, - }; - auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); - return SingleBuiltPath::Built{ - .drvPath = make_ref(std::move(drvPath)), - .output = {bfd.output, outputPath}, - }; - }, - }, - b.raw()); -} - const BuiltPathWithResult & InstallableWithBuildResult::getSuccess() const { if (auto * failure = std::get_if(&result)) { @@ -627,34 +601,15 @@ std::vector Installable::build2( } continue; } - auto & success = std::get(buildResult.inner); for (auto & aux : backmap[buildResult.path]) { - std::visit( - overloaded{ - [&](const DerivedPath::Built & bfd) { - std::map outputs; - for (auto & [outputName, realisation] : success.builtOutputs) - outputs.emplace(outputName, realisation.outPath); - res.push_back( - {.installable = aux.installable, - .result = InstallableWithBuildResult::Success{ - .path = - BuiltPath::Built{ - .drvPath = make_ref( - getBuiltPath(evalStore, store, *bfd.drvPath)), - .outputs = outputs, - }, - .info = aux.info, - .result = buildResult}}); - }, - [&](const DerivedPath::Opaque & bo) { - res.push_back( - {.installable = aux.installable, - .result = InstallableWithBuildResult::Success{ - .path = BuiltPath::Opaque{bo.path}, .info = aux.info, .result = buildResult}}); - }, + res.push_back({ + aux.installable, + BuiltPathWithResult{ + .path = toBuiltPath(buildResult, evalStore, store), + .info = aux.info, + .result = buildResult, }, - buildResult.path.raw()); + }); } } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index b14478fb4802..680b8136dd32 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 8eebeec25b5e..5f2417dc1331 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -18,11 +18,10 @@ extern "C" { } #endif -#include "nix/util/signals.hh" #include "nix/util/finally.hh" #include "nix/cmd/repl-interacter.hh" #include "nix/util/file-system.hh" -#include "nix/cmd/repl.hh" +#include "nix/util/serialise.hh" #include "nix/util/environment-variables.hh" namespace nix { @@ -123,9 +122,38 @@ ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleter logWarning(e.info()); } #if !USE_READLINE - el_hist_size = 1000; -#endif + /* editline's read_history uses a fixed 256-byte buffer (SCREEN_INC), + which silently splits lines longer than 255 characters into separate + history entries. Read the file ourselves to avoid the length limit. See: + https://github.com/troglobit/editline/blob/2e0504d31e6878208036a4dd91f44841dabb1ee7/src/editline.c#L1617-L1635 + + ::rl_initialize must be called before the subsequent calls to + ::add_history to ensure that the buffer has actually been allocated + (but before setting ::el_hist_size). + Best I can tell it's supposed to idempotent, e.g. ::readline calls it + unconditionally anyway. */ + + ::el_hist_size = 1000; /* FIXME: Why the arbitrary limit? */ + ::rl_initialize(); + + auto fd = openFileReadonly(historyFile); + if (!fd) { + NativeSysError err("opening file %s", PathFmt(historyFile)); + if (!err.is(std::errc::no_such_file_or_directory) && !err.is(std::errc::not_a_directory)) + logWarning(err.info()); + } else { + try { + FdSource source(fd.get()); + while (true) + add_history(source.readLine().c_str()); + } catch (EndOfFile &) { + } catch (SystemError & e) { + logWarning(e.info()); + } + } +#else read_history(historyFile.string().c_str()); +#endif auto oldRepl = curRepl; curRepl = repl; Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); @@ -175,8 +203,25 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT setupSignals(); #endif - char * s = readline(promptForType(promptType)); - Finally doFree([&]() { free(s); }); + + /* Buffer for the non-interactive input. */ + std::string buffer; + const char * s = nullptr; + char * rl = nullptr; + + /* Use plain std::getline for non-interactive mode, which we also use for + testing purposes. readline/editline seem to disagree too much about how + to handle final prompts etc., so it's easier to bypass those. The tests + are mostly about testing the core repl logic, not input handling. */ + if (isInteractive) { + rl = ::readline(promptForType(promptType)); + s = rl; + } else { + s = std::getline(std::cin, buffer) ? buffer.c_str() : nullptr; + } + + Finally doFree([&]() { ::free(rl); }); + #ifndef _WIN32 // TODO use more signals.hh for this restoreSignals(); #endif @@ -187,15 +232,12 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return true; } - // editline doesn't echo the input to the output when non-interactive, unlike readline - // this results in a different behavior when running tests. The echoing is - // quite useful for reading the test output, so we add it here. + /* Echo the prompt into the output if run in non-interactive mode, somewhat + for the purposes of characterisation tests. */ if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") { -#if !USE_READLINE // This is probably not right for multi-line input, but we don't use that // in the characterisation tests, so it's fine. std::cout << promptForType(promptType) << s << std::endl; -#endif } if (!s) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 0186afcac527..6eaab847a0a6 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -17,6 +17,7 @@ #include "nix/cmd/get-build-log.hh" #include "nix/expr/get-drvs.hh" #include "nix/store/derivations.hh" +#include "nix/store/outputs-query.hh" #include "nix/store/globals.hh" #include "nix/flake/flake.hh" #include "nix/flake/lockfile.hh" @@ -74,7 +75,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc RunNix * runNixPtr; - void runNix(const std::string & program, OsStrings args, const std::optional & input = {}); + void runNix(const std::string & program, OsStrings args); std::unique_ptr interacter; @@ -160,7 +161,24 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi return out; } -MakeError(IncompleteReplExpr, ParseError); +/** + * Thrown when the REPL's own input is incomplete (e.g. unclosed multi-line + * string or open parenthesis). The mainLoop catches this to prompt for + * continuation lines instead of showing an error. + * + * Only parseString and parseReplBindings may throw this. Evaluation can also + * produce "unexpected end of file" ParseErrors (e.g. `import ./broken.nix`), + * but those must be reported as errors, not trigger continuation. The + * exception subtype is what distinguishes the two cases. + */ +MakeError(IncompleteReplExpr, Error); + +static bool isIncompleteInput(const ParseError & e) +{ + return e.msg().find("unexpected end of file") != std::string::npos; +} + +void IncompleteReplExpr::anchor() {} static bool isFirstRepl = true; @@ -171,7 +189,7 @@ ReplExitStatus NixRepl::mainLoop() if (state->debugRepl) { debuggerNotice = " debugger"; } - notice("Nix %s\nType :? for help.", version(), debuggerNotice); + notice("Nix %1%%2%\nType :? for help.", version(), debuggerNotice); } isFirstRepl = false; @@ -484,8 +502,8 @@ ProcessLineResult NixRepl::processLine(std::string line) } }(); - // Open in EDITOR - auto args = editorFor(path, line); + /* Open file in EDITOR, or edit a read-only copy if the file doesn't have a physical path. */ + auto [args, fd, autoDel] = editorFor(path, line, /*readOnly=*/true); auto editor = args.front(); args.pop_front(); @@ -494,13 +512,16 @@ ProcessLineResult NixRepl::processLine(std::string line) runProgram2({ .program = editor, .lookupPath = true, - .args = toOsStrings(std::move(args)), + .args = std::move(args), .isInteractive = true, }); - // Reload right after exiting the editor - state->resetFileCache(); - reloadFilesAndFlakes(); + /* If we had to open a temporary read-only file, there's no need to + reload (no files could have changed anyway). */ + if (!fd) { + state->resetFileCache(); + reloadFilesAndFlakes(); + } } else if (command == ":t") { @@ -536,7 +557,7 @@ ProcessLineResult NixRepl::processLine(std::string line) }); auto drv = state->store->readDerivation(drvPath); logger->cout("\nThis derivation produced the following outputs:"); - for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) { + for (auto & [outputName, outputPath] : deepQueryDerivationOutputMap(*state->store, drvPath)) { auto localStore = state->store.dynamic_pointer_cast(); if (localStore && command == ":bl") { std::string symlink = "repl-result-" + outputName; @@ -657,13 +678,7 @@ ProcessLineResult NixRepl::processLine(std::string line) else { // Try parsing as bindings first (handles `x = 1`, `inherit ...`, etc.) - ExprAttrs * bindings = nullptr; - try { - bindings = parseReplBindings(line); - } catch (IncompleteReplExpr &) { - throw; - } catch (ParseError &) { - } + ExprAttrs * bindings = parseReplBindings(line); if (bindings) { Env * inheritEnv = bindings->inheritFromExprs ? bindings->buildInheritFromEnv(*state, *env) : nullptr; @@ -687,12 +702,13 @@ ProcessLineResult NixRepl::processLine(std::string line) void NixRepl::loadFile(const std::filesystem::path & path) { - loadedFiles.remove(path); - loadedFiles.push_back(path); Value v, v2; state->evalFile(lookupFileArg(*state, path.string()), v); state->autoCallFunction(*autoArgs, v, v2); addAttrsToScope(v2); + // Remember for :reload only on success. + loadedFiles.remove(path); + loadedFiles.push_back(path); } void NixRepl::loadFlake(const std::string & flakeRefS) @@ -700,9 +716,6 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - loadedFlakes.remove(flakeRefS); - loadedFlakes.push_back(flakeRefS); - std::filesystem::path cwd; try { cwd = std::filesystem::current_path(); @@ -729,6 +742,10 @@ void NixRepl::loadFlake(const std::string & flakeRefS) }), v); addAttrsToScope(v); + + // Remember for :reload only on success. + loadedFlakes.remove(flakeRefS); + loadedFlakes.push_back(flakeRefS); } void NixRepl::initEnv() @@ -771,28 +788,44 @@ void NixRepl::reloadFilesAndFlakes() void NixRepl::loadFiles() { - decltype(loadedFiles) old = loadedFiles; - loadedFiles.clear(); + // loadFile() rebuilds loadedFiles; keep failed entries and continue. + decltype(loadedFiles) old; + std::swap(old, loadedFiles); for (auto & i : old) { notice("Loading %1%...", PathFmt(i)); - loadFile(i); + try { + loadFile(i); + } catch (Error & e) { + loadedFiles.push_back(i); + printMsg(lvlError, e.msg()); + } } for (auto & [i, what] : getValues()) { notice("Loading installable '%1%'...", what); - addAttrsToScope(*i); + try { + addAttrsToScope(*i); + } catch (Error & e) { + printMsg(lvlError, e.msg()); + } } } void NixRepl::loadFlakes() { - Strings old = loadedFlakes; - loadedFlakes.clear(); + // See loadFiles(). + Strings old; + std::swap(old, loadedFlakes); for (auto & i : old) { notice("Loading flake '%1%'...", i); - loadFlake(i); + try { + loadFlake(i); + } catch (Error & e) { + loadedFlakes.push_back(i); + printMsg(lvlError, e.msg()); + } } } @@ -853,12 +886,9 @@ Expr * NixRepl::parseString(std::string s) try { return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv); } catch (ParseError & e) { - if (e.msg().find("unexpected end of file") != std::string::npos) - // For parse errors on incomplete input, we continue waiting for the next line of - // input without clearing the input so far. + if (isIncompleteInput(e)) throw IncompleteReplExpr(e.msg()); - else - throw; + throw; } } @@ -867,11 +897,9 @@ ExprAttrs * NixRepl::parseReplBindings(std::string s) auto basePath = state->rootPath("."); // Try parsing as bindings - std::exception_ptr bindingsError; try { return state->parseReplBindings(s, basePath, staticEnv); } catch (ParseError &) { - bindingsError = std::current_exception(); } // Try with semicolon appended (for `inherit foo` shorthand) @@ -879,14 +907,10 @@ ExprAttrs * NixRepl::parseReplBindings(std::string s) try { return state->parseReplBindings(s + ";", s, basePath, staticEnv); } catch (ParseError & e) { - // All binding parse attempts failed. If the last error indicates - // incomplete input (e.g. unclosed multi-line string), signal the - // mainLoop to read continuation lines instead of falling through - // to expression parsing which would produce a misleading error. - if (e.msg().find("unexpected end of file") != std::string::npos) + if (isIncompleteInput(e)) throw IncompleteReplExpr(e.msg()); - // Semicolon retry failed; rethrow the original bindings error - std::rethrow_exception(bindingsError); + // Semicolon retry also failed; not valid binding syntax. + return nullptr; } } @@ -897,10 +921,10 @@ void NixRepl::evalString(std::string s, Value & v) state->forceValue(v, v.determinePos(noPos)); } -void NixRepl::runNix(const std::string & program, OsStrings args, const std::optional & input) +void NixRepl::runNix(const std::string & program, OsStrings args) { if (runNixPtr) - (*runNixPtr)(program, std::move(args), input); + (*runNixPtr)(program, std::move(args)); else throw Error( "Cannot run '%s' because no method of calling the Nix CLI was provided. This is a configuration problem pertaining to how this program was built. See Nix 2.25 release notes", diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index 09d726de66c5..84a72b597c5a 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -9,6 +9,8 @@ #include "nix/util/unix-domain-socket.hh" #include "nix/util/util.hh" +#include + #include #include #include @@ -19,6 +21,8 @@ namespace nix::unix { +void AbortServeSocket::anchor() {} + PeerInfo getPeerInfo(Descriptor remote) { PeerInfo peer; @@ -64,9 +68,16 @@ PeerInfo getPeerInfo(Descriptor remote) if (listenFds) { if (getEnv("LISTEN_PID") != std::to_string(getpid())) throw Error("unexpected systemd environment variables"); + + auto fdNames = tokenizeString>(getEnv("LISTEN_FDNAMES").value_or(""), ":"); auto count = string2Int(*listenFds); assert(count); for (unsigned int i = 0; i < count; ++i) { + // Not all implementations of LISTEN_FDS will implement names, + // listen anyway if we do not have enough names + if (i < fdNames.size() && options.activationName != "" && fdNames[i] != options.activationName) + continue; + AutoCloseFD fdSocket(SD_LISTEN_FDS_START + i); closeOnExec(fdSocket.get()); listeningSockets.push_back(std::move(fdSocket)); @@ -83,6 +94,9 @@ PeerInfo getPeerInfo(Descriptor remote) for (auto & i : listeningSockets) fds.push_back({.fd = i.get(), .events = POLLIN}); + if (options.auxiliaryFd != INVALID_DESCRIPTOR) + fds.push_back({.fd = options.auxiliaryFd, .events = POLLIN}); + // Loop accepting connections. while (1) { try { @@ -95,7 +109,11 @@ PeerInfo getPeerInfo(Descriptor remote) throw SysError("polling for incoming connections"); } - for (auto & fd : fds) { + if (options.auxiliaryFd != INVALID_DESCRIPTOR && options.onAuxiliaryFdPollin && fds.back().revents & POLLIN) + /* Useful for reaping children. */ + options.onAuxiliaryFdPollin(); + + for (auto & fd : std::views::take(fds, listeningSockets.size())) { if (!fd.revents) continue; diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 1ef3404a25ea..90435b4095bb 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -47,15 +47,32 @@ headers = files( subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixexprc', - sources, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : prelink, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixexprc', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixexprc', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libexpr-c/meson.options b/src/libexpr-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libexpr-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 943d90a211e6..d78c504bd8c7 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -19,27 +19,6 @@ # include #endif -/** - * @brief Allocate and initialize using self-reference - * - * This allows a brace initializer to reference the object being constructed. - * - * @warning Use with care, as the pointer points to an object that is not fully constructed yet. - * - * @tparam T Type to allocate - * @tparam F A function type for `init`, taking a T* and returning the initializer for T - * @param init Function that takes a T* and returns the initializer for T - * @return Pointer to allocated and initialized object - */ -template -static T * unsafe_new_with_self(F && init) -{ - // Allocate - void * p = ::operator new(sizeof(T), static_cast(alignof(T))); - // Initialize with placement new - return new (p) T(init(static_cast(p))); -} - extern "C" { nix_err nix_libexpr_init(nix_c_context * context) @@ -134,23 +113,20 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto if (context) context->last_err_code = NIX_OK; try { - return unsafe_new_with_self([&](auto * self) { - return nix_eval_state_builder{ - .store = nix::ref(store->ptr), - .settings = nix::EvalSettings{/* &bool */ self->readOnlyMode}, - .fetchSettings = nix::fetchers::Settings{}, - .readOnlyMode = true, - }; - }); + auto readOnly = nix::make_ref(true); + return new nix_eval_state_builder{ + .store = nix::ref(store->ptr), + .settings = nix::EvalSettings{/* &bool */ *readOnly}, + .fetchSettings = nix::fetchers::Settings{}, + .readOnlyMode = readOnly, + }; } NIXC_CATCH_ERRS_NULL } void nix_eval_state_builder_free(nix_eval_state_builder * builder) { - if (builder) - builder->~nix_eval_state_builder(); - operator delete(builder, static_cast(alignof(nix_eval_state_builder))); + delete builder; } nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder) @@ -159,7 +135,7 @@ nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_buil context->last_err_code = NIX_OK; try { // TODO: load in one go? - builder->settings.readOnlyMode = nix::settings.readOnlyMode; + builder->settings.readOnlyMode = &nix::settings.readOnlyMode; loadConfFile(builder->settings); loadConfFile(builder->fetchSettings); } diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index b38aeaf7b498..3f7e4bf1df12 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -18,8 +18,7 @@ struct nix_eval_state_builder nix::EvalSettings settings; nix::fetchers::Settings fetchSettings; nix::LookupPath lookupPath; - // TODO: make an EvalSettings setting own this instead? - bool readOnlyMode; + nix::ref readOnlyMode; }; struct EvalState diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index a874d9a0861e..cf2fa2117273 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -1,7 +1,4 @@ -#include "nix/expr/attr-set.hh" -#include "nix/util/configuration.hh" #include "nix/expr/eval.hh" -#include "nix/store/globals.hh" #include "nix/expr/value.hh" #include "nix_api_expr.h" @@ -44,6 +41,8 @@ nix_err nix_external_add_string_context(nix_c_context * context, nix_string_cont } // extern "C" +namespace { + class NixCExternalValue : public nix::ExternalValueBase { NixCExternalValueDesc & desc; @@ -169,6 +168,8 @@ class NixCExternalValue : public nix::ExternalValueBase virtual ~NixCExternalValue() override {}; }; +} // namespace + extern "C" { ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v) diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index e7a439aef196..b0f96e3504c6 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -1,8 +1,6 @@ #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" -#include "nix/util/configuration.hh" #include "nix/expr/eval.hh" -#include "nix/store/globals.hh" #include "nix/store/path.hh" #include "nix/expr/primops.hh" #include "nix/expr/value.hh" @@ -13,7 +11,6 @@ #include "nix_api_util_internal.h" #include "nix_api_store_internal.h" #include "nix_api_value.h" -#include "nix/expr/value/context.hh" // Internal helper functions to check [in] and [out] `Value *` parameters static const nix::Value & check_value_not_null(const nix_value * value) diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index ec92ecce1054..937d7d3911c4 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -8,6 +8,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -25,7 +26,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -37,6 +38,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh b/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh index 376761d76325..4aa55688def2 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh @@ -12,21 +12,22 @@ class nix_api_expr_test : public nix_api_store_test { protected: - nix_api_expr_test() + void SetUp() override { + nix_api_store_test::SetUp(); nix_libexpr_init(ctx); state = nix_state_create(nullptr, nullptr, store); value = nix_alloc_value(nullptr, state); } - ~nix_api_expr_test() + void TearDown() override { nix_gc_decref(nullptr, value); nix_state_free(state); } - EvalState * state; - nix_value * value; + EvalState * state = nullptr; + nix_value * value = nullptr; }; } // namespace nixC diff --git a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh index 2311f3941c13..ed0eba94c30d 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh @@ -4,38 +4,26 @@ #include #include "nix/expr/value/context.hh" +#include "nix/store/tests/derived-path.hh" // IWYU pragma: keep namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); -}; - -template<> -struct Arbitrary -{ - static Gen arbitrary(); -}; - -template<> -struct Arbitrary -{ - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index e8a8bf513bf0..7d6f871d7acd 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr-test-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc index 8ce84fb51f54..f82bd1c552e8 100644 --- a/src/libexpr-test-support/tests/value/context.cc +++ b/src/libexpr-test-support/tests/value/context.cc @@ -1,7 +1,6 @@ -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include -#include "nix/store/tests/path.hh" #include "nix/expr/tests/value/context.hh" namespace rc { diff --git a/src/libexpr-tests/derived-path.cc b/src/libexpr-tests/derived-path.cc index e9f9fcd0720d..3997280ee79e 100644 --- a/src/libexpr-tests/derived-path.cc +++ b/src/libexpr-tests/derived-path.cc @@ -1,6 +1,6 @@ #include #include -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" @@ -18,8 +18,6 @@ class DerivedPathExpressionTest : public LibExprTest // See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args TEST_F(DerivedPathExpressionTest, force_init) {} -#ifndef COVERAGE - RC_GTEST_FIXTURE_PROP(DerivedPathExpressionTest, prop_opaque_path_round_trip, (const SingleDerivedPath::Opaque & o)) { auto * v = state.allocValue(); @@ -61,6 +59,4 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(SingleDerivedPath{b} == d); } -#endif - } /* namespace nix */ diff --git a/src/libexpr-tests/dynamic-attrs-bench.cc b/src/libexpr-tests/dynamic-attrs-bench.cc index 1b1c199bdff7..ea0736564f59 100644 --- a/src/libexpr-tests/dynamic-attrs-bench.cc +++ b/src/libexpr-tests/dynamic-attrs-bench.cc @@ -5,7 +5,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/store/store-open.hh" -using namespace nix; +namespace nix { static std::string mkDynamicAttrsExpr(size_t attrCount) { @@ -54,3 +54,5 @@ static void BM_EvalDynamicAttrs(benchmark::State & state) } BENCHMARK(BM_EvalDynamicAttrs)->Arg(100)->Arg(500)->Arg(2'000); + +} // namespace nix diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index e722cc48499a..9f2d1f92fa3a 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -54,1270 +54,4 @@ TEST_F(ErrorTraceTest, NestedThrows) } } -#define ASSERT_TRACE1(args, type, message) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 1u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE2(args, type, message, context) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 2u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE3(args, type, message, context1, context2) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 3u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE4(args, type, message, context1, context2, context3) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 4u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context3)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -// We assume that expr starts with "builtins.derivationStrict { name =", -// otherwise the name attribute position (1, 29) would be invalid. -#define DERIVATION_TRACE_HINTFMT(name) \ - HintFmt( \ - "while evaluating derivation '%s'\n" \ - " whose name attribute is located at %s", \ - name, \ - Pos(1, 29, Pos::String{.source = make_ref(expr)})) - -// To keep things simple, we also assume that derivation name is "foo". -#define ASSERT_DERIVATION_TRACE1(args, type, message) \ - ASSERT_TRACE2(args, type, message, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE2(args, type, message, context) \ - ASSERT_TRACE3(args, type, message, context, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE3(args, type, message, context1, context2) \ - ASSERT_TRACE4(args, type, message, context1, context2, DERIVATION_TRACE_HINTFMT("foo")) - -TEST_F(ErrorTraceTest, replaceStrings) -{ - ASSERT_TRACE2( - "replaceStrings 0 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [] 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.replaceStrings")); - - ASSERT_TRACE1( - "replaceStrings [ 0 ] [] {}", - EvalError, - HintFmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); - - ASSERT_TRACE2( - "replaceStrings [ 1 ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"oo\" ] [ true ] \"foo\"", - TypeError, - HintFmt("expected a string but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"old\" ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument passed to builtins.replaceStrings")); -} - -TEST_F(ErrorTraceTest, scopedImport) {} - -TEST_F(ErrorTraceTest, import) {} - -TEST_F(ErrorTraceTest, typeOf) {} - -TEST_F(ErrorTraceTest, isNull) {} - -TEST_F(ErrorTraceTest, isFunction) {} - -TEST_F(ErrorTraceTest, isInt) {} - -TEST_F(ErrorTraceTest, isFloat) {} - -TEST_F(ErrorTraceTest, isString) {} - -TEST_F(ErrorTraceTest, isBool) {} - -TEST_F(ErrorTraceTest, isPath) {} - -TEST_F(ErrorTraceTest, break) {} - -TEST_F(ErrorTraceTest, abort) {} - -TEST_F(ErrorTraceTest, throw) {} - -TEST_F(ErrorTraceTest, addErrorContext) {} - -TEST_F(ErrorTraceTest, ceil) -{ - ASSERT_TRACE2( - "ceil \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.ceil")); -} - -TEST_F(ErrorTraceTest, floor) -{ - ASSERT_TRACE2( - "floor \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.floor")); -} - -TEST_F(ErrorTraceTest, tryEval) {} - -TEST_F(ErrorTraceTest, getEnv) -{ - ASSERT_TRACE2( - "getEnv [ ]", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getEnv")); -} - -TEST_F(ErrorTraceTest, seq) {} - -TEST_F(ErrorTraceTest, deepSeq) {} - -TEST_F(ErrorTraceTest, trace) {} - -TEST_F(ErrorTraceTest, placeholder) -{ - ASSERT_TRACE2( - "placeholder []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.placeholder")); -} - -TEST_F(ErrorTraceTest, toPath) -{ - ASSERT_TRACE2( - "toPath []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.toPath")); - - ASSERT_TRACE2( - "toPath \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the first argument passed to builtins.toPath")); -} - -TEST_F(ErrorTraceTest, storePath) -{ - ASSERT_TRACE2( - "storePath true", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.storePath'")); -} - -TEST_F(ErrorTraceTest, pathExists) -{ - ASSERT_TRACE2( - "pathExists []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while realising the context of a path")); - - ASSERT_TRACE2( - "pathExists \"zorglub\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "zorglub"), - HintFmt("while realising the context of a path")); -} - -TEST_F(ErrorTraceTest, baseNameOf) -{ - ASSERT_TRACE2( - "baseNameOf []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.baseNameOf")); -} - -TEST_F(ErrorTraceTest, dirOf) {} - -TEST_F(ErrorTraceTest, readFile) {} - -TEST_F(ErrorTraceTest, findFile) {} - -TEST_F(ErrorTraceTest, hashFile) {} - -TEST_F(ErrorTraceTest, readDir) {} - -TEST_F(ErrorTraceTest, toXML) {} - -TEST_F(ErrorTraceTest, toJSON) {} - -TEST_F(ErrorTraceTest, fromJSON) {} - -TEST_F(ErrorTraceTest, toFile) {} - -TEST_F(ErrorTraceTest, filterSource) -{ - ASSERT_TRACE2( - "filterSource [] []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] ./.", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.filterSource")); - - // Unsupported by store "dummy" - - // ASSERT_TRACE2("filterSource (_: 1) ./.", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while adding path '/home/layus/projects/nix'")); - - // ASSERT_TRACE2("filterSource (_: _: 1) ./.", - // TypeError, - // HintFmt("expected a Boolean but found %s: %s", "an integer", "1"), - // HintFmt("while evaluating the return value of the path filter function")); -} - -TEST_F(ErrorTraceTest, path) {} - -TEST_F(ErrorTraceTest, attrNames) -{ - ASSERT_TRACE2( - "attrNames []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrNames")); -} - -TEST_F(ErrorTraceTest, attrValues) -{ - ASSERT_TRACE2( - "attrValues []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrValues")); -} - -TEST_F(ErrorTraceTest, getAttr) -{ - ASSERT_TRACE2( - "getAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" {}", - TypeError, - HintFmt("attribute '%s' missing", "foo"), - HintFmt("in the attribute set under consideration")); -} - -TEST_F(ErrorTraceTest, unsafeGetAttrPos) {} - -TEST_F(ErrorTraceTest, hasAttr) -{ - ASSERT_TRACE2( - "hasAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.hasAttr")); - - ASSERT_TRACE2( - "hasAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.hasAttr")); -} - -TEST_F(ErrorTraceTest, isAttrs) {} - -TEST_F(ErrorTraceTest, removeAttrs) -{ - ASSERT_TRACE2( - "removeAttrs \"\" \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ \"1\" ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); -} - -TEST_F(ErrorTraceTest, listToAttrs) -{ - ASSERT_TRACE2( - "listToAttrs 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the argument passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ {} ]", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in a {name=...; value=...;} pair")); - - ASSERT_TRACE2( - "listToAttrs [ { name = 1; } ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ { name = \"foo\"; } ]", - TypeError, - HintFmt("attribute '%s' missing", "value"), - HintFmt("in a {name=...; value=...;} pair")); -} - -TEST_F(ErrorTraceTest, intersectAttrs) -{ - ASSERT_TRACE2( - "intersectAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.intersectAttrs")); - - ASSERT_TRACE2( - "intersectAttrs {} []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.intersectAttrs")); -} - -TEST_F(ErrorTraceTest, catAttrs) -{ - ASSERT_TRACE2( - "catAttrs [] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ { foo = 1; } 1 { bar = 5;} ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); -} - -TEST_F(ErrorTraceTest, functionArgs) -{ - ASSERT_TRACE1("functionArgs {}", TypeError, HintFmt("'functionArgs' requires a function")); -} - -TEST_F(ErrorTraceTest, mapAttrs) -{ - ASSERT_TRACE2( - "mapAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.mapAttrs")); - - // XXX: deferred - // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: x + \"1\") { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: y: x + 1) { foo.bar = 1; }", - // TypeError, - // HintFmt("cannot coerce %s to a string", "an integer"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, zipAttrsWith) -{ - ASSERT_TRACE2( - "zipAttrsWith [] [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.zipAttrsWith")); - - ASSERT_TRACE2( - "zipAttrsWith (_: 1) [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); - - // XXX: How to properly tell that the function takes two arguments ? - // The same question also applies to sort, and maybe others. - // Due to laziness, we only create a thunk, and it fails later on. - // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while evaluating the attribute 'foo'")); - - // XXX: Also deferred deeply - // ASSERT_TRACE2("zipAttrsWith (a: b: a + b) [ { foo = 1; } { foo = 2; } ]", - // TypeError, - // HintFmt("cannot coerce %s to a string", "a list"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, isList) {} - -TEST_F(ErrorTraceTest, elemAt) -{ - ASSERT_TRACE2( - "elemAt \"foo\" (-1)", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.elemAt'")); - - ASSERT_TRACE1( - "elemAt [] (-1)", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", -1, 0)); - - ASSERT_TRACE1( - "elemAt [\"foo\"] 3", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", 3, 1)); -} - -TEST_F(ErrorTraceTest, head) -{ - ASSERT_TRACE2( - "head 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.head'")); - - ASSERT_TRACE1("head []", Error, HintFmt("'builtins.head' called on an empty list")); -} - -TEST_F(ErrorTraceTest, tail) -{ - ASSERT_TRACE2( - "tail 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.tail'")); - - ASSERT_TRACE1("tail []", Error, HintFmt("'builtins.tail' called on an empty list")); -} - -TEST_F(ErrorTraceTest, map) -{ - ASSERT_TRACE2( - "map 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.map")); - - ASSERT_TRACE2( - "map 1 [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.map")); -} - -TEST_F(ErrorTraceTest, filter) -{ - ASSERT_TRACE2( - "filter 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter (_: 5) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "5" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the filtering function passed to builtins.filter")); -} - -TEST_F(ErrorTraceTest, elem) -{ - ASSERT_TRACE2( - "elem 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.elem")); -} - -TEST_F(ErrorTraceTest, concatLists) -{ - ASSERT_TRACE2( - "concatLists 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ 1 ]", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ [1] \"foo\" ]", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); -} - -TEST_F(ErrorTraceTest, length) -{ - ASSERT_TRACE2( - "length 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); - - ASSERT_TRACE2( - "length \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); -} - -TEST_F(ErrorTraceTest, foldlPrime) -{ - ASSERT_TRACE2( - "foldl' 1 \"foo\" true", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.foldlStrict")); - - ASSERT_TRACE2( - "foldl' (_: 1) \"foo\" true", - TypeError, - HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the third argument passed to builtins.foldlStrict")); - - ASSERT_TRACE1( - "foldl' (_: 1) \"foo\" [ true ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "foldl' (a: b: a && b) \"foo\" [ true ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("in the left operand of the AND (&&) operator")); -} - -TEST_F(ErrorTraceTest, any) -{ - ASSERT_TRACE2( - "any 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.any")); -} - -TEST_F(ErrorTraceTest, all) -{ - ASSERT_TRACE2( - "all 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.all")); -} - -TEST_F(ErrorTraceTest, genList) -{ - ASSERT_TRACE2( - "genList 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.genList")); - - ASSERT_TRACE2( - "genList 1 2", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.genList")); - - // XXX: deferred - // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO", - // TypeError, - // HintFmt("cannot add %s to an integer", "a string"), - // HintFmt("while evaluating anonymous lambda")); - - ASSERT_TRACE1("genList false (-3)", EvalError, HintFmt("cannot create list of size %d", -3)); -} - -TEST_F(ErrorTraceTest, sort) -{ - ASSERT_TRACE2( - "sort 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.sort")); - - ASSERT_TRACE2( - "sort 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.sort")); - - ASSERT_TRACE1( - "sort (_: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "sort (_: _: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the sorting function passed to builtins.sort")); - - // XXX: Trace too deep, need better asserts - // ASSERT_TRACE1("sort (a: b: a <= b) [ \"foo\" {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s", "a string", "a set")); - - // ASSERT_TRACE1("sort (a: b: a <= b) [ {} {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); -} - -TEST_F(ErrorTraceTest, partition) -{ - ASSERT_TRACE2( - "partition 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the partition function passed to builtins.partition")); -} - -TEST_F(ErrorTraceTest, groupBy) -{ - ASSERT_TRACE2( - "groupBy 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (x: x) [ \"foo\" \"bar\" 1 ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); -} - -TEST_F(ErrorTraceTest, concatMap) -{ - ASSERT_TRACE2( - "concatMap 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) [ \"foo\" ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: \"foo\") [ 1 2 ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); -} - -TEST_F(ErrorTraceTest, add) -{ - ASSERT_TRACE2( - "add \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the addition")); - - ASSERT_TRACE2( - "add 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the addition")); -} - -TEST_F(ErrorTraceTest, sub) -{ - ASSERT_TRACE2( - "sub \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the subtraction")); - - ASSERT_TRACE2( - "sub 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the subtraction")); -} - -TEST_F(ErrorTraceTest, mul) -{ - ASSERT_TRACE2( - "mul \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the multiplication")); - - ASSERT_TRACE2( - "mul 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the multiplication")); -} - -TEST_F(ErrorTraceTest, div) -{ - ASSERT_TRACE2( - "div \"foo\" 1 # TODO: an integer was expected -> a number", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first operand of the division")); - - ASSERT_TRACE2( - "div 1 \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second operand of the division")); - - ASSERT_TRACE1("div \"foo\" 0", EvalError, HintFmt("division by zero")); -} - -TEST_F(ErrorTraceTest, bitAnd) -{ - ASSERT_TRACE2( - "bitAnd 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitAnd")); - - ASSERT_TRACE2( - "bitAnd 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitAnd")); -} - -TEST_F(ErrorTraceTest, bitOr) -{ - ASSERT_TRACE2( - "bitOr 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitOr")); - - ASSERT_TRACE2( - "bitOr 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitOr")); -} - -TEST_F(ErrorTraceTest, bitXor) -{ - ASSERT_TRACE2( - "bitXor 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitXor")); - - ASSERT_TRACE2( - "bitXor 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitXor")); -} - -TEST_F(ErrorTraceTest, lessThan) -{ - ASSERT_TRACE1( - "lessThan 1 \"foo\"", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL))); - - ASSERT_TRACE1( - "lessThan {} {}", - EvalError, - HintFmt( - "cannot compare %s with %s; values of that type are incomparable (values are %s and %s)", - "a set", - "a set", - Uncolored("{ }"), - Uncolored("{ }"))); - - ASSERT_TRACE2( - "lessThan [ 1 2 ] [ \"foo\" ]", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while comparing two list elements")); -} - -TEST_F(ErrorTraceTest, toString) -{ - ASSERT_TRACE2( - "toString { a = 1; }", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ a = " ANSI_CYAN "1" ANSI_NORMAL "; }")), - HintFmt("while evaluating the first argument passed to builtins.toString")); -} - -TEST_F(ErrorTraceTest, substring) -{ - ASSERT_TRACE2( - "substring {} \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the first argument (the start offset) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 3 \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument (the substring length) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 0 3 {}", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument (the string) passed to builtins.substring")); - - ASSERT_TRACE1("substring (-3) 3 \"sometext\"", EvalError, HintFmt("negative start position in 'substring'")); -} - -TEST_F(ErrorTraceTest, stringLength) -{ - ASSERT_TRACE2( - "stringLength {} # TODO: context is missing ???", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the argument passed to builtins.stringLength")); -} - -TEST_F(ErrorTraceTest, hashString) -{ - ASSERT_TRACE2( - "hashString 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.hashString")); - - ASSERT_TRACE1( - "hashString \"foo\" \"content\"", - UsageError, - HintFmt("unknown hash algorithm '%s', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); - - ASSERT_TRACE2( - "hashString \"sha256\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.hashString")); -} - -TEST_F(ErrorTraceTest, match) -{ - ASSERT_TRACE2( - "match 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.match")); - - ASSERT_TRACE2( - "match \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.match")); - - ASSERT_TRACE1("match \"(.*\" \"\"", EvalError, HintFmt("invalid regular expression '%s'", "(.*")); -} - -TEST_F(ErrorTraceTest, split) -{ - ASSERT_TRACE2( - "split 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.split")); - - ASSERT_TRACE2( - "split \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.split")); - - ASSERT_TRACE1("split \"f(o*o\" \"1foo2\"", EvalError, HintFmt("invalid regular expression '%s'", "f(o*o")); -} - -TEST_F(ErrorTraceTest, concatStringsSep) -{ - ASSERT_TRACE2( - "concatStringsSep 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt( - "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); -} - -TEST_F(ErrorTraceTest, parseDrvName) -{ - ASSERT_TRACE2( - "parseDrvName 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.parseDrvName")); -} - -TEST_F(ErrorTraceTest, compareVersions) -{ - ASSERT_TRACE2( - "compareVersions 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.compareVersions")); - - ASSERT_TRACE2( - "compareVersions \"abd\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.compareVersions")); -} - -TEST_F(ErrorTraceTest, splitVersion) -{ - ASSERT_TRACE2( - "splitVersion 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.splitVersion")); -} - -TEST_F(ErrorTraceTest, traceVerbose) {} - -TEST_F(ErrorTraceTest, derivationStrict) -{ - ASSERT_TRACE2( - "derivationStrict \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", "\"\""), - HintFmt("while evaluating the argument passed to builtins.derivationStrict")); - - ASSERT_TRACE2( - "derivationStrict {}", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in the attrset passed as argument to builtins.derivationStrict")); - - ASSERT_TRACE3( - "derivationStrict { name = 1; }", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute passed to builtins.derivationStrict"), - HintFmt("while evaluating the derivation attribute 'name'")); - - ASSERT_DERIVATION_TRACE1( - "derivationStrict { name = \"foo\"; }", EvalError, HintFmt("required attribute 'builder' missing")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __structuredAttrs = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __ignoreNulls = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = 15; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "15"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = \"custom\"; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "custom"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "system", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drvPath\"; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; outputs = \"out\"; __structuredAttrs = true; }", - EvalError, - HintFmt("expected a list but found %s: %s", "a string", "\"out\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = []; }", - EvalError, - HintFmt("derivation cannot have an empty set of outputs"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"drvPath\" ]; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"out\" \"out\" ]; }", - EvalError, - HintFmt("duplicate derivation output '%s'", "out"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __contentAddressed = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__contentAddressed", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = \"foo\"; }", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", "\"foo\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "FOO", "foo")); -} - } /* namespace nix */ diff --git a/src/libexpr-tests/eval.cc b/src/libexpr-tests/eval.cc index 7562a9da21ad..985564f13e7d 100644 --- a/src/libexpr-tests/eval.cc +++ b/src/libexpr-tests/eval.cc @@ -3,7 +3,6 @@ #include "nix/expr/eval.hh" #include "nix/expr/tests/libexpr.hh" -#include "nix/util/memory-source-accessor.hh" namespace nix { diff --git a/src/libexpr-tests/get-drvs-bench.cc b/src/libexpr-tests/get-drvs-bench.cc index a5cd59154f22..9241e480480b 100644 --- a/src/libexpr-tests/get-drvs-bench.cc +++ b/src/libexpr-tests/get-drvs-bench.cc @@ -6,8 +6,7 @@ #include "nix/store/store-open.hh" #include "nix/util/fmt.hh" -using namespace nix; - +namespace nix { namespace { struct GetDerivationsEnv @@ -64,3 +63,5 @@ static void BM_GetDerivationsAttrScan(benchmark::State & state) } BENCHMARK(BM_GetDerivationsAttrScan)->Arg(1'000)->Arg(5'000)->Arg(10'000); + +} // namespace nix diff --git a/src/libexpr-tests/lazy-fetcher-attr.cc b/src/libexpr-tests/lazy-fetcher-attr.cc new file mode 100644 index 000000000000..4c36424ecb55 --- /dev/null +++ b/src/libexpr-tests/lazy-fetcher-attr.cc @@ -0,0 +1,100 @@ +#include + +#include "nix/expr/fetch-tree.hh" +#include "nix/expr/tests/libexpr.hh" +#include "nix/fetchers/attrs.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/store/path.hh" + +namespace nix { + +class LazyFetcherAttrTest : public LibExprTest +{ +protected: + StorePath dummyPath() + { + return StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-test"}; + } +}; + +TEST_F(LazyFetcherAttrTest, nonLazyAttrProducesImmediateValue) +{ + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("revCount", uint64_t(5)); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 5); +} + +TEST_F(LazyFetcherAttrTest, lazyAttrProducesThunk) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(42); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + + // Not yet forced, so the lazy function should not have been called + EXPECT_EQ(calls, 0); + + // Force the thunk + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 42); + EXPECT_EQ(calls, 1); +} + +TEST_F(LazyFetcherAttrTest, lazyFunctionOnlyCalledOnAccess) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("lastModified", uint64_t(1000)); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(99); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + // Access lastModified, so should not trigger lazy revCount + auto * lmAttr = v.attrs()->get(state.symbols.create("lastModified")); + ASSERT_NE(lmAttr, nullptr); + state.forceValue(*lmAttr->value, noPos); + EXPECT_EQ(lmAttr->value->integer().value, 1000); + EXPECT_EQ(calls, 0); + + // Now access revCount + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 99); + EXPECT_EQ(calls, 1); +} + +} // namespace nix diff --git a/src/libexpr-tests/main.cc b/src/libexpr-tests/main.cc index 88a9d6684d5b..5a717d6bb053 100644 --- a/src/libexpr-tests/main.cc +++ b/src/libexpr-tests/main.cc @@ -1,18 +1,16 @@ #include #include "nix/store/tests/test-main.hh" -#include "nix/util/config-global.hh" - -using namespace nix; +#include "nix/util/configuration.hh" int main(int argc, char ** argv) { - auto res = testMainForBuidingPre(argc, argv); + auto res = nix::testMainForBuidingPre(argc, argv); if (res) return res; // For pipe operator tests in trivial.cc - experimentalFeatureSettings.set("experimental-features", "pipe-operators"); + nix::experimentalFeatureSettings.set("experimental-features", "pipe-operators"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index c5b72851da53..50d158209ba7 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -51,6 +51,7 @@ sources = files( 'error_traces.cc', 'eval.cc', 'json.cc', + 'lazy-fetcher-attr.cc', 'main.cc', 'nix_api_expr.cc', 'nix_api_external.cc', @@ -84,6 +85,8 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index 27c7d69ef07e..d9b203128553 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -20,8 +20,8 @@ TEST_F(nix_api_expr_test, nix_eval_state_lookup_path) auto delTmpDir = std::make_unique(tmpDir, true); auto nixpkgs = tmpDir / "pkgs"; auto nixos = tmpDir / "cfg"; - std::filesystem::create_directories(nixpkgs); - std::filesystem::create_directories(nixos); + nix::createDirs(nixpkgs); + nix::createDirs(nixos); std::string nixpkgsEntry = "nixpkgs=" + nixpkgs.string(); std::string nixosEntry = "nixos-config=" + nixos.string(); diff --git a/src/libexpr-tests/nix_api_external.cc b/src/libexpr-tests/nix_api_external.cc index 885b9c1d2dd3..e17a52c31dfc 100644 --- a/src/libexpr-tests/nix_api_external.cc +++ b/src/libexpr-tests/nix_api_external.cc @@ -1,5 +1,3 @@ -#include "nix_api_store.h" -#include "nix_api_util.h" #include "nix_api_expr.h" #include "nix_api_value.h" #include "nix_api_external.h" diff --git a/src/libexpr-tests/nix_api_value.cc b/src/libexpr-tests/nix_api_value.cc index 830637f3ec50..01d15744a750 100644 --- a/src/libexpr-tests/nix_api_value.cc +++ b/src/libexpr-tests/nix_api_value.cc @@ -1,4 +1,3 @@ -#include "nix_api_store.h" #include "nix_api_util.h" #include "nix_api_expr.h" #include "nix_api_value.h" diff --git a/src/libexpr-tests/primops.cc b/src/libexpr-tests/primops.cc index 2261c6cb3d60..03014903fe0b 100644 --- a/src/libexpr-tests/primops.cc +++ b/src/libexpr-tests/primops.cc @@ -637,7 +637,7 @@ TEST_F(PrimOpTest, toStringLambdaThrows) } class ToStringPrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(ToStringPrimOpTest, toString) @@ -651,7 +651,7 @@ TEST_P(ToStringPrimOpTest, toString) INSTANTIATE_TEST_SUITE_P( toString, ToStringPrimOpTest, - testing::Values( + ::testing::Values( CASE(R"("foo")", "foo"), CASE(R"(1)", "1"), CASE(R"([1 2 3])", "1 2 3"), @@ -799,7 +799,7 @@ TEST_F(PrimOpTest, splitVersion) } class CompareVersionsPrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(CompareVersionsPrimOpTest, compareVersions) @@ -813,7 +813,7 @@ TEST_P(CompareVersionsPrimOpTest, compareVersions) INSTANTIATE_TEST_SUITE_P( compareVersions, CompareVersionsPrimOpTest, - testing::Values( + ::testing::Values( // The first two are weird cases. Intuition tells they should // be the same but they aren't. CASE(1.0, 1.0.0, -1), @@ -835,7 +835,7 @@ INSTANTIATE_TEST_SUITE_P( class ParseDrvNamePrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(ParseDrvNamePrimOpTest, parseDrvName) @@ -857,7 +857,7 @@ TEST_P(ParseDrvNamePrimOpTest, parseDrvName) INSTANTIATE_TEST_SUITE_P( parseDrvName, ParseDrvNamePrimOpTest, - testing::Values( + ::testing::Values( std::make_tuple("nix-0.12pre12876", "nix", "0.12pre12876"), std::make_tuple("a-b-c-1234pre5+git", "a-b-c", "1234pre5+git"))); diff --git a/src/libexpr-tests/regex-cache-bench.cc b/src/libexpr-tests/regex-cache-bench.cc index 2eb17b212ab0..c0c61e58e993 100644 --- a/src/libexpr-tests/regex-cache-bench.cc +++ b/src/libexpr-tests/regex-cache-bench.cc @@ -5,7 +5,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/store/store-open.hh" -using namespace nix; +namespace nix { static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state) { @@ -44,3 +44,5 @@ static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state) } BENCHMARK(BM_EvalManyBuiltinsMatchSameRegex); + +} // namespace nix diff --git a/src/libexpr-tests/value/context.cc b/src/libexpr-tests/value/context.cc index fe3072b64ffd..c2c5cd3e8bbc 100644 --- a/src/libexpr-tests/value/context.cc +++ b/src/libexpr-tests/value/context.cc @@ -2,9 +2,8 @@ #include #include -#include "nix/store/tests/path.hh" -#include "nix/expr/tests/libexpr.hh" #include "nix/expr/tests/value/context.hh" +#include "nix/store/store-dir-config.hh" namespace nix { @@ -121,8 +120,6 @@ TEST(NixStringContextElemTest, built_built_xp) NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); } -#ifndef COVERAGE - RC_GTEST_PROP(NixStringContextElemTest, prop_round_rip, (const NixStringContextElem & o)) { ExperimentalFeatureSettings xpSettings; @@ -130,6 +127,4 @@ RC_GTEST_PROP(NixStringContextElemTest, prop_round_rip, (const NixStringContextE RC_ASSERT(o == NixStringContextElem::parse(o.to_string(), xpSettings)); } -#endif - } // namespace nix diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 63197e4634fd..7ba510e43f7e 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -5,6 +5,10 @@ namespace nix { +void AttrPathNotFound::anchor() {} + +void NoPositionInfo::anchor() {} + static Strings parseAttrPath(std::string_view s) { Strings res; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index b775acf5e98e..d2f041a06bf8 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -10,6 +10,8 @@ namespace nix::eval_cache { +void CachedEvalError::anchor() {} + CachedEvalError::CachedEvalError(ref cursor, Symbol attr) : CloneableError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr)) , cursor(cursor) @@ -106,7 +108,7 @@ struct AttrDb } template - AttrId doSQLite(F && fun) + AttrId doSQLite(const F & fun) { if (failed) return 0; @@ -779,14 +781,17 @@ StorePath AttrCursor::forceDerivation() auto aDrvPath = getAttr(root->state.s.drvPath); auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); drvPath.requireDerivation(); - if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { - /* The eval cache contains 'drvPath', but the actual path has - been garbage-collected. So force it to be regenerated. */ - aDrvPath->forceValue(); - root->state.waitForPath(drvPath); - if (!root->state.store->isValidPath(drvPath)) - throw Error( - "don't know how to recreate store derivation '%s'!", root->state.store->printStorePath(drvPath)); + if (!settings.readOnlyMode) { + root->state.store->addTempRoot(drvPath); + if (!root->state.store->isValidPath(drvPath)) { + /* The eval cache contains 'drvPath', but the actual path has + been garbage-collected. So force it to be regenerated. */ + aDrvPath->forceValue(); + root->state.waitForPath(drvPath); + if (!root->state.store->isValidPath(drvPath)) + throw Error( + "don't know how to recreate store derivation '%s'!", root->state.store->printStorePath(drvPath)); + } } return drvPath; } diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 38a60883bca7..ba72b05e98a6 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -6,7 +6,7 @@ namespace nix { InvalidPathError::InvalidPathError(EvalState & state, const StorePath & path) - : CloneableError(state, "path '%s' is not valid", path.to_string()) + : CloneableError(state, "path '%s' is not valid", state.store->printStorePath(path)) , path{path} { } @@ -97,7 +97,7 @@ void EvalErrorBuilder::debugThrow() auto error = std::move(this->error); delete this; - throw error; + throw std::move(error); } template @@ -123,4 +123,32 @@ template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; +void EvalBaseError::anchor() {} + +void ParseError::anchor() {} + +void EvalError::anchor() {} + +void AssertionError::anchor() {} + +void ThrownError::anchor() {} + +void Abort::anchor() {} + +void TypeError::anchor() {} + +void UndefinedVarError::anchor() {} + +void MissingArgumentError::anchor() {} + +void InfiniteRecursionError::anchor() {} + +void StackOverflowError::anchor() {} + +void InvalidPathError::anchor() {} + +void IFDError::anchor() {} + +void RecoverableEvalError::anchor() {} + } // namespace nix diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index c1e974e053b8..1aef8b844d4f 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -1,8 +1,6 @@ -#include "nix/util/error.hh" #include "nix/util/environment-variables.hh" #include "nix/expr/eval-settings.hh" #include "nix/util/config-global.hh" -#include "nix/util/serialise.hh" #include "nix/expr/eval-gc.hh" #include "nix/expr/value.hh" @@ -161,6 +159,21 @@ static inline void initGCReal() GC_set_sp_corrector(&fixupBoehmStackPointer); assert(GC_get_sp_corrector()); + /* Funnel boehm warnings into debug logs. */ + GC_set_warn_proc([](char * msg, GC_word word) noexcept { + std::array buffer{}; + auto res = snprintf(buffer.data(), buffer.size(), msg, word); + /* Ignore garbage. */ + if (res < 0) + return; + + try { + debug("%s", chomp(std::string_view(buffer.data(), std::min(res, buffer.size() - 1)))); + } catch (...) { + /* Swallow all errors. */ + } + }); + /* Set the initial heap size to something fairly big (80% of free RAM, up to a maximum of 4 GiB) so that in most cases we don't need to garbage collect at all. (Collection has a diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index daf8dbc452b3..630f7b67067f 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -7,6 +7,8 @@ namespace nix { +void EvalSettings::anchor() {} + void DeprecatedWarnSetting::assign(const bool & v) { value = v; @@ -70,7 +72,7 @@ Strings EvalSettings::parseNixPath(const std::string & s) } EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks) - : readOnlyMode{readOnlyMode} + : readOnlyMode{&readOnlyMode} , lookupPathHooks{lookupPathHooks} { auto var = getEnv("NIX_ABORT_ON_WARN"); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index befec174aa43..0023c52ca7fd 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -6,6 +6,7 @@ #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" #include "nix/util/exit.hh" +#include "nix/util/signals.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" #include "nix/util/environment-variables.hh" @@ -44,6 +45,7 @@ #include #include #include +#include #include #include @@ -298,6 +300,9 @@ EvalState::EvalState( ? storeFS.cast() : makeUnionSourceAccessor({getFSSourceAccessor(), storeFS}, storeFS.cast()); + /* Cache positive lstat/readlink results to speed up resolveSymlinks. */ + accessor = makeCachingSourceAccessor(accessor); + /* Apply access control if needed. */ if (settings.restrictEval || settings.pureEval) accessor = AllowListSourceAccessor::create( @@ -344,6 +349,17 @@ EvalState::EvalState( , attrSelects(make_ref()) , executor{make_ref(settings)} { +#ifndef _WIN32 + static std::once_flag stackSizeBumped; + std::call_once(stackSizeBumped, []() { + // Increase the default stack size for the evaluator and for + // libstdc++'s std::regex. + // This used to be 64 MiB, but macOS as deployed on GitHub Actions has a + // hard limit slightly under that, so we round it down a bit. + nix::ensureStackSizeAtLeast(60 * 1024 * 1024); + }); +#endif + corepkgsFS->setPathDisplay(""); internalFS->setPathDisplay("«nix-internal»", ""); @@ -680,27 +696,34 @@ std::optional EvalState::getDoc(Value & v) return {}; } +static StaticEnv::Vars lexicographicOrder(const SymbolTable & st, StaticEnv::Vars vars) +{ + std::ranges::sort(vars, [&st](const auto & lhs, const auto & rhs) { + return std::string_view(st[lhs.first]) < std::string_view(st[rhs.first]); + }); + return vars; +} + // just for the current level of StaticEnv, not the whole chain. -void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) +static void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) { std::cout << ANSI_MAGENTA; - for (auto & i : se.vars) - std::cout << st[i.first] << " "; + for (auto & [name, displacement] : lexicographicOrder(st, se.vars)) + std::cout << st[name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } // just for the current level of Env, not the whole chain. -void printWithBindings(const SymbolTable & st, const Env & env) +static void printWithBindings(const SymbolTable & st, const Env & env) { if (env.values[0]->isFinished()) { std::cout << "with: "; std::cout << ANSI_MAGENTA; - auto j = env.values[0]->attrs()->begin(); - while (j != env.values[0]->attrs()->end()) { - std::cout << st[j->name] << " "; - ++j; - } + auto * bindings = env.values[0]->attrs(); + /* TODO: Don't print the whole attribute set, since it can be quite large. */ + for (const Attr * attr : bindings->lexicographicOrder(st)) + std::cout << st[attr->name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } @@ -721,7 +744,7 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::cout << ANSI_MAGENTA; // for the top level, don't print the double underscore ones; // they are in builtins. - for (auto & i : se.vars) + for (auto & i : lexicographicOrder(st, se.vars)) if (!hasPrefix(st[i.first], "__")) std::cout << st[i.first] << " "; std::cout << ANSI_NORMAL; @@ -1113,6 +1136,8 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) return &v; } +namespace { + /** * A helper `Expr` class to lets us parse and evaluate Nix expressions * from a thunk, ensuring that every file is parsed/evaluated only @@ -1155,6 +1180,8 @@ struct ExprParseFile : Expr } }; +} // namespace + void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) { auto resolvedPath = getConcurrent(*importResolutionCache, path); @@ -1197,7 +1224,8 @@ void EvalState::resetFileCache() importResolutionCache->clear(); fileEvalCache->clear(); inputCache->clear(); - positions.clear(); + lookupPathResolved->clear(); + rootFS->invalidateCache(); } void EvalState::eval(Expr * e, Value & v) @@ -2055,22 +2083,21 @@ void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) Value v2; e2->eval(state, env, v2); Value * lists[2] = {&v1, &v2}; - state.concatLists(v, 2, lists, pos, "while evaluating one of the elements to concatenate"); + state.concatLists(v, lists, pos, "while evaluating one of the elements to concatenate"); } -void EvalState::concatLists( - Value & v, size_t nrLists, Value * const * lists, const PosIdx pos, std::string_view errorCtx) +void EvalState::concatLists(Value & v, std::span lists, const PosIdx pos, std::string_view errorCtx) { nrListConcats++; - Value * nonEmpty = 0; + Value * nonEmpty = nullptr; size_t len = 0; - for (size_t n = 0; n < nrLists; ++n) { - forceList(*lists[n], pos, errorCtx); - auto l = lists[n]->listSize(); + for (auto * list : lists) { + forceList(*list, pos, errorCtx); + auto l = list->listSize(); len += l; if (l) - nonEmpty = lists[n]; + nonEmpty = list; } if (nonEmpty && len == nonEmpty->listSize()) { @@ -2080,12 +2107,13 @@ void EvalState::concatLists( auto list = buildList(len); auto out = list.elems; - for (size_t n = 0, pos = 0; n < nrLists; ++n) { - auto listView = lists[n]->listView(); - auto l = listView.size(); - if (l) - memcpy(out + pos, listView.data(), l * sizeof(Value *)); - pos += l; + size_t pos2 = 0; + for (auto * l : lists) { + auto listView = l->listView(); + auto n = listView.size(); + if (n) + memcpy(out + pos2, listView.data(), n * sizeof(Value *)); + pos2 += n; } v.mkList(list); } @@ -2543,7 +2571,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat fetchSettings, *store, path.resolveSymlinks(SymlinkResolution::Ancestors), - settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + settings.isReadOnly() ? FetchMode::DryRun : FetchMode::Copy, computeBaseName(path, pos), ContentAddressMethod::Raw::NixArchive, nullptr, @@ -2583,7 +2611,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); - return rootPath(path); + return rootPath(CanonPath(path)); } StorePath @@ -3237,14 +3265,28 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ continue; auto r = *rOpt; - auto res = (r / CanonPath(suffix)).resolveSymlinks(); - if (res.pathExists()) + auto suffixPath = CanonPath(suffix); + if (auto cachedRes = getConcurrent(*rOpt->resolvedPaths, suffixPath)) { + if (*cachedRes) + return **cachedRes; + else + // Cached negative lookup. + continue; + } + + auto res = (r.path / suffixPath).resolveSymlinks(); + if (res.pathExists()) { + r.resolvedPaths->emplace(suffixPath, res); return res; + } // Backward compatibility hack: throw an exception if access // to this path is not allowed. if (auto accessor = res.accessor.dynamic_pointer_cast()) accessor->checkAccess(res.path); + + // Cache negative lookups too. + r.resolvedPaths->emplace(suffixPath, std::nullopt); } if (hasPrefix(path, "nix/")) @@ -3258,17 +3300,22 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ .debugThrow(); } -std::optional EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) +std::shared_ptr +EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) { auto & value = value0.s; if (auto cached = getConcurrent(*lookupPathResolved, value)) return *cached; - auto finish = [&](std::optional res) { - if (res) - debug("resolved search path element '%s' to '%s'", value, *res); - else + auto finish = [&](std::optional maybePath) { + std::shared_ptr res; + if (maybePath) { + debug("resolved search path element '%s' to '%s'", value, *maybePath); + res = std::make_shared( + *maybePath, make_ref()); + } else { debug("failed to resolve search path element '%s'", value); + } lookupPathResolved->emplace(std::string(value), res); return res; }; @@ -3413,7 +3460,7 @@ void forceNoNullByte(std::string_view s, std::function pos) if (pos) { error.atPos(pos()); } - throw error; + throw std::move(error); } } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 03a1aa455ce0..1acc591b05b7 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -5,7 +5,6 @@ #include "nix/store/path-with-outputs.hh" #include -#include namespace nix { @@ -163,14 +162,14 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) - throw errMsg; + throw std::move(errMsg); Outputs result; for (auto elem : outTI->listView()) { if (elem->type() != nString) - throw errMsg; + throw std::move(errMsg); auto out = outputs.find(elem->string_view()); if (out == outputs.end()) - throw errMsg; + throw std::move(errMsg); result.insert(*out); } return result; diff --git a/src/libexpr/include/nix/expr/diagnose.hh b/src/libexpr/include/nix/expr/diagnose.hh index 8dfe052134a1..4a360970ba21 100644 --- a/src/libexpr/include/nix/expr/diagnose.hh +++ b/src/libexpr/include/nix/expr/diagnose.hh @@ -28,11 +28,7 @@ enum struct Diagnose { Fatal, }; -template<> -Diagnose BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(Diagnose) /** * Check a diagnostic setting and either do nothing, log a warning, or throw an error. @@ -48,7 +44,7 @@ std::string BaseSetting::to_string() const; * @throws The error returned by mkError if level is `Fatal` and mkError returns a value */ template -void diagnose(const Setting & setting, F && mkError) +void diagnose(const Setting & setting, const F & mkError) { auto withError = [&](bool fatal, auto && handler) { auto maybeError = mkError(fatal); @@ -68,7 +64,7 @@ void diagnose(const Setting & setting, F && mkError) withError(false, [](auto && error) { logWarning(error.info()); }); return; case Diagnose::Fatal: - withError(true, [](auto && error) { throw std::move(error); }); + withError(true, [](auto && error) { throw std::forward(error); }); return; } } diff --git a/src/libexpr/include/nix/expr/eval-cache.hh b/src/libexpr/include/nix/expr/eval-cache.hh index 9f1657a57047..df237d704b23 100644 --- a/src/libexpr/include/nix/expr/eval-cache.hh +++ b/src/libexpr/include/nix/expr/eval-cache.hh @@ -17,6 +17,9 @@ class AttrCursor; struct CachedEvalError : CloneableError { +private: + void anchor() override; +public: const ref cursor; const Symbol attr; diff --git a/src/libexpr/include/nix/expr/eval-error.hh b/src/libexpr/include/nix/expr/eval-error.hh index 68aa7b0643a2..8dce750e073c 100644 --- a/src/libexpr/include/nix/expr/eval-error.hh +++ b/src/libexpr/include/nix/expr/eval-error.hh @@ -23,6 +23,9 @@ class EvalBaseError : public CloneableError { template friend class EvalErrorBuilder; + + void anchor() override; + public: EvalState & state; @@ -61,8 +64,11 @@ MakeError(InfiniteRecursionError, EvalError); * Inherits from EvalBaseError (not EvalError) because resource exhaustion * should not be cached. */ -struct StackOverflowError : public CloneableError +class StackOverflowError : public CloneableError { + void anchor() override; + +public: StackOverflowError(EvalState & state) : CloneableError(state, "stack overflow; max-call-depth exceeded") { @@ -79,8 +85,10 @@ MakeError(IFDError, EvalBaseError); */ MakeError(RecoverableEvalError, EvalBaseError); -struct InvalidPathError : public CloneableError +class InvalidPathError : public CloneableError { + void anchor() override; + public: StorePath path; diff --git a/src/libexpr/include/nix/expr/eval-profiler-settings.hh b/src/libexpr/include/nix/expr/eval-profiler-settings.hh index 32138e7f13f0..4fec151b8061 100644 --- a/src/libexpr/include/nix/expr/eval-profiler-settings.hh +++ b/src/libexpr/include/nix/expr/eval-profiler-settings.hh @@ -7,10 +7,6 @@ namespace nix { enum struct EvalProfilerMode { disabled, flamegraph }; -template<> -EvalProfilerMode BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(EvalProfilerMode) } // namespace nix diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index b58f0cd45943..76fe402e11c6 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -43,6 +43,10 @@ public: struct EvalSettings : Config { +private: + void anchor() override; + +public: /** * Function used to interpret look path entries of a given scheme. * @@ -71,7 +75,14 @@ struct EvalSettings : Config EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {}); - bool & readOnlyMode; + /* FIXME: This really shouldn't be public. The C API should have non-global settings instead. */ + bool * readOnlyMode = nullptr; + + bool isReadOnly() const + { + assert(readOnlyMode); + return *readOnlyMode; + } static Strings getDefaultNixPath(); diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index b5261f6450d5..b0ae6c6ecb0f 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -28,6 +28,7 @@ #include #include #include +#include namespace nix { @@ -482,7 +483,15 @@ private: LookupPath lookupPath; - const ref, StringViewHash, std::equal_to<>>> + struct LookupPathResolvedState + { + SourcePath path; + const ref>> resolvedPaths; + }; + + const ref< + boost:: + concurrent_flat_map, StringViewHash, std::equal_to<>>> lookupPathResolved; /** @@ -624,9 +633,10 @@ public: * * If the specified search path element is a URI, download it. * - * If it is not found, return `std::nullopt`. + * If it is not found, return `nullptr`. */ - std::optional resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); + std::shared_ptr + resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); /** * Evaluate an expression to normal form @@ -994,7 +1004,10 @@ public: */ void mkSingleDerivedPathString(const SingleDerivedPath & p, Value & v); - void concatLists(Value & v, size_t nrLists, Value * const * lists, const PosIdx pos, std::string_view errorCtx); + /** + * @brief Concatenate values with an n-ary version of the `++` operator. + */ + void concatLists(Value & v, std::span lists, const PosIdx pos, std::string_view errorCtx); /** * Print statistics, if enabled. diff --git a/src/libexpr/include/nix/expr/fetch-tree.hh b/src/libexpr/include/nix/expr/fetch-tree.hh new file mode 100644 index 000000000000..3eb8a01c0c5f --- /dev/null +++ b/src/libexpr/include/nix/expr/fetch-tree.hh @@ -0,0 +1,18 @@ +#pragma once + +#include "nix/expr/eval.hh" + +namespace nix { + +/** + * Convert a libfetchers `Input` to libexpr `Value`. + */ +void emitTreeAttrs( + EvalState & state, + const StorePath & storePath, + const fetchers::Input & input, + Value & v, + bool emptyRevFallback = false, + bool forceDirty = false); + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/meson.build b/src/libexpr/include/nix/expr/meson.build index b72b8d912440..83661f1c7165 100644 --- a/src/libexpr/include/nix/expr/meson.build +++ b/src/libexpr/include/nix/expr/meson.build @@ -20,6 +20,7 @@ headers = [ config_pub_h ] + files( 'eval-profiler.hh', 'eval-settings.hh', 'eval.hh', + 'fetch-tree.hh', 'function-trace.hh', 'gc-small-vector.hh', 'get-drvs.hh', diff --git a/src/libexpr/include/nix/expr/nixexpr.hh b/src/libexpr/include/nix/expr/nixexpr.hh index 9bce1a9b91ad..85a6c0c3e417 100644 --- a/src/libexpr/include/nix/expr/nixexpr.hh +++ b/src/libexpr/include/nix/expr/nixexpr.hh @@ -16,6 +16,7 @@ #include "nix/expr/counter.hh" #include "nix/util/pos-table.hh" #include "nix/util/error.hh" +#include "nix/util/bump-memory-resource.hh" namespace nix { @@ -545,7 +546,7 @@ public: std::numeric_limits::max()); if (pos) err.atPos(positions[pos]); - throw err; + throw std::move(err); } std::uninitialized_copy_n(formals.formals.begin(), nFormals, formalsStart); }; @@ -795,9 +796,10 @@ struct ExprPos : Expr class Exprs { - // FIXME: use std::pmr::monotonic_buffer_resource when parallel - // eval is disabled? - std::pmr::synchronized_pool_resource buffer; + /* Thread-safe fallback resource, which might be a bit slower. */ + std::pmr::synchronized_pool_resource fallbackResource; + BumpMemoryResource buffer{BumpMemoryResource::defaultReserveSize, &fallbackResource}; + public: std::pmr::polymorphic_allocator alloc{&buffer}; diff --git a/src/libexpr/include/nix/expr/parser-state.hh b/src/libexpr/include/nix/expr/parser-state.hh index f9bd06589e42..2482d53ea041 100644 --- a/src/libexpr/include/nix/expr/parser-state.hh +++ b/src/libexpr/include/nix/expr/parser-state.hh @@ -89,7 +89,7 @@ public: * @see https://github.com/NixOS/nix/issues/14642 */ template - void visit(F && f) + void visit(const F & f) { std::visit( overloaded{ diff --git a/src/libexpr/include/nix/expr/value/context.hh b/src/libexpr/include/nix/expr/value/context.hh index 4973ce0a9d94..29cf9ca0b841 100644 --- a/src/libexpr/include/nix/expr/value/context.hh +++ b/src/libexpr/include/nix/expr/value/context.hh @@ -11,6 +11,8 @@ namespace nix { class BadNixStringContextElem final : public CloneableError { + void anchor() override; + public: std::string_view raw; diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 77020d704294..378a046687b3 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -3,13 +3,14 @@ #include "nix/expr/eval.hh" #include -#include #include using json = nlohmann::json; namespace nix { +namespace { + // for more information, refer to // https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp class JSONSax : nlohmann::json_sax @@ -201,6 +202,8 @@ class JSONSax : nlohmann::json_sax } }; +} // namespace + void parseJSON(EvalState & state, const std::string_view & s_, Value & v) { JSONSax parser(state, v); @@ -209,4 +212,6 @@ void parseJSON(EvalState & state, const std::string_view & s_, Value & v) throw JSONParseError("Invalid JSON Value"); } +void JSONParseError::anchor() {} + } // namespace nix diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc index 59f6f6f70dfd..085ac7df4698 100644 --- a/src/libexpr/lexer-helpers.cc +++ b/src/libexpr/lexer-helpers.cc @@ -26,3 +26,5 @@ void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, Parser::location_type * loc->beginOffset = loc->endOffset; loc->endOffset += len; } + +nix::Parser::~Parser() {} diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 5bdb5335b841..477eee1df9e6 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -33,6 +33,8 @@ namespace nix { struct LexerState; } +// OK because we keep this in a separate compilation unit even for unity +// builds. using namespace nix; using namespace nix::lexer::internal; diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 25fc9bc8e56d..995202dbcb30 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -7,10 +7,8 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'unity=on', - 'unity_size=1024', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -231,7 +229,7 @@ parser_library = static_library( 'nixexpr-parser', parser_tab, lexer_tab, - files('lexer-helpers.cc'), + 'lexer-helpers.cc', cpp_args : parser_library_cpp_args, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, @@ -242,6 +240,7 @@ parser_library = static_library( override_options : [ 'b_ndebug=@0@'.format(not get_option('debug')), 'b_lto=@0@'.format(get_option('b_lto') and cxx.get_id() != 'gcc'), + # Unity builds mess up the recursive header dependency between flex and bison generated code. 'unity=off', ], ) diff --git a/src/libexpr/parser-scanner-decls.hh b/src/libexpr/parser-scanner-decls.hh index e4e06188334c..1bec23837b99 100644 --- a/src/libexpr/parser-scanner-decls.hh +++ b/src/libexpr/parser-scanner-decls.hh @@ -11,7 +11,9 @@ namespace nix { class Parser : public parser::BisonParser { +public: using BisonParser::BisonParser; + ~Parser() override; }; } // namespace nix diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index e5e4241ea7b0..77e63ed6299f 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -426,7 +426,7 @@ path_start /* Absolute paths are always interpreted relative to the root filesystem accessor, rather than the accessor of the current Nix expression. */ - auto path = canonPath(literal).string(); + auto path = CanonPath(literal).abs(); /* add back in the trailing '/' to the first segment */ if (literal.size() > 1 && literal.back() == '/') path += '/'; @@ -442,8 +442,7 @@ path_start return std::nullopt; }); - auto basePath = std::filesystem::path(state->basePath.path.abs()); - auto path = absPath(literal, &basePath).string(); + auto path = CanonPath(literal, state->basePath.path).abs(); /* add back in the trailing '/' to the first segment */ if (literal.size() > 1 && literal.back() == '/') path += '/'; diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 00165b44cc4e..6d644fe961e8 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -12,6 +12,8 @@ SourcePath EvalState::rootPath(CanonPath path) SourcePath EvalState::rootPath(std::string_view path) { + /* FIXME: Move this out of EvalState, since it's using native + std::filesystem::path and current working directory. */ return {rootFS, CanonPath(absPath(path).string())}; } @@ -27,7 +29,7 @@ StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites) fetchSettings, *store, SourcePath{ref(mount)}, - settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + settings.isReadOnly() ? FetchMode::DryRun : FetchMode::Copy, path.name()); assert(storePath.name() == path.name()); if (rewrites) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 91dc5bfdabfe..cf009515c57c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -697,7 +697,7 @@ static RegisterPrimOp primop_isPath({ }); template -static inline void withExceptionContext(Trace trace, Callable && func) +static inline void withExceptionContext(Trace trace, const Callable & func) { try { func(); @@ -1043,7 +1043,35 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value ** a static RegisterPrimOp primop_addErrorContext( PrimOp{ .name = "__addErrorContext", + .args = {"context", "value"}, .arity = 2, + .doc = R"( + Evaluate *context*, which can be coerced to a string, + and append it to any error or stack traces displayed while evaluating *value*. + Then return *value*. + + This function is useful for providing helpful context in complex Nix expressions + when the evaluation of *value* fails. + The additional context is applied when evaluating *value* itself fails, + not when attributes or elements of *value* are evaluated. + + For example, the module system from nixpkgs uses this to show + the relevant information about the options that were evaluating + when an error occurs. + + ```nix-repl + nix-repl> addErrorContext "while evaluating foo" (throw "bar") + error: + … while evaluating foo + + … while calling the 'throw' builtin + at «string»:1:56: + 1| with builtins; addErrorContext "while evaluating foo" (throw "bar") + | ^ + + error: bar + ``` + )", // The normal trace item is redundant .addTrace = false, .impl = prim_addErrorContext, @@ -1995,21 +2023,21 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V .debugThrow(); NixStringContext context; - auto path = - state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'") - .path; + SourcePath sourcePath = state.coerceToPath( + pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'"); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ - if (!state.store->isStorePath(path.abs())) - path = CanonPath(canonPath(path.abs(), true).string()); - if (!state.store->isInStore(path.abs())) - state.error("path '%1%' is not in the Nix store", path).atPos(pos).debugThrow(); - auto path2 = state.store->toStorePath(path.abs()).first; - if (!settings.readOnlyMode) - state.store->ensurePath(path2); - context.insert(NixStringContextElem::Opaque{.path = path2}); - v.mkString(path.abs(), context, state.mem); + if (!state.store->isStorePath(sourcePath.path.abs())) + sourcePath = sourcePath.resolveSymlinks(SymlinkResolution::Full); + if (!state.store->isInStore(sourcePath.path.abs())) + state.error("path '%1%' is not in the Nix store", sourcePath).atPos(pos).debugThrow(); + auto storePath = state.store->toStorePath(sourcePath.path.abs()).first; + if (!state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath))) && !settings.readOnlyMode) + state.store->ensurePath(storePath); + context.insert(NixStringContextElem::Opaque{.path = storePath}); + v.mkString(sourcePath.path.abs(), context, state.mem); } static RegisterPrimOp primop_storePath({ @@ -3171,6 +3199,8 @@ static RegisterPrimOp primop_attrNames({ Return the names of the attributes in the set *set* in an alphabetically sorted list. For instance, `builtins.attrNames { y = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`. + + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrNames, }); @@ -3203,6 +3233,8 @@ static RegisterPrimOp primop_attrValues({ .doc = R"( Return the values of the attributes in the set *set* in the order corresponding to the sorted attribute names. + + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrValues, }); @@ -3228,6 +3260,8 @@ static RegisterPrimOp primop_getAttr({ aborts if the attribute doesn’t exist. This is a dynamic version of the `.` operator, since *s* is an expression rather than an identifier. + + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_getAttr, }); @@ -3316,6 +3350,8 @@ static RegisterPrimOp primop_hasAttr({ `hasAttr` returns `true` if *set* has an attribute named *s*, and `false` otherwise. This is a dynamic version of the `?` operator, since *s* is an expression rather than an identifier. + + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_hasAttr, }); @@ -3375,6 +3411,8 @@ static RegisterPrimOp primop_removeAttrs({ ``` evaluates to `{ y = 2; }`. + + Has `O(n + k log k)` time complexity, where `n` is number of attributes in the *set* and `k` is the size of *list*. )", .impl = prim_removeAttrs, }); @@ -3462,6 +3500,8 @@ static RegisterPrimOp primop_listToAttrs({ ```nix { foo = 123; bar = 456; } ``` + + Has `O(n log n)` time complexity, where `n` is size of the list. )", .impl = prim_listToAttrs, }); @@ -3538,7 +3578,7 @@ static RegisterPrimOp primop_intersectAttrs({ Return a set consisting of the attributes in the set *e2* which have the same name as some attribute in *e1*. - Performs in O(*n* log *m*) where *n* is the size of the smaller set and *m* the larger set's size. + Has `O(n log m)` time complexity, where `n` and `m` are the sizes of the smallest and largest set respectively. )", .impl = prim_intersectAttrs, }); @@ -3578,6 +3618,8 @@ static RegisterPrimOp primop_catAttrs({ ``` evaluates to `[1 2]`. + + Has `O(n)` time complexity, where `n` is the size of the *list*. )", .impl = prim_catAttrs, }); @@ -3621,6 +3663,8 @@ static RegisterPrimOp primop_functionArgs({ "Formal argument" here refers to the attributes pattern-matched by the function. Plain lambdas are not included, e.g. `functionArgs (x: ...) = { }`. + + Has constant time complexity. )", .impl = prim_functionArgs, }); @@ -3653,6 +3697,10 @@ static RegisterPrimOp primop_mapAttrs({ ``` evaluates to `{ a = 10; b = 20; }`. + + Has `O(n)` time complexity, where `n` is the size of the *attrset*. + Note that no calls to *f* are performed by the builtin. + The function *f* is called on demand when a resulting attribute value is evaluated. )", .impl = prim_mapAttrs, }); @@ -3822,6 +3870,8 @@ static RegisterPrimOp primop_zipAttrsWith({ b = { name = "b"; values = [ "z" ]; }; } ``` + + Has `O(n log n)` time complexity, where `n` is the number of attributes across all sets. )", .impl = prim_zipAttrsWith, }); @@ -3856,8 +3906,9 @@ static void prim_elemAt(EvalState & state, const PosIdx pos, Value ** args, Valu state.error("'builtins.elemAt' called with index %d on a list of size %d", n, args[0]->listSize()) .atPos(pos) .debugThrow(); - state.forceValue(*args[0]->listView()[n], pos); - v = *args[0]->listView()[n]; + auto ptr = args[0]->listView()[n]; + state.forceValue(*ptr, pos); + v = *ptr; } static RegisterPrimOp primop_elemAt({ @@ -3887,6 +3938,8 @@ static RegisterPrimOp primop_head({ Return the first element of a list; abort evaluation if the argument isn’t a list or is an empty list. You can test whether a list is empty by comparing it with `[]`. + + Has constant time complexity. )", .impl = prim_head, }); @@ -3952,6 +4005,10 @@ static RegisterPrimOp primop_map({ ``` evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + Has `O(n)` time complexity, where `n` is the size of the *list*. + Note that no calls to *f* are performed by the builtin, but *f* itself is evaluated and its type is checked eagerly. + The function *f* is called on demand when a resulting list element is evaluated. )", .impl = prim_map, }); @@ -4001,6 +4058,7 @@ static RegisterPrimOp primop_filter({ .doc = R"( Return a list consisting of the elements of *list* for which the function *f* returns `true`. + Has linear time complexity in the size of the input *list*. )", .impl = prim_filter, }); @@ -4024,6 +4082,7 @@ static RegisterPrimOp primop_elem({ .doc = R"( Return `true` if a value equal to *x* occurs in the list *xs*, and `false` otherwise. + Short-circuits and does not evaluate elements that occur in the list after the first match. )", .impl = prim_elem, }); @@ -4033,12 +4092,7 @@ static void prim_concatLists(EvalState & state, const PosIdx pos, Value ** args, { state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.concatLists"); auto listView = args[0]->listView(); - state.concatLists( - v, - args[0]->listSize(), - listView.data(), - pos, - "while evaluating a value of the list passed to builtins.concatLists"); + state.concatLists(v, listView.span(), pos, "while evaluating a value of the list passed to builtins.concatLists"); } static RegisterPrimOp primop_concatLists({ @@ -4094,17 +4148,40 @@ static RegisterPrimOp primop_foldlStrict({ .args = {"op", "nul", "list"}, .doc = R"( Reduce a list by applying a binary operator, from left to right, - e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. + e.g. + ```nix + foldl' op nul [ x0 x1 x2 ] + = + let + strictly = f: a: builtins.seq a (f a); + y0 = op nul x0; + y1 = strictly op y0 x1; + y2 = strictly op y1 x2; + in + y2 + + # and, ignoring strictness/laziness + == + op (op (op nul x0) x1) x2 + ``` For example, `foldl' (acc: elem: acc + elem) 0 [1 2 3]` evaluates to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {} ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`. The first argument of `op` is the accumulator whereas the second - argument is the current element being processed. The return value - of each application of `op` is evaluated immediately, even for - intermediate values. + argument is the current element being processed. + + The return value of each application of `op` is evaluated immediately, + even for intermediate values. + This way, `foldl'` can operate in constant stack space, allowing it to operate on large lists, + regardless of [max-call-depth](@docroot@/command-ref/conf-file.md#conf-max-call-depth). + + Conventionally, a fold function without the `'` ("prime") preserves laziness, + but lacks these benefits. + See also [Nixpkgs `lib.foldl`](https://nixos.org/manual/nixpkgs/unstable/#function-library-lib.lists.foldl). + + Has linear time complexity in the size of the list. )", .impl = prim_foldlStrict, }); @@ -4143,6 +4220,7 @@ static RegisterPrimOp primop_any({ .doc = R"( Return `true` if the function *pred* returns `true` for at least one element of *list*, and `false` otherwise. + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `true`. )", .impl = prim_any, }); @@ -4158,6 +4236,7 @@ static RegisterPrimOp primop_all({ .doc = R"( Return `true` if the function *pred* returns `true` for all elements of *list*, and `false` otherwise. + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `false`. )", .impl = prim_all, }); @@ -4196,6 +4275,8 @@ static RegisterPrimOp primop_genList({ ``` returns the list `[ 0 1 4 9 16 ]`. + + Has linear time complexity. )", .impl = prim_genList, }); @@ -4306,6 +4387,9 @@ static RegisterPrimOp primop_sort({ If the *comparator* violates any of these properties, then `builtins.sort` reorders elements in an unspecified manner. + + Runs in `O(n log n)` time on average, where `n` is the size of the *list*. + Uses an adaptive sort that exploits existing sorted runs in the input, down to `O(n)` when the list is already sorted. )", .impl = prim_sort, }); @@ -4367,6 +4451,8 @@ static RegisterPrimOp primop_partition({ ```nix { right = [ 23 42 ]; wrong = [ 1 9 3 ]; } ``` + + Runs in linear time in the size of the *list*. )", .impl = prim_partition, }); @@ -4418,6 +4504,8 @@ static RegisterPrimOp primop_groupBy({ ```nix { b = [ "bar" "baz" ]; f = [ "foo" ]; } ``` + + Has `O(n log n)` time complexity, where `n` is the size of the input *list*. )", .impl = prim_groupBy, }); @@ -5248,6 +5336,8 @@ static RegisterPrimOp primop_replaceStrings({ ``` evaluates to `"fabir"`. + + Has `O(n k)` time complexity, where `n` is the length of *s* and `k` is the number of replacements. )", .impl = prim_replaceStrings, }); diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix index d3b341a23713..06c7eb8a67e0 100644 --- a/src/libexpr/primops/derivation.nix +++ b/src/libexpr/primops/derivation.nix @@ -47,7 +47,7 @@ let outputToAttrListElement = outputName: { name = outputName; value = commonAttrs // { - outPath = builtins.getAttr outputName strict; + outPath = strict.${outputName}; drvPath = strict.drvPath; type = "derivation"; inherit outputName; diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 84bce6116ea3..aebafb35b0d4 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -2,7 +2,6 @@ #include "nix/store/store-open.hh" #include "nix/store/realisation.hh" #include "nix/store/make-content-addressed.hh" -#include "nix/util/url.hh" #include "nix/util/environment-variables.hh" namespace nix { diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 59ffeabf1ef8..172c37948c93 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,9 +1,7 @@ #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" -#include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" -#include "nix/util/url.hh" #include "nix/util/url-parts.hh" namespace nix { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index fff8ddfc12e0..392294abc7e0 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,7 +1,9 @@ +#include "nix/expr/value.hh" #include "nix/fetchers/attrs.hh" #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" #include "nix/store/filetransfer.hh" @@ -16,10 +18,104 @@ #include #include -#include namespace nix { +/** + * Adapter for putting libfetchers data into a thunk closure. + * Used as the argument to prim_forceLazyFetcherAttr in a lazy apply thunk. + */ +class LazyFetcherAttr : public ExternalValueBase, public gc_cleanup +{ +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); + fetchers::LazyAttr lazy; + +public: + LazyFetcherAttr(fetchers::LazyAttr lazy) + : lazy(std::move(lazy)) + { + } + + fetchers::ResolvedAttr force() + { + return lazy->compute(); + } + +protected: + std::ostream & print(std::ostream & str) const override + { + unreachable(); + } + +public: + std::string showType() const override + { + unreachable(); + } + + std::string typeOf() const override + { + unreachable(); + } +}; + +void LazyFetcherAttr::anchor() {} + +/** + * Initialize a `Value` from a resolved fetcher attribute. + */ +static void resolvedAttrToValue(EvalState & state, Value & v, const fetchers::ResolvedAttr & resolved) +{ + std::visit( + overloaded{ + [&](const std::string & s) { v.mkString(s, state.mem); }, + [&](uint64_t n) { v.mkInt(n); }, + [&](const Explicit & b) { v.mkBool(b.t); }, + }, + resolved); +} + +/** + * internal primop: Force a LazyFetcherAttr external value. + */ +static void prim_forceLazyFetcherAttr(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + Value & arg = *args[0]; + + state.forceValue(arg, pos); + // We only construct this primop with LazyFetcherAttr preapplied. + assert(arg.type() == nExternal); + auto * ext = dynamic_cast(args[0]->external()); + assert(ext); + + resolvedAttrToValue(state, v, ext->force()); +} + +/** + * Emit a lazy thunk for a LazyAttr: mkApp(primop, externalValue). + */ +static void emitLazyAttrThunk(EvalState & state, const fetchers::LazyAttr & lazyAttr, Value & dest) +{ + // not user-callable (unregistered, internal) + static PrimOp forcePrimOp{ + .name = "__forceLazyFetcherAttr", + .arity = 1, + .impl = prim_forceLazyFetcherAttr, + .internal = true, + }; + + auto * vExt = state.allocValue(); + vExt->mkExternal(new LazyFetcherAttr(lazyAttr)); + + auto * vPrimOp = state.allocValue(); + vPrimOp->mkPrimOp(&forcePrimOp); + + dest.mkApp(vPrimOp, vExt); +} + void emitTreeAttrs( EvalState & state, const StorePath & storePath, @@ -56,7 +152,9 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem); } - if (auto revCount = input.getRevCount()) + if (auto revCount = maybeGetLazyAttr(input.attrs, "revCount")) + emitLazyAttrThunk(state, *revCount, attrs.alloc("revCount")); + else if (auto revCount = input.getRevCount()) attrs.alloc("revCount").mkInt(*revCount); else if (emptyRevFallback) attrs.alloc("revCount").mkInt(0); @@ -578,7 +676,12 @@ static RegisterPrimOp primop_fetchGit({ - `url` - The URL of the repo. + The [Git URL] of the repo. SCP-like syntax is supported, but relative + paths are rewritten to absolute ones. For example: + + `git@github.com:repo/path` becomes `ssh://git@github.com/repo/path` + + [Git URL]: https://git-scm.com/docs/git-clone#_git_urls - `name` (default: `source`) diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index b25f8fb759b0..33477e1bc86d 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -5,7 +5,6 @@ #include "nix/expr/parallel-eval.hh" #include -#include #include namespace nix { @@ -146,6 +145,8 @@ json printValueAsJSON( return res; } +void JSONSerializationError::anchor() {} + void printValueAsJSON( EvalState & state, bool strict, diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 3bdc73f94aee..13b346af11fc 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -2,10 +2,10 @@ #include "nix/expr/value/context.hh" #include "nix/store/store-dir-config.hh" -#include - namespace nix { +void BadNixStringContextElem::anchor() {} + NixStringContextElem NixStringContextElem::parse(std::string_view s0, const ExperimentalFeatureSettings & xpSettings) { std::string_view s = s0; diff --git a/src/libfetchers-c/meson.build b/src/libfetchers-c/meson.build index ce74dae99886..44d0e7c31876 100644 --- a/src/libfetchers-c/meson.build +++ b/src/libfetchers-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -50,15 +50,32 @@ headers += files('nix_api_fetchers.h') subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixfetchersc', - sources, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : prelink, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixfetchersc', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixfetchersc', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libfetchers-c/meson.options b/src/libfetchers-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libfetchers-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libfetchers-c/nix_api_fetchers.cc b/src/libfetchers-c/nix_api_fetchers.cc index 7fefedb0c70f..2b3dd56631c5 100644 --- a/src/libfetchers-c/nix_api_fetchers.cc +++ b/src/libfetchers-c/nix_api_fetchers.cc @@ -7,7 +7,7 @@ extern "C" { nix_fetchers_settings * nix_fetchers_settings_new(nix_c_context * context) { try { - auto fetchersSettings = nix::make_ref(nix::fetchers::Settings{}); + auto fetchersSettings = nix::make_ref(); return new nix_fetchers_settings{ .settings = fetchersSettings, }; diff --git a/src/libfetchers-c/package.nix b/src/libfetchers-c/package.nix index 13ec30d566eb..39147228f2c0 100644 --- a/src/libfetchers-c/package.nix +++ b/src/libfetchers-c/package.nix @@ -10,6 +10,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -27,7 +28,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -41,6 +42,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libfetchers-tests/access-tokens.cc b/src/libfetchers-tests/access-tokens.cc index 26cdcfb83fc9..a482b70dcecf 100644 --- a/src/libfetchers-tests/access-tokens.cc +++ b/src/libfetchers-tests/access-tokens.cc @@ -3,8 +3,6 @@ #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/fetch-settings.hh" -#include "nix/util/json-utils.hh" -#include "nix/util/tests/characterization.hh" namespace nix::fetchers { diff --git a/src/libfetchers-tests/attrs.cc b/src/libfetchers-tests/attrs.cc new file mode 100644 index 000000000000..4d0cbbb9ef74 --- /dev/null +++ b/src/libfetchers-tests/attrs.cc @@ -0,0 +1,75 @@ +#include + +#include "nix/fetchers/attrs.hh" + +#include + +namespace nix::fetchers { + +TEST(LazyAttr, resolveToInt) +{ + Attrs attrs; + attrs.insert_or_assign( + "count", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(42); + }}))); + EXPECT_EQ(maybeGetIntAttr(attrs, "count"), 42); +} + +TEST(LazyAttr, resolveToString) +{ + Attrs attrs; + attrs.insert_or_assign( + "name", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("hello"); + }}))); + EXPECT_EQ(maybeGetStrAttr(attrs, "name"), "hello"); +} + +TEST(LazyAttr, resolveToBool) +{ + Attrs attrs; + attrs.insert_or_assign( + "flag", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return Explicit{true}; + }}))); + EXPECT_EQ(maybeGetBoolAttr(attrs, "flag"), true); +} + +TEST(LazyAttr, attrsToJSONForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "x", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(99); + }}))); + auto json = attrsToJSON(attrs); + EXPECT_EQ(json["x"], 99); +} + +TEST(LazyAttr, attrsToQueryForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "v", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("val"); + }}))); + auto query = attrsToQuery(attrs); + EXPECT_EQ(query.at("v"), "val"); +} + +TEST(LazyAttr, notCalledUntilForced) +{ + int calls = 0; + Attrs attrs; + attrs.insert_or_assign( + "lazy", LazyAttr(make_ref(LazyAttrComputation{.compute = [&calls]() -> ResolvedAttr { + calls++; + return uint64_t(1); + }}))); + EXPECT_EQ(calls, 0); + maybeGetIntAttr(attrs, "lazy"); + EXPECT_EQ(calls, 1); +} + +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/git-lfs-fetch.cc b/src/libfetchers-tests/git-lfs-fetch.cc new file mode 100644 index 000000000000..d01a43151df5 --- /dev/null +++ b/src/libfetchers-tests/git-lfs-fetch.cc @@ -0,0 +1,28 @@ +#include "nix/fetchers/git-lfs-fetch.hh" +#include "nix/util/url.hh" + +#include + +namespace nix::lfs { + +struct GitLFSParameterizedTestFixture : public ::testing::TestWithParam> +{}; + +TEST_P(GitLFSParameterizedTestFixture, get_lfs_api) +{ + auto & [input, expected] = GetParam(); + ASSERT_EQ(getLfsApi(parseURL(input)).endpoint, expected); +}; + +INSTANTIATE_TEST_SUITE_P( + GitLFSTests, + GitLFSParameterizedTestFixture, + ::testing::Values( + std::pair{"https://git-server.com/foo/bar", "https://git-server.com/foo/bar.git/info/lfs"}, + std::pair{"https://git-server.com/foo/bar.git", "https://git-server.com/foo/bar.git/info/lfs"}, + std::pair{"https://git-server.com", "https://git-server.com/.git/info/lfs"}, + std::pair{"https://git-server.com/", "https://git-server.com/.git/info/lfs"}, + std::pair{"https://git-server.com//", "https://git-server.com//.git/info/lfs"}, + std::pair{"https://git-server.com/foo/bar/", "https://git-server.com/foo/bar.git/info/lfs"})); + +} // namespace nix::lfs diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index 0b21fd0c67d5..580769936d41 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -10,12 +10,11 @@ #include #include "nix/util/fs-sink.hh" #include "nix/util/serialise.hh" -#include "nix/fetchers/git-lfs-fetch.hh" #include #include -namespace nix { +namespace nix::fetchers { class GitUtilsTest : public ::testing::Test { @@ -234,4 +233,4 @@ TEST(GitUtils, isLegalRefName) ASSERT_FALSE(isLegalRefName("")); } -} // namespace nix +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/git.cc b/src/libfetchers-tests/git.cc index abc3dd74c5c6..e41bbe520f41 100644 --- a/src/libfetchers-tests/git.cc +++ b/src/libfetchers-tests/git.cc @@ -1,9 +1,9 @@ -#include "nix/store/store-open.hh" #include "nix/store/globals.hh" #include "nix/store/dummy-store.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/git-utils.hh" +#include "nix/util/url.hh" #include #include @@ -97,7 +97,7 @@ static void commitAll(git_repository * repo, const char * msg) } // namespace -using namespace nix; +namespace nix::fetchers { class GitTest : public ::testing::Test { @@ -190,7 +190,7 @@ TEST_F(GitTest, submodulePeriodSupport) auto input = fetchers::Input::fromAttrs( settings, { - {"url", "file://" + repoPath.string()}, + {"url", "file://" + encodeUrlPath(pathToUrlPath(repoPath))}, {"submodules", Explicit{true}}, {"type", "git"}, {"ref", "main"}, @@ -200,3 +200,5 @@ TEST_F(GitTest, submodulePeriodSupport) ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n"); } + +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/input.cc b/src/libfetchers-tests/input.cc index faff55f2c2d0..170ff4485c7b 100644 --- a/src/libfetchers-tests/input.cc +++ b/src/libfetchers-tests/input.cc @@ -1,8 +1,12 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/attrs.hh" #include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/util/tests/gmock-matchers.hh" +#include "nix/util/url.hh" #include +#include #include @@ -58,4 +62,18 @@ INSTANTIATE_TEST_SUITE_P( }), [](const ::testing::TestParamInfo & info) { return info.param.description; }); +namespace fetchers { + +class GitHubInputTest : public ::testing::Test +{}; + +TEST_F(GitHubInputTest, throwOnInvalidURLParam) +{ + EXPECT_THAT( + []() { Input::fromURL(fetchers::Settings{}, "github:a/b?tag=foo"); }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("tag"))); +} + +} // namespace fetchers + } // namespace nix diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index 6bccdb05c9a1..22cd6b378302 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -40,6 +40,8 @@ subdir('nix-meson-build-support/common') sources = files( 'access-tokens.cc', + 'attrs.cc', + 'git-lfs-fetch.cc', 'git-utils.cc', 'git.cc', 'input.cc', @@ -66,6 +68,8 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) diff --git a/src/libfetchers-tests/nix_api_fetchers.cc b/src/libfetchers-tests/nix_api_fetchers.cc index 8f3e6e3c5839..ff2c5d8ff319 100644 --- a/src/libfetchers-tests/nix_api_fetchers.cc +++ b/src/libfetchers-tests/nix_api_fetchers.cc @@ -1,4 +1,3 @@ -#include "gmock/gmock.h" #include #include "nix_api_fetchers.h" diff --git a/src/libfetchers-tests/public-key.cc b/src/libfetchers-tests/public-key.cc index 2991223f6b35..dabbd464424c 100644 --- a/src/libfetchers-tests/public-key.cc +++ b/src/libfetchers-tests/public-key.cc @@ -1,6 +1,5 @@ #include #include "nix/fetchers/fetchers.hh" -#include "nix/util/json-utils.hh" #include "nix/util/tests/json-characterization.hh" namespace nix { diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 648d48545431..84c74df275df 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -1,10 +1,21 @@ #include "nix/fetchers/attrs.hh" -#include "nix/fetchers/fetchers.hh" #include namespace nix::fetchers { +ResolvedAttr forceAttr(const Attr & attr) +{ + return std::visit( + overloaded{ + [](const LazyAttr & lazy) -> ResolvedAttr { return lazy->compute(); }, + [](const std::string & v) -> ResolvedAttr { return v; }, + [](uint64_t v) -> ResolvedAttr { return v; }, + [](const Explicit & v) -> ResolvedAttr { return v; }, + }, + attr); +} + Attrs jsonToAttrs(const nlohmann::json & json) { Attrs attrs; @@ -30,11 +41,12 @@ nlohmann::json attrsToJSON(const Attrs & attrs) /* The __final attribute is purely internal, so never serialize it. */ if (attr.first == "__final") continue; - if (auto v = std::get_if(&attr.second)) { + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { json[attr.first] = v->t; } else unreachable(); @@ -42,12 +54,23 @@ nlohmann::json attrsToJSON(const Attrs & attrs) return json; } +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) + return {}; + if (auto v = std::get_if(&i->second)) + return *v; + return {}; +} + std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump()); } @@ -65,7 +88,8 @@ std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not an integer", name); } @@ -83,7 +107,8 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if>(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if>(&resolved)) return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -100,11 +125,12 @@ StringMap attrsToQuery(const Attrs & attrs) { StringMap query; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, fmt("%d", *v)); - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, *v); - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); } else unreachable(); diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index d01b127be678..0c24b1072a4d 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -26,6 +26,8 @@ create table if not exists Cache ( struct CacheImpl : Cache { + void anchor() override; + struct State { SQLite db; @@ -160,6 +162,10 @@ struct CacheImpl : Cache } }; +void Cache::anchor() {} + +void CacheImpl::anchor() {} + ref Settings::getCache() const { auto cache(_cache.lock()); diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index f50177f094e3..568e805a0b57 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -5,6 +5,8 @@ namespace nix::fetchers { Settings::Settings() {} +void Settings::anchor() {} + } // namespace nix::fetchers namespace nix { diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 8bab1f7bd15e..4062d5899a4b 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -105,6 +105,7 @@ std::pair fetchToStore2( auto [storePath, hash] = mode == FetchMode::DryRun ? [&]() { + // FIXME: we may have already computed this above. auto [storePath, hash] = store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2); debug( diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 595f518c7cac..b1bc134ac34e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,17 +1,17 @@ #include "nix/fetchers/fetchers.hh" #include "nix/store/store-api.hh" +#include "nix/util/fs-sink.hh" #include "nix/util/source-path.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/json-utils.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/provenance.hh" #include "nix/util/url.hh" -#include "nix/util/forwarding-source-accessor.hh" -#include "nix/util/archive.hh" #include "nix/util/users.hh" #include "nix/store/pathlocks.hh" #include "nix/util/environment-variables.hh" +#include #include #include @@ -340,9 +340,12 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings reuse it. We only do this for final inputs, since otherwise there is a risk that we don't return the same attributes (like `lastModified`) that the "real" fetcher would return. */ - if (storePath && store.isValidPath(*storePath)) { - debug("using input '%s' in '%s'", to_string(), store.printStorePath(*storePath)); - return makeStoreAccessor(); + if (storePath) { + store.addTempRoot(*storePath); + if (store.isValidPath(*storePath)) { + debug("using input '%s' in '%s'", to_string(), store.printStorePath(*storePath)); + return makeStoreAccessor(); + } } auto fixupAccessor = [&](ref accessor, Input result) -> std::pair, Input> { @@ -365,7 +368,8 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings PathLocks lock( {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); - if (getEnv("_NIX_TEST_CONCURRENT_FETCHES")) + static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; + if (inTest) std::this_thread::sleep_for(std::chrono::seconds(1)); /* See if the input is in the cache of the fetcher. */ @@ -378,7 +382,6 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings /* If not, try to substitute the input. */ if (storePath) { try { - store.addTempRoot(*storePath); store.ensurePath(*storePath); return makeStoreAccessor(); } diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index a42b5822245e..c881718fe5a4 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -1,7 +1,7 @@ #include "nix/fetchers/filtering-source-accessor.hh" #include "nix/util/sync.hh" -#include +#include namespace nix { @@ -69,11 +69,6 @@ std::shared_ptr FilteringSourceAccessor::getProvenance(const C return next->getProvenance(prefix / path); } -void FilteringSourceAccessor::invalidateCache(const CanonPath & path) -{ - next->invalidateCache(prefix / path); -} - void FilteringSourceAccessor::checkAccess(const CanonPath & path) { if (!isAllowed(path)) @@ -82,23 +77,27 @@ void FilteringSourceAccessor::checkAccess(const CanonPath & path) struct AllowListSourceAccessorImpl : AllowListSourceAccessor { +private: + void anchor() override {}; +public: SharedSync> allowedPrefixes; - SharedSync> allowedPaths; + boost::concurrent_flat_set allowedPaths; AllowListSourceAccessorImpl( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError) : AllowListSourceAccessor(SourcePath(next), std::move(makeNotAllowedError)) - , allowedPrefixes(std::move(allowedPrefixes)) - , allowedPaths(std::move(allowedPaths)) + , allowedPrefixes(allowedPrefixes.begin(), allowedPrefixes.end()) + , allowedPaths(allowedPaths.begin(), allowedPaths.end()) { } bool isAllowed(const CanonPath & path) override { - return allowedPaths.readLock()->contains(path) || path.isAllowed(*allowedPrefixes.readLock()); + /* Read lock is held for the duration of the full expression if the || doesn't short-circuit. */ + return allowedPaths.contains(path) || path.isAllowed(*allowedPrefixes.readLock()); } void allowPrefix(CanonPath prefix) override @@ -109,12 +108,11 @@ struct AllowListSourceAccessorImpl : AllowListSourceAccessor ref AllowListSourceAccessor::create( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError) { - return make_ref( - next, std::move(allowedPrefixes), std::move(allowedPaths), std::move(makeNotAllowedError)); + return make_ref(next, allowedPrefixes, allowedPaths, std::move(makeNotAllowedError)); } bool CachingFilteringSourceAccessor::isAllowed(const CanonPath & path) diff --git a/src/libfetchers/git-lfs-fetch.cc b/src/libfetchers/git-lfs-fetch.cc index 79393e6fe470..c80732a0c21f 100644 --- a/src/libfetchers/git-lfs-fetch.cc +++ b/src/libfetchers/git-lfs-fetch.cc @@ -1,6 +1,8 @@ #include "nix/fetchers/git-lfs-fetch.hh" #include "nix/fetchers/git-utils.hh" #include "nix/store/filetransfer.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/util/url.hh" @@ -23,9 +25,7 @@ namespace nix::lfs { static void downloadToSink( const std::string & url, const std::optional & authHeader, - // FIXME: passing a StringSink is superfluous, we may as well - // return a string. Or use an abstract Sink for streaming. - StringSink & sink, + Sink & sink, std::string sha256Expected, size_t sizeExpected) { @@ -34,29 +34,25 @@ static void downloadToSink( if (authHeader.has_value()) headers.push_back({"Authorization", *authHeader}); request.headers = headers; - getFileTransfer()->download(std::move(request), sink); - auto sizeActual = sink.s.length(); - if (sizeExpected != sizeActual) - throw Error("size mismatch while fetching %s: expected %d but got %d", url, sizeExpected, sizeActual); + HashSink hashSink(HashAlgorithm::SHA256); + TeeSink teeSink(hashSink, sink); - auto sha256Actual = hashString(HashAlgorithm::SHA256, sink.s).to_string(HashFormat::Base16, false); + getFileTransfer()->download(std::move(request), teeSink); + + auto hashResult = hashSink.finish(); + + if (sizeExpected != hashResult.numBytesDigested) + throw Error( + "size mismatch while fetching %s: expected %d but got %d", url, sizeExpected, hashResult.numBytesDigested); + + auto sha256Actual = hashResult.hash.to_string(HashFormat::Base16, false); if (sha256Actual != sha256Expected) throw Error( "hash mismatch while fetching %s: expected sha256:%s but got sha256:%s", url, sha256Expected, sha256Actual); } -namespace { - -struct LfsApiInfo -{ - std::string endpoint; - std::optional authHeader; -}; - -} // namespace - -static LfsApiInfo getLfsApi(const ParsedURL & url) +LfsApiInfo getLfsApi(ParsedURL url) { assert(url.authority.has_value()); if (url.scheme == "ssh") { @@ -96,7 +92,41 @@ static LfsApiInfo getLfsApi(const ParsedURL & url) return {queryResp.at("href").get(), authIt->get()}; } - return {url.to_string() + "/info/lfs", std::nullopt}; + /** + * Try to mimic what git-lfs will do to plain remotes + * https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md + * + * Try to be smarter with remotes ending in a /, like + * `https://github.com/NixOS/nix/`. This should be + * `https://github.com/NixOS/nix.git/info/lfs`, not + * `https://github.com/NixOS/nix/.git/info/lfs` + */ + bool hasDotGit = false; + for (auto it = url.path.rbegin(); it != url.path.rend(); ++it) { + if (it->empty()) + continue; + if (!it->ends_with(".git")) + *it += ".git"; + hasDotGit = true; + break; + } + if (!hasDotGit) { + if (url.path.size() > 1) // e.g. {"", ""} (single trailing slash) + url.path.back() = ".git"; + else if (url.path.size() == 1) // {""} + url.path.push_back(".git"); + else { // {} + url.path.push_back(""); + url.path.push_back(".git"); + } + } + if (url.path.back().empty()) + url.path.back() = "info"; + else + url.path.push_back("info"); + url.path.push_back("lfs"); + + return {url.to_string(), std::nullopt}; } typedef std::unique_ptr> GitConfig; @@ -255,7 +285,7 @@ std::vector Fetch::fetchUrls(const std::vector & pointe void Fetch::fetch( const std::string & content, const CanonPath & pointerFilePath, - StringSink & sink, + Sink & sink, std::function sizeCallback) const { debug("trying to fetch '%s' using git-lfs", pointerFilePath); @@ -279,9 +309,13 @@ void Fetch::fetch( std::string key = hashString(HashAlgorithm::SHA256, pointerFilePath.rel()).to_string(HashFormat::Base16, false) + "/" + pointer->oid; auto cachePath = cacheDir / key; - if (pathExists(cachePath)) { + AutoCloseFD cacheFile(openFileReadonly(cachePath, FinalSymlink::DontFollow)); + if (cacheFile) { debug("using cache entry %s -> %s", key, PathFmt(cachePath)); - sink(readFile(cachePath)); + FdSource cacheSource(cacheFile.get()); + auto size = getFileSize(cacheFile.get()); + sizeCallback(size); + cacheSource.drainInto(sink, size); return; } debug("did not find cache entry for %s", key); @@ -319,13 +353,23 @@ void Fetch::fetch( pointer->size); } - sizeCallback(size); - downloadToSink(ourl, authHeader, sink, sha256, size); - debug("creating cache entry %s -> %s", key, PathFmt(cachePath)); + if (!pathExists(cachePath.parent_path())) createDirs(cachePath.parent_path()); - writeFile(cachePath, sink.s); + auto [tempFile, tempPath] = createTempFile(cachePath.parent_path(), {}); + AutoDelete tempDeleter(tempPath); + FdSink tempSink(tempFile.get()); + downloadToSink(ourl, authHeader, tempSink, sha256, size); + tempSink.flush(); + + std::filesystem::rename(tempPath, cachePath); + tempDeleter.cancel(); + + FdSource cacheSource(tempFile.get()); + cacheSource.restart(); + sizeCallback(size); + cacheSource.drainInto(sink, size); debug("%s fetched with git-lfs", pointerFilePath); } catch (const nlohmann::json::out_of_range & e) { diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 9f59a48cebe2..27476a3e6098 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -44,9 +44,7 @@ #include #include #include -#include #include -#include #include namespace std { @@ -77,6 +75,8 @@ namespace nix { struct GitSourceAccessor; +namespace { + struct GitError final : public CloneableError { template @@ -100,6 +100,8 @@ struct GitError final : public CloneableError } }; +} // namespace + typedef std::unique_ptr> Repository; typedef std::unique_ptr> TreeEntry; typedef std::unique_ptr> Tree; @@ -229,7 +231,7 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options options) { - if (pathExists(path.string())) + if (pathExists(path)) return; if (!options.create) @@ -573,7 +575,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this /* Get submodule info. */ auto modulesFile = path / ".gitmodules"; - if (pathExists(modulesFile.string())) + if (pathExists(modulesFile)) info.submodules = parseSubmodules(modulesFile); return info; @@ -644,7 +646,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this // Remove shallow.lock left behind by a previously interrupted `git fetch`, as it would prevent `git fetch` // from running. Note that we already have a repository-wide `PathLock` (see git.cc), so this is safe. - std::filesystem::remove(dir / "shallow.lock"); + tryUnlink(dir / "shallow.lock"); OsStrings gitArgs{"-C", dir.native(), "--git-dir", ".", "fetch", "--progress", "--force"}; if (shallow) { @@ -804,6 +806,9 @@ std::string GitAccessorOptions::makeFingerprint(const Hash & rev) const */ struct GitSourceAccessor : SourceAccessor { +private: + void anchor() override {}; +public: struct State { ref repo; @@ -833,20 +838,16 @@ struct GitSourceAccessor : SourceAccessor if (state->lfsFetch) { if (state->lfsFetch->shouldFetch(path)) { - StringSink s; try { // FIXME: do we need to hold the state lock while // doing this? auto contents = std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get())); - state->lfsFetch->fetch(contents, path, s, [&s](uint64_t size) { s.s.reserve(size); }); + state->lfsFetch->fetch(contents, path, sink, sizeCallback); } catch (Error & e) { e.addTrace({}, "while smudging git-lfs file '%s'", path); throw; } - sizeCallback(s.s.size()); - StringSource source{s.s}; - source.drainInto(sink); return; } } @@ -1083,6 +1084,9 @@ struct GitSourceAccessor : SourceAccessor struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor { +private: + void anchor() override {}; +public: ref repo; std::optional rev; @@ -1141,6 +1145,10 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor } }; +void GitFileSystemObjectSink::anchor() {} + +namespace { + struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink { ref repo; @@ -1436,6 +1444,8 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink } }; +} // namespace + ref GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options) { auto self = ref(shared_from_this()); @@ -1458,13 +1468,15 @@ ref GitRepoImpl::getAccessor( const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) { auto self = ref(shared_from_this()); - ref fileAccessor = AllowListSourceAccessor::create( - makeFSSourceAccessor(path), - std::set{wd.files}, - // Always allow access to the root, but not its children. - boost::unordered_flat_set{CanonPath::root}, - std::move(makeNotAllowedError)) - .cast(); + ref fileAccessor = + AllowListSourceAccessor::create( + // Follow the final symlink to the repo. Older nix versions used to do this (maybe somewhat accidentally). + makeFSSourceAccessor(path, /*trackLastModified=*/false, FinalSymlink::Follow), + /*allowedPrefixes=*/wd.files, + // Always allow access to the root, but not its children. + /*allowedPaths=*/{CanonPath::root}, + std::move(makeNotAllowedError)) + .cast(); if (options.exportIgnore) fileAccessor = make_ref(self, fileAccessor, std::nullopt); return fileAccessor; @@ -1488,6 +1500,7 @@ std::vector> GitRepoImpl::getSubmodules auto configS = accessor->readFile(modulesFile); auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules"); + AutoDelete delTemp(pathTemp, /*recursive=*/false); try { writeFull(fdTemp.get(), configS); } catch (SystemError & e) { @@ -1506,6 +1519,7 @@ std::vector> GitRepoImpl::getSubmodules result.push_back({std::move(submodule), *rev}); } + delTemp.deletePath(); return result; } @@ -1526,9 +1540,11 @@ ref Settings::getTarballCache() const } // namespace fetchers +static Sync> workdirInfoCache_; + GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) { - static Sync> _cache; + auto & _cache = workdirInfoCache_; { auto cache(_cache.lock()); auto i = cache->find(path); @@ -1540,6 +1556,11 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & return workdirInfo; } +void GitRepo::invalidateWorkdirInfoCache() +{ + workdirInfoCache_.lock()->clear(); +} + bool isLegalRefName(const std::string & refName) { initLibGit2(); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index fadc68400166..f4d9003cb503 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,26 +1,23 @@ +#include "nix/util/environment-variables.hh" #include "nix/util/error.hh" #include "nix/fetchers/fetchers.hh" #include "nix/util/users.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" #include "nix/util/tarfile.hh" #include "nix/store/store-api.hh" -#include "nix/util/url-parts.hh" #include "nix/store/pathlocks.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/util/git.hh" #include "nix/fetchers/git-utils.hh" #include "nix/util/logging.hh" -#include "nix/util/finally.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/util/json-utils.hh" #include "nix/util/archive.hh" +#include "nix/util/memo.hh" #include "nix/util/mounted-source-accessor.hh" #include "nix/fetchers/fetch-to-store.hh" -#include -#include #include #ifndef _WIN32 @@ -169,6 +166,13 @@ std::vector getPublicKeys(const Attrs & attrs) static const Hash nullRev{HashAlgorithm::SHA1}; +static LazyAttr makeLazyAttr(fun compute) +{ + return make_ref(LazyAttrComputation{ + .compute = memo(std::move(compute)), + }); +} + struct GitInputScheme : InputScheme { std::optional inputFromURL(const Settings & settings, const ParsedURL & url, bool requireTree) const override @@ -463,7 +467,7 @@ struct GitInputScheme : InputScheme args.push_back(destDir.native()); - runProgram("git", true, args, {}, true); + runProgram("git", true, args, true); } std::optional getSourcePath(const Input & input) const override @@ -523,6 +527,10 @@ struct GitInputScheme : InputScheme }); if (commitMsg) { + auto [tempFd, tempPath] = createTempFile("nix-msg"); + AutoDelete delTemp(tempPath, /*recursive=*/false); + writeFull(tempFd.get(), *commitMsg); + // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` auto suspension = logger->suspend(); runProgram( @@ -536,9 +544,10 @@ struct GitInputScheme : InputScheme OS_STR("commit"), string_to_os_string(std::string(path.rel())), OS_STR("-F"), - OS_STR("-"), - }, - *commitMsg); + tempPath.native(), + }); + + delTemp.deletePath(); } } } @@ -726,14 +735,12 @@ struct GitInputScheme : InputScheme } uint64_t getRevCount( - const Settings & settings, - const RepoInfo & repoInfo, - const std::filesystem::path & repoDir, - const Hash & rev) const + ref cache, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { - Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; + if (GitRepo::openRepo(repoDir, {})->isShallow()) + throw Error("'%s' is a shallow Git repository, so 'revCount' is not available", repoInfo.locationToArg()); - auto cache = settings.getCache(); + Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); @@ -748,6 +755,18 @@ struct GitInputScheme : InputScheme return revCount; } + LazyAttr lazyRevCount( + const Settings & settings, + const RepoInfo & repoInfo, + const std::filesystem::path & repoDir, + const Hash & rev) const + { + auto cache = settings.getCache(); + return makeLazyAttr([this, cache, repoInfo, repoDir, rev]() -> ResolvedAttr { + return getRevCount(cache, repoInfo, repoDir, rev); + }); + } + std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const { auto head = std::visit( @@ -771,7 +790,7 @@ struct GitInputScheme : InputScheme "\n" "To make it visible to Nix, run:\n" "\n" - "git -C %2% add \"%1%\"", + "git -C %2% add -N \"%1%\"", path.rel(), PathFmt(repoPath)); else @@ -942,7 +961,7 @@ struct GitInputScheme : InputScheme } } - std::filesystem::create_directories(cacheDir.parent_path()); + createDirs(cacheDir.parent_path()); PathLocks cacheDirLock({cacheDir.string()}); auto repo = GitRepo::openRepo(cacheDir, {.create = true, .bare = true}); @@ -1040,7 +1059,7 @@ struct GitInputScheme : InputScheme "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.locationToArg()); - input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoDir, rev)); } printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); @@ -1205,8 +1224,11 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", rev.gitRev()); if (!getShallowAttr(input)) { - input.attrs.insert_or_assign( - "revCount", rev == nullRev ? 0 : getRevCount(settings, repoInfo, repoPath, rev)); + if (rev == nullRev) { + input.attrs.insert_or_assign("revCount", uint64_t(0)); + } else { + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoPath, rev)); + } } verifyCommit(input, repo); @@ -1267,7 +1289,7 @@ struct GitInputScheme : InputScheme for (auto & file : repoInfo.workdirInfo.dirtyFiles) { writeString("modified:", hashSink); writeString(file.abs(), hashSink); - dumpPath((*repoPath / file.rel()).string(), hashSink); + dumpPath(*repoPath / file.rel(), hashSink); } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 284620985f8a..930aeb050b5e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -1,6 +1,5 @@ #include "nix/store/filetransfer.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" #include "nix/store/store-api.hh" #include "nix/util/types.hh" #include "nix/util/url-parts.hh" @@ -13,7 +12,6 @@ #include #include -#include namespace nix::fetchers { @@ -41,16 +39,14 @@ struct GitArchiveInputScheme : InputScheme /* This ignores empty path segments for back-compat. Older versions used a tokenizeString here. */ auto path = url.pathSegments(/*skipEmpty=*/true) | std::ranges::to>(); - std::optional rev; - std::optional ref; - std::optional host_url; + Attrs attrs; auto size = path.size(); if (size == 3) { if (std::regex_match(path[2], revRegex)) - rev = path[2]; + attrs.insert_or_assign("rev", path[2]); else - ref = path[2]; + attrs.insert_or_assign("ref", path[2]); } else if (size > 3) { std::string rs; for (auto i = std::next(path.begin(), 2); i != path.end(); i++) { @@ -59,38 +55,30 @@ struct GitArchiveInputScheme : InputScheme rs += "/"; } } - ref = rs; + attrs.insert_or_assign("ref", rs); } else if (size < 2) throw BadURL("URL '%s' is invalid", url); for (auto & [name, value] : url.query) { if (name == "rev") { - if (rev) + if (attrs.contains(name)) throw BadURL("URL '%s' contains multiple commit hashes", url); - rev = value; + attrs.insert_or_assign("rev", value); } else if (name == "ref") { - if (ref) + if (attrs.contains(name)) throw BadURL("URL '%s' contains multiple branch/tag names", url); - ref = value; + attrs.insert_or_assign("ref", value); } else if (name == "host") - host_url = value; - // FIXME: barf on unsupported attributes + attrs.insert_or_assign("host", value); + else if (name == "narHash") + attrs.insert_or_assign("narHash", value); + else + throw BadURL("URL '%s' contains unknown parameter '%s'", url, name); } - Attrs attrs; attrs.insert_or_assign("type", std::string{schemeName()}); attrs.insert_or_assign("owner", path[0]); attrs.insert_or_assign("repo", path[1]); - if (rev) - attrs.insert_or_assign("rev", *rev); - if (ref) - attrs.insert_or_assign("ref", *ref); - if (host_url) - attrs.insert_or_assign("host", *host_url); - - auto narHash = url.query.find("narHash"); - if (narHash != url.query.end()) - attrs.insert_or_assign("narHash", narHash->second); return inputFromAttrs(settings, attrs); } diff --git a/src/libfetchers/include/nix/fetchers/attrs.hh b/src/libfetchers/include/nix/fetchers/attrs.hh index 8a21b8ddbf69..8eede58086e5 100644 --- a/src/libfetchers/include/nix/fetchers/attrs.hh +++ b/src/libfetchers/include/nix/fetchers/attrs.hh @@ -3,6 +3,8 @@ #include "nix/util/types.hh" #include "nix/util/hash.hh" +#include "nix/util/ref.hh" +#include "nix/util/fun.hh" #include @@ -12,7 +14,24 @@ namespace nix::fetchers { -typedef std::variant> Attr; +/** + * The resolved (non-lazy) subset of attribute value types. + */ +using ResolvedAttr = std::variant>; + +/** + * A deferred attribute computation. Wrapping in `ref<>` gives + * pointer-identity equality/ordering, which is correct: two lazy + * attrs are equal iff they are the same computation. + */ +struct LazyAttrComputation +{ + fun compute; +}; + +using LazyAttr = ref; + +using Attr = std::variant, LazyAttr>; /** * An `Attrs` can be thought of a JSON object restricted or simplified @@ -21,6 +40,16 @@ typedef std::variant> Attr; */ typedef std::map Attrs; +/** + * Force a potentially lazy attribute to its resolved value. + */ +ResolvedAttr forceAttr(const Attr & attr); + +/** + * Retrieve an attr, but only if it's a LazyAttr. + */ +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name); + Attrs jsonToAttrs(const nlohmann::json & json); nlohmann::json attrsToJSON(const Attrs & attrs); diff --git a/src/libfetchers/include/nix/fetchers/cache.hh b/src/libfetchers/include/nix/fetchers/cache.hh index 8cac076f1f20..fd21d7f384d3 100644 --- a/src/libfetchers/include/nix/fetchers/cache.hh +++ b/src/libfetchers/include/nix/fetchers/cache.hh @@ -12,6 +12,11 @@ namespace nix::fetchers { */ struct Cache { +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); +public: virtual ~Cache() {} /** diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 529da2c8bdb7..b68e2c0316f6 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -167,6 +167,8 @@ struct Settings : public Config private: + void anchor() override; + mutable Sync> _cache; mutable Sync> _tarballCache; diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 67fb29228ea8..6b7a9d83ad10 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -2,7 +2,8 @@ #include "nix/util/source-path.hh" -#include +#include +#include namespace nix { @@ -54,7 +55,10 @@ struct FilteringSourceAccessor : SourceAccessor std::shared_ptr getProvenance(const CanonPath & path) override; - void invalidateCache(const CanonPath & path) override; + void invalidateCache() override + { + next->invalidateCache(); + } /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` @@ -81,8 +85,8 @@ struct AllowListSourceAccessor : public FilteringSourceAccessor static ref create( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError); using FilteringSourceAccessor::FilteringSourceAccessor; diff --git a/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh b/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh index b59da391a056..444e0a648f14 100644 --- a/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh +++ b/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh @@ -38,9 +38,17 @@ struct Fetch void fetch( const std::string & content, const CanonPath & pointerFilePath, - StringSink & sink, + Sink & sink, std::function sizeCallback) const; std::vector fetchUrls(const std::vector & pointers) const; }; +struct LfsApiInfo +{ + std::string endpoint; + std::optional authHeader; +}; + +LfsApiInfo getLfsApi(ParsedURL url); + } // namespace nix::lfs diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index e462bc9e7b54..6ffc7372df0f 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -16,6 +16,10 @@ struct Settings; */ struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink { +private: + void anchor() override; + +public: /** * Flush builder and return a final Git hash. */ @@ -91,6 +95,9 @@ struct GitRepo static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path); + /* Drop all entries from the getCachedWorkdirInfo() cache. */ + static void invalidateWorkdirInfoCache(); + /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 652d5ce7976b..3fe96d8503bc 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -1,7 +1,7 @@ #include "nix/fetchers/input-cache.hh" +#include "nix/fetchers/git-utils.hh" #include "nix/fetchers/registry.hh" #include "nix/util/sync.hh" -#include "nix/util/source-path.hh" namespace nix::fetchers { @@ -66,6 +66,10 @@ struct InputCacheImpl : InputCache void clear() override { cache_.lock()->clear(); + /* The workdir info cache has the same "per evaluation" lifetime + as the input cache, so flush it here as well so that e.g. + `:reload` in `nix repl` picks up changes in git work trees. */ + GitRepo::invalidateWorkdirInfoCache(); } }; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index a53025dd7ff9..b7023efab609 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -7,8 +7,6 @@ #include "nix/util/environment-variables.hh" #include "nix/util/users.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" -#include "nix/util/tarfile.hh" #include "nix/store/store-api.hh" #include "nix/util/url-parts.hh" #include "nix/fetchers/fetch-settings.hh" @@ -30,10 +28,9 @@ static RunOptions hgOptions(OsStrings args) } // runProgram wrapper that uses hgOptions instead of stock RunOptions. -static std::string runHg(OsStrings args, const std::optional & input = {}) +static std::string runHg(OsStrings args) { RunOptions opts = hgOptions(std::move(args)); - opts.input = input; auto res = runProgram(std::move(opts)); @@ -243,26 +240,25 @@ struct MercurialInputScheme : InputScheme }), "\0"s); - auto actualPath = absPath(localPath); + /* FIXME: Check that the access to this path is allowed. */ + auto accessor = makeFSSourceAccessor(absPath(localPath)); PathFilter filter = [&](const std::string & p) -> bool { - assert(hasPrefix(p, actualPath.string())); - std::string file(p, actualPath.string().size() + 1); + auto cp = CanonPath(p); + auto st = accessor->lstat(cp); - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; + if (st.type == SourceAccessor::tDirectory) { + auto prefix = cp.rel() + "/"; auto i = files.lower_bound(prefix); return i != files.end() && hasPrefix(*i, prefix); } - return files.count(file); + return files.count(cp.rel()); }; return store.addToStore( input.getName(), - {getFSSourceAccessor(), CanonPath(actualPath.string())}, + {accessor, CanonPath::root}, ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, @@ -382,7 +378,7 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir / ".hg_archival.txt"); - auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir.string())}); + auto storePath = store.addToStore(name, {makeFSSourceAccessor(tmpDir), CanonPath::root}); Attrs infoAttrs({ {"revCount", (uint64_t) revCount}, diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 84f807db3b94..134fd496a7a1 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -7,10 +7,8 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'unity=on', - 'unity_size=1024', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 30f02c731f7f..9844c7a84b31 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -95,7 +95,7 @@ struct PathInputScheme : InputScheme query.erase("__final"); return ParsedURL{ .scheme = "path", - .path = splitString>(getStrAttr(input.attrs, "path"), "/"), + .path = pathToUrlPath(std::filesystem::path{getStrAttr(input.attrs, "path")}), .query = query, }; } diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 89990fe5f15c..82a65061deae 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,8 +1,8 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/registry.hh" #include "nix/fetchers/tarball.hh" +#include "nix/store/filetransfer.hh" #include "nix/util/users.hh" -#include "nix/store/globals.hh" #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index 55576506bde2..b1067a5d054e 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -50,15 +50,32 @@ headers += files('nix_api_flake.h') subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixflakec', - sources, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : prelink, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixflakec', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixflakec', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libflake-c/meson.options b/src/libflake-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libflake-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index 9ae3ec695154..344412b47fc0 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -10,6 +10,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -27,7 +28,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -41,6 +42,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index f1e7762af671..394380f80668 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -61,6 +61,7 @@ test( env : { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) diff --git a/src/libflake/config.cc b/src/libflake/config.cc index b7d5cd8999e5..6367d7b4ec42 100644 --- a/src/libflake/config.cc +++ b/src/libflake/config.cc @@ -11,14 +11,12 @@ #include #include #include -#include #include "nix/util/users.hh" #include "nix/util/config-global.hh" #include "nix/flake/settings.hh" #include "nix/flake/flake.hh" #include "nix/util/ansicolor.hh" -#include "nix/util/configuration.hh" #include "nix/util/file-system.hh" #include "nix/util/fmt.hh" #include "nix/util/logging.hh" diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index e9914bcb1035..b8838ab18e52 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -1,12 +1,8 @@ #include -#include #include #include -#include #include -#include #include -#include #include "nix/flake/flake-primops.hh" #include "nix/store/store-api.hh" @@ -22,12 +18,10 @@ #include "nix/expr/value.hh" #include "nix/fetchers/attrs.hh" #include "nix/fetchers/fetchers.hh" -#include "nix/util/configuration.hh" #include "nix/util/error.hh" #include "nix/util/experimental-features.hh" #include "nix/util/pos-idx.hh" #include "nix/util/pos-table.hh" -#include "nix/util/source-path.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" #include "nix/util/mounted-source-accessor.hh" @@ -92,7 +86,7 @@ PrimOp getFlake(const Settings & settings) .name = "__getFlake", .args = {"args"}, .doc = R"( - Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: + Fetch a flake from a flake reference or a path, and return its output attributes and some metadata. For example: ```nix (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix @@ -119,12 +113,13 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg for (const auto & [key, value] : attrs) { auto s = state.symbols.create(key); auto & vv = binds.alloc(s); + auto resolved = forceAttr(value); std::visit( overloaded{ [&vv, &state](const std::string & value) { vv.mkString(value, state.mem); }, [&vv](const uint64_t & value) { vv.mkInt(value); }, [&vv](const Explicit & value) { vv.mkBool(value.t); }}, - value); + resolved); } v.mkAttrs(binds); } diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 1dc030314e56..0ba33ac326f0 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include "nix/util/terminal.hh" #include "nix/util/ref.hh" @@ -40,6 +39,7 @@ #include "nix/fetchers/input-cache.hh" #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" @@ -58,7 +58,6 @@ #include "nix/util/logging.hh" #include "nix/util/pos-idx.hh" #include "nix/util/pos-table.hh" -#include "nix/util/position.hh" #include "nix/util/source-path.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" @@ -66,7 +65,6 @@ namespace nix { struct SourceAccessor; -using namespace flake; using namespace fetchers; namespace flake { @@ -915,8 +913,6 @@ LockedFlake lockFlake( CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), newLockFileS, commitMessage); - - flake.lockFilePath().invalidateCache(); } /* Rewriting the lockfile changed the top-level @@ -1038,11 +1034,24 @@ std::optional LockedFlake::getFingerprint(Store & store, const fetc *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); - /* Include revCount and lastModified because they're not - necessarily implied by the content fingerprint (e.g. for - tarball flakes) but can influence the evaluation result. */ - if (auto revCount = flake.lockedRef.input.getRevCount()) - *fingerprint += fmt(";revCount=%d", *revCount); + if (auto revCount = get(flake.lockedRef.input.attrs, "revCount")) { + if (std::get_if(revCount)) { + /* A lazy revCount is computed by the fetcher, so its + value is functionally determined by `rev`. We only + need to record its presence, not force its value. + + This means a lazy and a concrete revCount that would + resolve to the same value produce different + fingerprints, sacrificing some cache hits to avoid + the cost of forcing. */ + *fingerprint += ";hasRevCount"; + } else if (auto n = flake.lockedRef.input.getRevCount()) { + /* A concrete revCount comes from a lockfile or explicit + user input. The fetcher passes it through as-is, so + it can affect evaluation and must be fingerprinted. */ + *fingerprint += fmt(";revCount=%d", *n); + } + } if (auto lastModified = flake.lockedRef.input.getLastModified()) *fingerprint += fmt(";lastModified=%d", *lastModified); diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index 4fc97ac53a98..3af499ae6b80 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -148,7 +148,7 @@ std::pair parsePathFlakeRefWithFragment( // Save device to detect filesystem boundary dev_t device = lstat(path).st_dev; bool found = false; - while (path != "/") { + while (path.parent_path() != path) { if (pathExists(path / "flake.nix")) { found = true; break; @@ -173,7 +173,7 @@ std::pair parsePathFlakeRefWithFragment( auto flakeRoot = path; std::string subdir; - while (flakeRoot != "/") { + while (flakeRoot.parent_path() != flakeRoot) { if (pathExists(flakeRoot / ".git")) { auto parsedURL = ParsedURL{ .scheme = "git+file", @@ -272,7 +272,7 @@ std::pair parseFlakeRefWithFragment( bool isFlake, bool preserveRelativePaths) { - using namespace fetchers; + using namespace nix::fetchers; if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { return *res; diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 7156261ceb50..99720c9403a0 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -261,12 +261,4 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & v); } // namespace flake -void emitTreeAttrs( - EvalState & state, - const StorePath & storePath, - const fetchers::Input & input, - Value & v, - bool emptyRevFallback = false, - bool forceDirty = false); - } // namespace nix diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index dc5b79ffe510..c44ae77dff91 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -8,10 +8,7 @@ #include #include #include -#include #include -#include -#include #include #include #include @@ -32,10 +29,9 @@ #include "nix/flake/flakeref.hh" #include "nix/store/path.hh" #include "nix/util/ansicolor.hh" -#include "nix/util/configuration.hh" #include "nix/util/error.hh" #include "nix/util/fmt.hh" -#include "nix/util/hash.hh" +#include "nix/util/json-utils.hh" #include "nix/util/logging.hh" #include "nix/util/ref.hh" #include "nix/util/types.hh" diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 1e782dd42e58..1bf6f7e5fba6 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 156dcf19dce8..5b9106f455b9 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -42,15 +42,32 @@ headers = files( subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixmainc', - sources, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : prelink, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixmainc', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixmainc', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libmain-c/meson.options b/src/libmain-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libmain-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libmain-c/nix_api_main.cc b/src/libmain-c/nix_api_main.cc index 0ee965dc82ee..872949e423b9 100644 --- a/src/libmain-c/nix_api_main.cc +++ b/src/libmain-c/nix_api_main.cc @@ -1,5 +1,3 @@ -#include "nix_api_store.h" -#include "nix_api_store_internal.h" #include "nix_api_util.h" #include "nix_api_util_internal.h" diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index 17858d56f2e5..dd2faa913300 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -10,6 +10,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -27,7 +28,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -41,6 +42,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libmain/include/nix/main/common-args.hh b/src/libmain/include/nix/main/common-args.hh index d67fc2ad0c47..b20df3a99ec9 100644 --- a/src/libmain/include/nix/main/common-args.hh +++ b/src/libmain/include/nix/main/common-args.hh @@ -81,8 +81,8 @@ struct MixPrintJSON : virtual Args * This is a template to avoid accidental coercions from `string` to `json` in the caller, * to avoid mistakenly passing an already serialized JSON to this function. * - * It is not recommended to print a JSON string - see the JSON guidelines - * about extensibility, https://nix.dev/manual/nix/development/development/json-guideline.html - + * It is not recommended to print a JSON string - see the data modeling guidelines + * about extensibility, https://nix.dev/manual/nix/development/development/data-modeling.html - * but you _can_ print a sole JSON string by explicitly coercing it to * `nlohmann::json` first. */ diff --git a/src/libmain/include/nix/main/shared.hh b/src/libmain/include/nix/main/shared.hh index 19be9a04cf39..0416fc080343 100644 --- a/src/libmain/include/nix/main/shared.hh +++ b/src/libmain/include/nix/main/shared.hh @@ -28,7 +28,7 @@ void parseCmdLine( std::string version(); -void printVersion(const std::string & programName); +[[noreturn]] void printVersion(const std::string & programName); /** * Ugh. No better place to put this. diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index de4eb36c825e..a1c3768f2b15 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -50,6 +50,7 @@ void setLogFormat(const std::string & logFormatStr) void setLogFormat(const LogFormat & logFormat) { defaultLogFormat = logFormat; + logger->stop(); logger = makeDefaultLogger().release(); } diff --git a/src/libmain/meson.build b/src/libmain/meson.build index ac22e8d1dc0a..97b89b9fa79a 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 79fa6b08445a..baf29adcdd62 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,10 +1,12 @@ #include "nix/main/progress-bar.hh" #include "nix/util/terminal.hh" #include "nix/util/sync.hh" -#include "nix/store/store-api.hh" +#include "nix/util/signals.hh" +#include "nix/store/path.hh" +#include "nix/util/file-system.hh" #include "nix/store/names.hh" +#include "nix/util/util.hh" -#include #include #include #include @@ -13,17 +15,19 @@ namespace nix { +namespace { + static std::string_view getS(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tString); + if (n >= fields.size() || fields[n].type != Logger::Field::tString) + throw Error("could not get expected log field of type 'string' at index %d", n); return fields[n].s; } static uint64_t getI(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tInt); + if (n >= fields.size() || fields[n].type != Logger::Field::tInt) + throw Error("could not get expected log field of type 'int' at index %d", n); return fields[n].i; } @@ -34,10 +38,17 @@ static std::string_view storePathToName(std::string_view path) return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1); } -class ProgressBar : public Logger +static std::string_view storePathToNameWithoutDrvSuffix(std::string_view path) { -private: + auto res = storePathToName(path); + if (hasSuffix(res, drvExtension)) + res.remove_suffix(drvExtension.size()); + return res; +} +class ProgressBar final : public Logger +{ +private: struct ActInfo { std::string s, lastLine, phase; @@ -95,11 +106,30 @@ class ProgressBar : public Logger bool printBuildLogs = false; bool isTTY; + std::unique_ptr interruptCallback; + + void hideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25l"); + } + + void unhideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25h"); + } + public: ProgressBar(bool isTTY) : isTTY(isTTY) + , interruptCallback(createInterruptCallback([&]() { + pause(); + redraw("\rshutting down\e[K"); + })) { + hideCursorIfNeeded(); state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); @@ -125,7 +155,8 @@ class ProgressBar : public Logger auto state(state_.lock()); if (state->active) { state->active = false; - writeToStderr("\r\e[K"); + clearProgressDisplay(); + unhideCursorIfNeeded(); updateCV.notify_one(); quitCV.notify_one(); } @@ -144,12 +175,13 @@ class ProgressBar : public Logger } if (state->active) { - writeToStderr("\r\e[K"); + clearProgressDisplay(); /* Show activities that were previously only shown on the progress bar. Otherwise the user won't know what's happening. */ for (auto & act : state->activities) logActivity(*state, lvlNotice, act); + unhideCursorIfNeeded(); } } @@ -163,8 +195,10 @@ class ProgressBar : public Logger state->suspensions--; } if (state->suspensions == 0) { - if (state->active) - writeToStderr("\r\e[K"); + if (state->active) { + clearProgressDisplay(); + hideCursorIfNeeded(); + } state->haveUpdate = true; updateCV.notify_one(); } @@ -196,6 +230,7 @@ class ProgressBar : public Logger void log(State & state, Verbosity lvl, std::string_view s) noexcept { if (state.active) { + invalidateRedrawCache(); writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); draw(state); } else { @@ -230,9 +265,7 @@ class ProgressBar : public Logger logActivity(*state, lvl, *i); if (type == actBuild) { - std::string name(storePathToName(getS(fields, 0))); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); auto machineName = getS(fields, 1); if (machineName != "") @@ -251,9 +284,7 @@ class ProgressBar : public Logger } if (type == actPostBuildHook) { - auto name = storePathToName(getS(fields, 0)); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); i->name = DrvName(name).name; } @@ -409,6 +440,17 @@ class ProgressBar : public Logger } } + void invalidateRedrawCache() + { + *lastOutput_.lock() = ""; + } + + void clearProgressDisplay() + { + invalidateRedrawCache(); + writeToStderr("\r\e[K"); + } + std::chrono::milliseconds draw(State & state) noexcept { auto nextWakeup = std::chrono::milliseconds::max(); @@ -644,6 +686,7 @@ class ProgressBar : public Logger { auto state(state_.lock()); if (state->active) { + invalidateRedrawCache(); std::cerr << "\r\e[K"; Logger::writeToStdout(s); draw(*state); @@ -657,8 +700,11 @@ class ProgressBar : public Logger auto state(state_.lock()); if (!state->active) return {}; + invalidateRedrawCache(); std::cerr << fmt("\r\e[K%s ", msg); + unhideCursorIfNeeded(); auto s = trim(readLine(getStandardInput(), true)); + hideCursorIfNeeded(); if (s.size() != 1) return {}; draw(*state); @@ -671,6 +717,8 @@ class ProgressBar : public Logger } }; +} // namespace + std::unique_ptr makeProgressBar() { return std::make_unique(isTTY()); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 62fd0a41f84e..3fae87761e15 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -6,12 +6,10 @@ #include "nix/store/store-open.hh" #include "nix/store/gc-store.hh" #include "nix/main/loggers.hh" -#include "nix/main/progress-bar.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" #include -#include #include #include @@ -23,6 +21,9 @@ #ifndef _WIN32 # include #endif +#ifdef __APPLE__ +# include +#endif #ifdef __linux__ # include #endif @@ -65,8 +66,7 @@ void printMissing(ref store, const MissingPaths & missing, Verbosity lvl) else printMsg(lvl, "these %d derivations will be built:", missing.willBuild.size()); auto sorted = store->topoSortPaths(missing.willBuild); - reverse(sorted.begin(), sorted.end()); - for (auto & i : sorted) + for (auto & i : sorted | std::views::reverse) printMsg(lvl, " %s", store->printStorePath(i)); } @@ -134,15 +134,26 @@ void bumpFileLimit() if (getrlimit(RLIMIT_NOFILE, &limit) != 0) return; - if (limit.rlim_cur < limit.rlim_max) { - // Some software misbehaves really bad when we try to raise the - // limit to RLIM_INFINITY, so cap the limit at the 1048576 limit used - // by the daemon. - // - // GNU patch < 2.8 crashes with **** out of memory, which breaks in nixpkgs darwin bootstrap tools. - // This was fixed in: - // https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 - limit.rlim_cur = std::min(limit.rlim_max, rlim_t(1048576)); + rlim_t target = limit.rlim_max; + +# ifdef __APPLE__ + // On macOS the hard limit is typically RLIM_INFINITY, but + // setting rlim_cur to that causes problems: child processes + // (e.g. GNU patch in the Nix sandbox) may allocate memory + // proportional to the fd limit and OOM. Use the kernel's + // per-process file limit instead, which is the effective cap. + // + // GNU patch < 2.8 crashes with **** out of memory, which breaks in nixpkgs darwin bootstrap tools. + // This was fixed in: + // https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 + int maxfiles; + size_t len = sizeof(maxfiles); + if (sysctlbyname("kern.maxfilesperproc", &maxfiles, &len, nullptr, 0) == 0) + target = maxfiles; +# endif + + if (limit.rlim_cur < target) { + limit.rlim_cur = target; // Ignore errors, this is best effort. setrlimit(RLIMIT_NOFILE, &limit); } @@ -171,10 +182,10 @@ void initNix(bool loadConfig) if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD"); - /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ + /* Install a dummy NIX_SIG_MULTI_INT handler for use with pthread_kill(). */ act.sa_handler = sigHandler; - if (sigaction(SIGUSR1, &act, 0)) - throw SysError("handling SIGUSR1"); + if (sigaction(NIX_SIG_MULTI_INT, &act, 0)) + throw SysError("handling multiplexed interrupt"); /* Reset SIGQUIT to its default disposition. In particular, this unregisters any crash handler installed by `sentry_init()` @@ -349,12 +360,12 @@ void printVersion(const std::string & programName) std::cout << "System type: " << settings.thisSystem << "\n"; std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; - std::cout << "System configuration file: " << nixConfFile() << "\n"; + std::cout << "System configuration file: " << os_string_to_string(nixConfFile().native()) << "\n"; std::cout << "User configuration files: " << os_string_to_string(ExecutablePath{.directories = nixUserConfFiles()}.render()) << "\n"; std::cout << "Store directory: " << resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir << "\n"; - std::cout << "State directory: " << settings.nixStateDir << "\n"; + std::cout << "State directory: " << os_string_to_string(settings.nixStateDir.native()) << "\n"; } throw Exit(); } diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index e72d6c67eb57..4099e6931316 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -45,15 +45,32 @@ headers += files('nix_api_store_internal.h') subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixstorec', - sources, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : prelink, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixstorec', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixstorec', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libstore-c/meson.options b/src/libstore-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libstore-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index 0ce37e44c012..ecfc278cf660 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -8,6 +8,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -25,7 +26,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -37,6 +38,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libstore-test-support/derived-path.cc b/src/libstore-test-support/derived-path.cc index cb1d23ac5208..c27edc95ef9b 100644 --- a/src/libstore-test-support/derived-path.cc +++ b/src/libstore-test-support/derived-path.cc @@ -1,6 +1,4 @@ -#include - -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libstore-test-support/https-store.cc b/src/libstore-test-support/https-store.cc index 79548f61f3fe..549e3ce36172 100644 --- a/src/libstore-test-support/https-store.cc +++ b/src/libstore-test-support/https-store.cc @@ -60,7 +60,7 @@ void HttpsBinaryCacheStoreTest::SetUp() openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"}); // clang-format on -#ifndef _WIN32 /* FIXME: Can't yet start processes on windows */ +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ auto args = serverArgs(); serverPid = startProcess( [&] { @@ -91,7 +91,9 @@ void HttpsBinaryCacheStoreTest::SetUp() void HttpsBinaryCacheStoreTest::TearDown() { +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ serverPid.kill(); +#endif delTmpDir.reset(); testFileTransferSettings.reset(); } diff --git a/src/libstore-test-support/include/nix/store/tests/derived-path.hh b/src/libstore-test-support/include/nix/store/tests/derived-path.hh index b3b43474a914..7d93af08fd1f 100644 --- a/src/libstore-test-support/include/nix/store/tests/derived-path.hh +++ b/src/libstore-test-support/include/nix/store/tests/derived-path.hh @@ -9,36 +9,35 @@ #include "nix/store/tests/outputs-spec.hh" namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/https-store.hh b/src/libstore-test-support/include/nix/store/tests/https-store.hh index 3aa2cd3455e9..9d4db058e5a4 100644 --- a/src/libstore-test-support/include/nix/store/tests/https-store.hh +++ b/src/libstore-test-support/include/nix/store/tests/https-store.hh @@ -43,7 +43,7 @@ class TestHttpBinaryCacheStoreConfig : public HttpBinaryCacheStoreConfig { public: TestHttpBinaryCacheStoreConfig(ParsedURL url, const Store::Config::Params & params) - : StoreConfig(params) + : StoreConfig(params, FilePathType::Unix) , HttpBinaryCacheStoreConfig(url, params) { } @@ -65,7 +65,9 @@ protected: std::filesystem::path tmpDir, cacheDir; std::filesystem::path caCert, caKey, serverCert, serverKey; std::filesystem::path clientCert, clientKey; +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ Pid serverPid; +#endif uint16_t port = 8443; std::shared_ptr localCacheStore; diff --git a/src/libstore-test-support/include/nix/store/tests/libstore.hh b/src/libstore-test-support/include/nix/store/tests/libstore.hh index d79b5531232a..740cc2dcf427 100644 --- a/src/libstore-test-support/include/nix/store/tests/libstore.hh +++ b/src/libstore-test-support/include/nix/store/tests/libstore.hh @@ -10,6 +10,29 @@ namespace nix { +/** + * Scoped guard that enables an experimental feature for the lifetime + * of the object. + */ +struct EnableExperimentalFeature +{ + std::set previous; + + explicit EnableExperimentalFeature(std::string_view feature) + : previous(experimentalFeatureSettings.experimentalFeatures.get()) + { + experimentalFeatureSettings.set("extra-experimental-features", std::string{feature}); + } + + ~EnableExperimentalFeature() + { + experimentalFeatureSettings.experimentalFeatures.assign(previous); + } + + EnableExperimentalFeature(const EnableExperimentalFeature &) = delete; + EnableExperimentalFeature & operator=(const EnableExperimentalFeature &) = delete; +}; + class LibStoreTest : public virtual ::testing::Test { public: diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index bb9e5a3038fd..df48a7469ea8 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -22,13 +22,10 @@ public: }; ~nix_api_store_test_base() override - { - if (exists(std::filesystem::path{nixDir})) { - for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) { - std::filesystem::permissions(path, std::filesystem::perms::owner_all); - } - std::filesystem::remove_all(nixDir); - } + try { + nix::deletePath(nixDir); + } catch (...) { + nix::ignoreExceptionInDestructor(); } std::string nixDir; @@ -79,23 +76,23 @@ class nix_api_store_test : public nix_api_store_test_base { public: nix_api_store_test() - : nix_api_store_test_base{} - { - init_local_store(); - }; + : nix_api_store_test_base{} {}; - ~nix_api_store_test() override + void SetUp() override { - nix_store_free(store); +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif + store = open_local_store(); } - Store * store; - -protected: - void init_local_store() + ~nix_api_store_test() override { - store = open_local_store(); + if (store) + nix_store_free(store); } + + Store * store = nullptr; }; } // namespace nixC diff --git a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh index 5bbcc734086a..6cdb0a60ef96 100644 --- a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh +++ b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/outputs-spec.hh" @@ -9,12 +9,11 @@ #include "nix/store/tests/path.hh" namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/path.hh b/src/libstore-test-support/include/nix/store/tests/path.hh index ff80b1299a04..e2ecd23ca5f0 100644 --- a/src/libstore-test-support/include/nix/store/tests/path.hh +++ b/src/libstore-test-support/include/nix/store/tests/path.hh @@ -18,18 +18,17 @@ void showValue(const StorePath & p, std::ostream & os); } // namespace nix namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/protocol.hh b/src/libstore-test-support/include/nix/store/tests/protocol.hh index 563c8cfb6c4e..543facead0ec 100644 --- a/src/libstore-test-support/include/nix/store/tests/protocol.hh +++ b/src/libstore-test-support/include/nix/store/tests/protocol.hh @@ -122,4 +122,10 @@ public: VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) \ VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) +/// Has to be a `BufferedSink` for handshake. +struct NullBufferedSink : BufferedSink +{ + void writeUnbuffered(std::string_view data) override {} +}; + } // namespace nix diff --git a/src/libstore-test-support/libstore-network.cc b/src/libstore-test-support/libstore-network.cc index 8aa047bdd609..800dc17ed9f5 100644 --- a/src/libstore-test-support/libstore-network.cc +++ b/src/libstore-test-support/libstore-network.cc @@ -5,6 +5,8 @@ #ifdef __linux__ # include "nix/util/file-system.hh" # include "nix/util/linux-namespaces.hh" +# include +# include # include # include # include @@ -34,8 +36,9 @@ static void enterNetworkNamespace() if (!fd) throw SysError("cannot open IP socket for loopback interface"); + using namespace std::string_view_literals; struct ::ifreq ifr = {}; - strcpy(ifr.ifr_name, "lo"); + std::ranges::copy("lo"sv, ifr.ifr_name); ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; if (::ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) throw SysError("cannot set loopback interface flags"); diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index ba2d7583cd90..79f3c8235d2e 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore-test-support/outputs-spec.cc b/src/libstore-test-support/outputs-spec.cc index d5128a8bd91a..947f02eb539f 100644 --- a/src/libstore-test-support/outputs-spec.cc +++ b/src/libstore-test-support/outputs-spec.cc @@ -3,10 +3,13 @@ #include namespace rc { -using namespace nix; -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using nix::OutputsSpec; + using nix::StorePathName; + using nix::StringSet; + return gen::mapcat( gen::inRange(0, std::variant_size_v), [](uint8_t n) -> Gen { switch (n) { diff --git a/src/libstore-test-support/path.cc b/src/libstore-test-support/path.cc index 1459203104c6..bca404cde455 100644 --- a/src/libstore-test-support/path.cc +++ b/src/libstore-test-support/path.cc @@ -1,12 +1,8 @@ -#include // Needed by rapidcheck on Darwin -#include +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include -#include "nix/store/path-regex.hh" -#include "nix/store/store-api.hh" - #include "nix/util/tests/hash.hh" #include "nix/store/tests/path.hh" @@ -20,7 +16,6 @@ void showValue(const StorePath & p, std::ostream & os) } // namespace nix namespace rc { -using namespace nix; Gen storePathChar() { @@ -52,18 +47,19 @@ Gen storePathChar() gen::inRange(0, 10 + 2 * 26 + 6)); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { - return gen::construct( + return gen::construct( gen::suchThat(gen::container(storePathChar()), [](const std::string & s) { return !(s == "" || s == "." || s == ".." || s.starts_with(".-") || s.starts_with("..-")); })); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { - return gen::construct( - gen::arbitrary(), gen::apply([](StorePathName n) { return n.name; }, gen::arbitrary())); + return gen::construct( + gen::arbitrary(), + gen::apply([](nix::StorePathName n) { return n.name; }, gen::arbitrary())); } } // namespace rc diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc index b7e8f83f9e7c..a1d8ddee6412 100644 --- a/src/libstore-tests/build-result.cc +++ b/src/libstore-tests/build-result.cc @@ -75,25 +75,13 @@ INSTANTIATE_TEST_SUITE_P( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index d63cbeeeed2c..5afde2bdf7e2 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -1,12 +1,9 @@ -#include - #include #include #include "nix/util/json-utils.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" -#include "nix/store/build-result.hh" #include "nix/store/tests/protocol.hh" #include "nix/util/tests/characterization.hh" @@ -108,69 +105,6 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( - drvOutput, - "drv-output", - (std::tuple{ - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - })) - -CHARACTERIZATION_TEST( - realisation, - "realisation", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - -READ_CHARACTERIZATION_TEST( - realisation_with_deps, - "realisation-with-deps", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - CHARACTERIZATION_TEST( vector, "vector", diff --git a/src/libstore-tests/data/build-result/success.json b/src/libstore-tests/data/build-result/success.json index 4baadb547758..ec3479d5564b 100644 --- a/src/libstore-tests/data/build-result/success.json +++ b/src/libstore-tests/data/build-result/success.json @@ -1,14 +1,10 @@ { "builtOutputs": { "bar": { - "dependentRealisations": {}, - "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "signatures": [] }, "foo": { - "dependentRealisations": {}, - "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [] } diff --git a/src/libstore-tests/data/common-protocol/realisation.bin b/src/libstore-tests/data/common-protocol/realisation.bin index 44885cbdafda..cbd942ed477a 100644 Binary files a/src/libstore-tests/data/common-protocol/realisation.bin and b/src/libstore-tests/data/common-protocol/realisation.bin differ diff --git a/src/libstore-tests/data/common-protocol/realisation.json b/src/libstore-tests/data/common-protocol/realisation.json index 034d620306f0..ad1ae3ab5f4e 100644 --- a/src/libstore-tests/data/common-protocol/realisation.json +++ b/src/libstore-tests/data/common-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json new file mode 100644 index 000000000000..44006012fb8c --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json @@ -0,0 +1,31 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + } + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json new file mode 100644 index 000000000000..0706e1b64aa8 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json @@ -0,0 +1,27 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "exportReferencesGraph": "refs /19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "export-ref-subpath", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json new file mode 100644 index 000000000000..3f15525837fa --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json @@ -0,0 +1,9 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] +] diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json new file mode 100644 index 000000000000..2bfe56fc8d44 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json @@ -0,0 +1,28 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json new file mode 100644 index 000000000000..453efae619de --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json @@ -0,0 +1,19 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "exportReferencesGraph": "refs /nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ], + "name": "export-ref-subpath", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json new file mode 100644 index 000000000000..7bc4cad36c51 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json @@ -0,0 +1,23 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + } + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "perOutput": {} + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json new file mode 100644 index 000000000000..3e92638f9d6f --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json @@ -0,0 +1,34 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "__json": "{\"exportReferencesGraph\":{\"refs\":[\"/19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo\"]}}" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "export-ref-subpath-sa", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "exportReferencesGraph": { + "refs": [ + "/19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo" + ] + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json new file mode 100644 index 000000000000..3f15525837fa --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json @@ -0,0 +1,9 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] +] diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json new file mode 100644 index 000000000000..b3970f4a3788 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json @@ -0,0 +1,20 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "perOutput": {} + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json new file mode 100644 index 000000000000..d9befe3c96e7 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json @@ -0,0 +1,26 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "__json": "{\"exportReferencesGraph\":{\"refs\":[\"/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo\"]}}" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ], + "name": "export-ref-subpath-sa", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "exportReferencesGraph": { + "refs": [ + "/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo" + ] + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json new file mode 100644 index 000000000000..ed9b9bc48a56 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json @@ -0,0 +1,17 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "FOO": "bar" + }, + "inputs": [], + "name": "no-inputs", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json new file mode 100644 index 000000000000..8ade9a6d90a9 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json @@ -0,0 +1,20 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "FOO": "bar" + }, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "no-inputs", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json new file mode 100644 index 000000000000..fe51488c7066 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json @@ -0,0 +1 @@ +[] diff --git a/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json new file mode 100644 index 000000000000..cb57937ff53b --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json @@ -0,0 +1,25 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": {}, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "resolution-failure", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json new file mode 100644 index 000000000000..fe51488c7066 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json @@ -0,0 +1 @@ +[] diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json new file mode 100644 index 000000000000..5c318ff84084 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json @@ -0,0 +1,33 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "DEP1_DEV": "/nix/store/j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev", + "DEP1_OUT": "prefix-/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out-suffix", + "DEP2": "/nix/store/i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out", + "i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out", + "j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev" + ], + "name": "with-inputs", + "outputs": { + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "dep1out": "/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out", + "nested": { + "dep2": "before /nix/store/i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out after" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json new file mode 100644 index 000000000000..6b39e5fc067f --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json @@ -0,0 +1,46 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "DEP1_DEV": "/1r6jj1sjvjx4d1jqlnak71d8vzysdl2gyz2m01spjf31nhr1g55f", + "DEP1_OUT": "prefix-/10qd59vslmmj9w7hx8lpr0yxazwvhf3gfrnvbbc2s9071rhl3l55-suffix", + "DEP2": "/1zwa38661ns8faagbp31qzc8vx94hxsxcncs7hvcxhjvbcfyv9wb" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "with-inputs", + "outputs": { + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "dep1out": "/10qd59vslmmj9w7hx8lpr0yxazwvhf3gfrnvbbc2s9071rhl3l55", + "nested": { + "dep2": "before /1zwa38661ns8faagbp31qzc8vx94hxsxcncs7hvcxhjvbcfyv9wb after" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json new file mode 100644 index 000000000000..a15a69869a30 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json @@ -0,0 +1,23 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv", + "output": "dev" + }, + "j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev" + ], + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out" + ], + [ + { + "drvPath": "h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv", + "output": "out" + }, + "i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out" + ] +] diff --git a/src/libstore-tests/data/dummy-store/one-flat-file.json b/src/libstore-tests/data/dummy-store/one-flat-file.json index 804bbf07da6f..f7563abe6a68 100644 --- a/src/libstore-tests/data/dummy-store/one-flat-file.json +++ b/src/libstore-tests/data/dummy-store/one-flat-file.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/dummy-store/one-realisation.json b/src/libstore-tests/data/dummy-store/one-realisation.json index b5c8b8c5621d..576de0838177 100644 --- a/src/libstore-tests/data/dummy-store/one-realisation.json +++ b/src/libstore-tests/data/dummy-store/one-realisation.json @@ -1,8 +1,7 @@ { "buildTrace": { - "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv": { "out": { - "dependentRealisations": {}, "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [] } diff --git a/src/libstore-tests/data/nar-info/json-3/impure.json b/src/libstore-tests/data/nar-info/json-3/impure.json new file mode 100644 index 000000000000..eb6d4d48b83b --- /dev/null +++ b/src/libstore-tests/data/nar-info/json-3/impure.json @@ -0,0 +1,31 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "compression": "xz", + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "downloadSize": 4029176, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ], + "storeDir": "/nix/store", + "ultimate": true, + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", + "version": 3 +} diff --git a/src/libstore-tests/data/nar-info/json-3/pure.json b/src/libstore-tests/data/nar-info/json-3/pure.json new file mode 100644 index 000000000000..32fc916489e1 --- /dev/null +++ b/src/libstore-tests/data/nar-info/json-3/pure.json @@ -0,0 +1,14 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/empty_impure.json b/src/libstore-tests/data/path-info/json-3/empty_impure.json new file mode 100644 index 000000000000..47d5030318b3 --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/empty_impure.json @@ -0,0 +1,12 @@ +{ + "ca": null, + "deriver": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/empty_pure.json b/src/libstore-tests/data/path-info/json-3/empty_pure.json new file mode 100644 index 000000000000..6cc032782a4b --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/empty_pure.json @@ -0,0 +1,8 @@ +{ + "ca": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/impure.json b/src/libstore-tests/data/path-info/json-3/impure.json new file mode 100644 index 000000000000..f7ec907dcef3 --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/impure.json @@ -0,0 +1,27 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ], + "storeDir": "/nix/store", + "ultimate": true, + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/pure.json b/src/libstore-tests/data/path-info/json-3/pure.json new file mode 100644 index 000000000000..32fc916489e1 --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/pure.json @@ -0,0 +1,14 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/realisation/simple-fingerprint.txt b/src/libstore-tests/data/realisation/simple-fingerprint.txt new file mode 100644 index 000000000000..c9700924c28d --- /dev/null +++ b/src/libstore-tests/data/realisation/simple-fingerprint.txt @@ -0,0 +1 @@ +{"key":{"drvPath":"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv","outputName":"foo"},"value":{"outPath":"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}} \ No newline at end of file diff --git a/src/libstore-tests/data/realisation/simple-sig.json b/src/libstore-tests/data/realisation/simple-sig.json new file mode 100644 index 000000000000..084dbf0ba7ee --- /dev/null +++ b/src/libstore-tests/data/realisation/simple-sig.json @@ -0,0 +1,4 @@ +{ + "keyName": "test-key", + "sig": "WsjxK4/EI74COtzMDI+VCjQU9O4FydK0+YeY1CE5hUevogd+T+CPvNXza7oog3GTMS+ZlBwsC2S3ppwusKnJDg==" +} diff --git a/src/libstore-tests/data/realisation/simple.json b/src/libstore-tests/data/realisation/simple.json index 2ccb1e721198..1e4760b56827 100644 --- a/src/libstore-tests/data/realisation/simple.json +++ b/src/libstore-tests/data/realisation/simple.json @@ -1,6 +1,10 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } } diff --git a/src/libstore-tests/data/realisation/unkeyed-simple.json b/src/libstore-tests/data/realisation/unkeyed-simple.json new file mode 100644 index 000000000000..2cf61d215206 --- /dev/null +++ b/src/libstore-tests/data/realisation/unkeyed-simple.json @@ -0,0 +1,4 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] +} diff --git a/src/libstore-tests/data/realisation/unkeyed-with-signature.json b/src/libstore-tests/data/realisation/unkeyed-with-signature.json new file mode 100644 index 000000000000..c51c6127a765 --- /dev/null +++ b/src/libstore-tests/data/realisation/unkeyed-with-signature.json @@ -0,0 +1,9 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] +} diff --git a/src/libstore-tests/data/realisation/with-dependent-realisations.json b/src/libstore-tests/data/realisation/with-dependent-realisations.json deleted file mode 100644 index a58e0d7fe1c2..000000000000 --- a/src/libstore-tests/data/realisation/with-dependent-realisations.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "dependentRealisations": { - "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" - }, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [] -} diff --git a/src/libstore-tests/data/realisation/with-signature-fingerprint.txt b/src/libstore-tests/data/realisation/with-signature-fingerprint.txt new file mode 100644 index 000000000000..bc67f80df0ef --- /dev/null +++ b/src/libstore-tests/data/realisation/with-signature-fingerprint.txt @@ -0,0 +1 @@ +{"key":{"drvPath":"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv","outputName":"foo"},"value":{"outPath":"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}} \ No newline at end of file diff --git a/src/libstore-tests/data/realisation/with-signature-sig.json b/src/libstore-tests/data/realisation/with-signature-sig.json new file mode 100644 index 000000000000..41c243b61ba4 --- /dev/null +++ b/src/libstore-tests/data/realisation/with-signature-sig.json @@ -0,0 +1,4 @@ +{ + "keyName": "test-key", + "sig": "ZE30UrOh26/EYvUaczjJtz6c4WcRYke6IEhDNN5TiFRF6Uu+H8lHofponlhfZxDDrCUlfstS/vtv3Xm2F6M/AA==" +} diff --git a/src/libstore-tests/data/realisation/with-signature-unstructured.json b/src/libstore-tests/data/realisation/with-signature-unstructured.json new file mode 100644 index 000000000000..7952e43697b0 --- /dev/null +++ b/src/libstore-tests/data/realisation/with-signature-unstructured.json @@ -0,0 +1,12 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] + } +} diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json index 3270f1cdebf4..a9efc97ca9e6 100644 --- a/src/libstore-tests/data/realisation/with-signature.json +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -1,8 +1,15 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] + } } diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin new file mode 100644 index 000000000000..fd745215c033 Binary files /dev/null and b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin differ diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json new file mode 100644 index 000000000000..9c97678dadad --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.bin b/src/libstore-tests/data/serve-protocol/build-result-2.8.bin new file mode 100644 index 000000000000..fa072599578a Binary files /dev/null and b/src/libstore-tests/data/serve-protocol/build-result-2.8.bin differ diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.json b/src/libstore-tests/data/serve-protocol/build-result-2.8.json new file mode 100644 index 000000000000..9c97678dadad --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/build-result-2.8.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin b/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin new file mode 100644 index 000000000000..5be0b15a3456 Binary files /dev/null and b/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin differ diff --git a/src/libstore-tests/data/serve-protocol/drv-output-2.8.json b/src/libstore-tests/data/serve-protocol/drv-output-2.8.json new file mode 100644 index 000000000000..6afbf20e83a8 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/drv-output-2.8.json @@ -0,0 +1,10 @@ +[ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "quux" + } +] diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/realisation-2.8.bin new file mode 100644 index 000000000000..7d1fd462f998 Binary files /dev/null and b/src/libstore-tests/data/serve-protocol/realisation-2.8.bin differ diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.json b/src/libstore-tests/data/serve-protocol/realisation-2.8.json new file mode 100644 index 000000000000..d6dde7327670 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/realisation-2.8.json @@ -0,0 +1,19 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] + } +} diff --git a/src/libstore-tests/data/serve-protocol/realisation.bin b/src/libstore-tests/data/serve-protocol/realisation.bin index 44885cbdafda..cbd942ed477a 100644 Binary files a/src/libstore-tests/data/serve-protocol/realisation.bin and b/src/libstore-tests/data/serve-protocol/realisation.bin differ diff --git a/src/libstore-tests/data/serve-protocol/realisation.json b/src/libstore-tests/data/serve-protocol/realisation.json index 034d620306f0..ad1ae3ab5f4e 100644 --- a/src/libstore-tests/data/serve-protocol/realisation.json +++ b/src/libstore-tests/data/serve-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin new file mode 100644 index 000000000000..fce3a25dd881 Binary files /dev/null and b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin differ diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json new file mode 100644 index 000000000000..9c8277922c5a --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json @@ -0,0 +1,13 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] +} diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json index 0f593f4248d1..55bbd81bf14a 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json @@ -9,7 +9,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -23,6 +23,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json index 9c1fa3134d49..5c0f436e1f60 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json @@ -11,7 +11,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": { @@ -27,11 +27,17 @@ ], "registrationTime": null, "signatures": [ - "fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "fake-sig-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "fake-sig-2", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin new file mode 100644 index 000000000000..fd745215c033 Binary files /dev/null and b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin differ diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json new file mode 100644 index 000000000000..9c97678dadad --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin new file mode 100644 index 000000000000..11bec3e6e3b7 Binary files /dev/null and b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin differ diff --git a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json new file mode 100644 index 000000000000..1d566b13bb1a --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json @@ -0,0 +1,39 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "cpuSystem": 604000000, + "cpuUser": 500000000, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin new file mode 100644 index 000000000000..5be0b15a3456 Binary files /dev/null and b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin differ diff --git a/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.json new file mode 100644 index 000000000000..6afbf20e83a8 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.json @@ -0,0 +1,10 @@ +[ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "quux" + } +] diff --git a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.bin new file mode 100644 index 000000000000..7d1fd462f998 Binary files /dev/null and b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.bin differ diff --git a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json new file mode 100644 index 000000000000..d6dde7327670 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json @@ -0,0 +1,19 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] + } +} diff --git a/src/libstore-tests/data/worker-protocol/realisation.bin b/src/libstore-tests/data/worker-protocol/realisation.bin index 44885cbdafda..cbd942ed477a 100644 Binary files a/src/libstore-tests/data/worker-protocol/realisation.bin and b/src/libstore-tests/data/worker-protocol/realisation.bin differ diff --git a/src/libstore-tests/data/worker-protocol/realisation.json b/src/libstore-tests/data/worker-protocol/realisation.json index 034d620306f0..ad1ae3ab5f4e 100644 --- a/src/libstore-tests/data/worker-protocol/realisation.json +++ b/src/libstore-tests/data/worker-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin new file mode 100644 index 000000000000..fce3a25dd881 Binary files /dev/null and b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin differ diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json new file mode 100644 index 000000000000..9c8277922c5a --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json @@ -0,0 +1,13 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] +} diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json index 9cc53c6804e9..fb2fbeef0c1d 100644 --- a/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json +++ b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json @@ -9,7 +9,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -23,6 +23,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json index 427c286ddfbc..e62fd7a236bb 100644 --- a/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json @@ -10,7 +10,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -26,6 +26,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json index 2c377145a10c..8bffd4410f6c 100644 --- a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json @@ -10,7 +10,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": true, - "version": 2 + "version": 3 }, { "ca": null, @@ -24,12 +24,18 @@ ], "registrationTime": 23423, "signatures": [ - "fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "fake-sig-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "fake-sig-2", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": { @@ -48,6 +54,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json index 7f0f62f876c0..ce95439d6e01 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json @@ -1,8 +1,7 @@ { "buildTrace": { - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", "signatures": [] } @@ -31,7 +30,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json index 93f4fb22a620..c596ae7f88fa 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json @@ -1,8 +1,7 @@ { "buildTrace": { - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", "signatures": [] } @@ -31,7 +30,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json index 617984a27eb6..ff2d1a627b84 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json @@ -1,11 +1,16 @@ { "buildTrace": { - "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "am42vl3rp8w29f5hyp9ljckxll50icpl-root-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", "signatures": [] } + }, + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "out": { + "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", + "signatures": [] + } } }, "config": { @@ -31,7 +36,29 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 + } + }, + "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": { + "contents": { + "contents": "I am the dependency output", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "narSize": 144, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json index a63d6243fae3..61f4d8a760a2 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json @@ -1,15 +1,13 @@ { "buildTrace": { - "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "am42vl3rp8w29f5hyp9ljckxll50icpl-root-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", "signatures": [] } }, - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", "signatures": [] } @@ -38,7 +36,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } }, "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": { @@ -60,9 +58,30 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, - "derivations": {} + "derivations": { + "am42vl3rp8w29f5hyp9ljckxll50icpl-root-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [ + "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out" + ] + }, + "name": "root-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } } diff --git a/src/libstore-tests/data/worker-substitution/single/substituter.json b/src/libstore-tests/data/worker-substitution/single/substituter.json index f22d4c7dfbf1..55e16a6d38f8 100644 --- a/src/libstore-tests/data/worker-substitution/single/substituter.json +++ b/src/libstore-tests/data/worker-substitution/single/substituter.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/with-dep/substituter.json b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json index 3f2994dfb455..765340031047 100644 --- a/src/libstore-tests/data/worker-substitution/with-dep/substituter.json +++ b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } }, "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": { @@ -47,7 +47,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 16988613c633..bb7ad28f3d78 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -1,14 +1,11 @@ #include -#include -#include "nix/util/experimental-features.hh" #include "nix/store/derivations.hh" #include "nix/store/derived-path.hh" #include "nix/store/derivation-options.hh" #include "nix/store/globals.hh" #include "nix/store/parsed-derivations.hh" #include "nix/util/types.hh" -#include "nix/util/json-utils.hh" #include "nix/store/tests/libstore.hh" #include "nix/util/tests/json-characterization.hh" @@ -307,7 +304,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) }; DerivationOptions advancedAttributes_structuredAttrs_defaults = { - .outputChecks = std::map::OutputChecks>{}, + .outputChecks = std::map::OutputChecks, std::less<>>{}, .unsafeDiscardReferences = {}, .passAsFile = {}, .exportReferencesGraph = {}, @@ -352,7 +349,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) { DerivationOptions expected = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"dev", DerivationOptions::OutputChecks{ .maxSize = 789, @@ -384,7 +381,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) { // Delete all keys but "dev" in options.outputChecks auto * outputChecksMapP = - std::get_if::OutputChecks>>( + std::get_if::OutputChecks, std::less<>>>( &options.outputChecks); ASSERT_TRUE(outputChecksMapP); auto & outputChecksMap = *outputChecksMapP; @@ -405,7 +402,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) DerivationOptions advancedAttributes_structuredAttrs_ia = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"out", DerivationOptions::OutputChecks{ .allowedReferences = std::set>{pathFoo}, @@ -447,7 +444,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) DerivationOptions advancedAttributes_structuredAttrs_ca = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"out", DerivationOptions::OutputChecks{ .allowedReferences = std::set>{placeholderFoo}, @@ -489,24 +486,137 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) {"rainbow", "uid-range", "ca-derivations"}); }; -#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2) \ - TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json) \ - { \ - nix::readJsonTest>( \ - *this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ - } \ - TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json) \ - { \ - nix::readJsonTest>( \ - *this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ +#define TEST_JSON_OPTIONS(FIXUTURE, INPUT, VAR, VAR2) \ + TEST_F(FIXUTURE, DerivationOptions_##INPUT##_##VAR##_from_json) \ + { \ + nix::readJsonTest>(*this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ + } \ + TEST_F(FIXUTURE, DerivationOptions_##INPUT##_##VAR##_to_json) \ + { \ + nix::readJsonTest>(*this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ } -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, all_set, ia) -TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, all_set, ca) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_defaults, structuredAttrs_defaults) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ia) -TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ca) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, defaults, defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, all_set, ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, SingleDerivedPath, all_set, ca) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_defaults, structuredAttrs_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_all_set, structuredAttrs_ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_all_set, structuredAttrs_ca) + +/** + * `DerivationOptions` versions of the IA fixtures, used to + * exercise the `StorePath` JSON serializers. The IA test data files are + * reused as-is (see comment near the test invocations). + */ +static const StorePath spFoo{"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}, + spFooDev{"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}, spBar{"r5cff30838majxk5mp3ip2diffi8vpaj-bar"}, + spBarDev{"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}, spBarDrv{"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}; + +static const DerivationOptions advancedAttributes_sp_defaults = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, +}; + +static const DerivationOptions advancedAttributes_sp_all_set = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + .allowedReferences = std::set>{spFoo}, + .disallowedReferences = std::set>{spBar, OutputName{"dev"}}, + .allowedRequisites = std::set>{spFooDev, OutputName{"bin"}}, + .disallowedRequisites = std::set>{spBarDev}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph{ + {"refs1", {spFoo}}, + {"refs2", {spBarDrv}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + +static const DerivationOptions advancedAttributes_sp_structuredAttrs_defaults = { + .outputChecks = std::map::OutputChecks, std::less<>>{}, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, +}; + +static const DerivationOptions advancedAttributes_sp_structuredAttrs_all_set = { + .outputChecks = + std::map::OutputChecks, std::less<>>{ + {"out", + DerivationOptions::OutputChecks{ + .allowedReferences = std::set>{spFoo}, + .allowedRequisites = std::set>{spFooDev, OutputName{"bin"}}, + }}, + {"bin", + DerivationOptions::OutputChecks{ + .disallowedReferences = std::set>{spBar, OutputName{"dev"}}, + .disallowedRequisites = std::set>{spBarDev}, + }}, + {"dev", + DerivationOptions::OutputChecks{ + .maxSize = 789, + .maxClosureSize = 5909, + }}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = + { + {"refs1", {spFoo}}, + {"refs2", {spBarDrv}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + +/** + * Same JSON characterization tests, but for `DerivationOptions`. + * + * Since `DrvRef` and `DrvRef` (when only + * the `Opaque` case is used) JSON-encode identically, the IA test data + * files can be reused as-is. + */ +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, defaults, sp_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, all_set, sp_all_set) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, structuredAttrs_defaults, sp_structuredAttrs_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, structuredAttrs_all_set, sp_structuredAttrs_all_set) #undef TEST_JSON_OPTIONS diff --git a/src/libstore-tests/derivation-parser-bench.cc b/src/libstore-tests/derivation-parser-bench.cc index f0aa721cb7a9..5bc4cec9f38f 100644 --- a/src/libstore-tests/derivation-parser-bench.cc +++ b/src/libstore-tests/derivation-parser-bench.cc @@ -1,13 +1,12 @@ #include #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" -#include "nix/util/experimental-features.hh" #include "nix/util/tests/test-data.hh" #include "nix/store/store-open.hh" #include #include -using namespace nix; +namespace nix { // Benchmark parsing real derivation files static void BM_ParseRealDerivationFile(benchmark::State & state, const std::string & filename) @@ -54,3 +53,5 @@ BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, hello, (getUnitTestData() / "deriv BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string()); BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string()); BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string()); + +} // namespace nix diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index 528f0793d668..e31b3e85442f 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -185,44 +185,41 @@ struct DerivationJsonAtermTest : DerivationTest, MAKE_TEST_P(DerivationJsonAtermTest); -Derivation makeSimpleDrv() -{ - Derivation drv; - drv.name = "simple-derivation"; - drv.inputSrcs = { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { - .map = - { - { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - { - .value = - { - "cat", - "dog", - }, - }, - }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = StringPairs{ - { - "BIG_BAD", - "WOLF", - }, - }; - return drv; -} - -INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values(makeSimpleDrv())); +INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values([]() { + Derivation drv; + drv.name = "simple-derivation"; + drv.inputSrcs = { + StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = + { + { + StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + .value = + { + "cat", + "dog", + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = StringPairs{ + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; + }())); struct DynDerivationJsonAtermTest : DynDerivationTest, JsonCharacterizationTest, diff --git a/src/libstore-tests/derivation/invariants.cc b/src/libstore-tests/derivation/invariants.cc index 115d5bc4bb1f..e825655d66f4 100644 --- a/src/libstore-tests/derivation/invariants.cc +++ b/src/libstore-tests/derivation/invariants.cc @@ -6,8 +6,6 @@ #include "nix/store/dummy-store-impl.hh" #include "nix/util/tests/json-characterization.hh" -#include "derivation/test-support.hh" - namespace nix { class FillInOutputPathsTest : public LibStoreTest, public JsonCharacterizationTest diff --git a/src/libstore-tests/derivations.cc b/src/libstore-tests/derivations.cc new file mode 100644 index 000000000000..60b86f571205 --- /dev/null +++ b/src/libstore-tests/derivations.cc @@ -0,0 +1,365 @@ +#include +#include + +#include "nix/store/derivations.hh" +#include "nix/store/derivation-options.hh" +#include "nix/store/downstream-placeholder.hh" +#include "nix/store/parsed-derivations.hh" +#include "nix/store/tests/libstore.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +class TryResolveTest : public LibStoreTest, public CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "derivation" / "try-resolve"; + +public: + + /** + * A simple in-memory *derived* build trace. (e.g. we have + * `SingleDerivedPath::Built` not `StorePath` keys, because that is + * how the `tryResolved` callback works.) + * + * For a real-world build trace, derived build traces are much more + * work to get correct, and should be at most a caching layer atop + * an underlying source-of-truth build trace. (This is as described + * in the manual.) + * + * However, this is just for some simple in-memory unit tests, with + * tiny amounts of data we can hand-review, so it's fine. The point + * is unit testing `tryResolve`, not unit testing "deep queries" + * from a non-derived build trace (what is done in + * `outputs-query.cc`) anyways --- that would be a separate unit + * test. + */ + struct BuildTrace + { + std::map dict; + + bool operator==(const BuildTrace &) const = default; + }; + +protected: + + EnableExperimentalFeature caFeature{"ca-derivations"}; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } + + /** + * Just here because we do this a few times in a tests. + */ + static DerivationOutput caFloatingOutput() + { + return DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}; + } + + /** + * Build a callback from a BuildTrace lookup. + */ + static auto makeCallback(const BuildTrace & table) + { + return + [&table](ref drvPath, const std::string & outputName) -> std::optional { + if (auto p = get(table.dict, SingleDerivedPath::Built{drvPath, outputName})) + return *p; + return std::nullopt; + }; + } + + /** + * Helper for the `exportReferencesGraph` + placeholder subpath tests. + */ + void exportRefGraphSubpathTest(std::string_view stem, const Derivation & drv, const StructuredAttrs * parsed); + + /** + * Checkpoint before/buildTrace/after and assert the resolved derivation + * matches expected. + */ + void resolveExpect( + std::string_view stem, const Derivation & drv, const BuildTrace & buildTrace, const BasicDerivation & expected) + { + nix::checkpointJson(*this, std::string{stem} + "-before", drv); + nix::checkpointJson(*this, std::string{stem} + "-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + ASSERT_TRUE(resolved); + + nix::checkpointJson(*this, std::string{stem} + "-after", *resolved); + + EXPECT_EQ(*resolved, expected); + } +}; + +} // namespace nix + +JSON_IMPL(nix::TryResolveTest::BuildTrace); + +namespace nlohmann { + +using nix::SingleDerivedPath; +using nix::StorePath; +using nix::TryResolveTest; + +void adl_serializer::to_json(json & j, const TryResolveTest::BuildTrace & t) +{ + j = t.dict; +} + +TryResolveTest::BuildTrace adl_serializer::from_json(const json & j) +{ + return TryResolveTest::BuildTrace{ + .dict = j.get>(), + }; +} + +} // namespace nlohmann + +namespace nix { + +TEST_F(TryResolveTest, noInputs) +{ + resolveExpect( + "no-inputs", + [&] { + Derivation drv; + drv.name = "no-inputs"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.env = {{"FOO", "bar"}}; + return drv; + }(), + {}, + [&] { + BasicDerivation expected; + expected.name = "no-inputs"; + expected.platform = "x86_64-linux"; + expected.builder = "/bin/bash"; + expected.outputs = {{"out", caFloatingOutput()}}; + expected.env = {{"FOO", "bar"}}; + return expected; + }()); +} + +TEST_F(TryResolveTest, withInputs) +{ + // dep1 has two outputs (out, dev), dep2 has one (out) + StorePath dep1DrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv"}; + StorePath dep1OutPath{"f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out"}; + StorePath dep1DevPath{"j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev"}; + StorePath dep2DrvPath{"h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv"}; + StorePath dep2OutPath{"i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out"}; + + auto placeholder1Out = DownstreamPlaceholder::unknownCaOutput(dep1DrvPath, "out").render(); + auto placeholder1Dev = DownstreamPlaceholder::unknownCaOutput(dep1DrvPath, "dev").render(); + auto placeholder2Out = DownstreamPlaceholder::unknownCaOutput(dep2DrvPath, "out").render(); + + DerivationOutputs multiOutputs = { + {"out", caFloatingOutput()}, + {"dev", caFloatingOutput()}, + }; + + resolveExpect( + "with-inputs", + [&] { + Derivation drv; + drv.name = "with-inputs"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = multiOutputs; + drv.inputDrvs = { + .map = { + {dep1DrvPath, {.value = {"out", "dev"}}}, + {dep2DrvPath, {.value = {"out"}}}, + }}; + drv.env = { + {"DEP1_OUT", "prefix-" + placeholder1Out + "-suffix"}, + {"DEP1_DEV", placeholder1Dev}, + {"DEP2", placeholder2Out}, + }; + drv.structuredAttrs = StructuredAttrs{{ + {"dep1out", placeholder1Out}, + {"nested", nlohmann::json::object({{"dep2", "before " + placeholder2Out + " after"}})}, + }}; + return drv; + }(), + {.dict{ + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep1DrvPath), + .output = "out", + }, + dep1OutPath, + }, + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep1DrvPath), + .output = "dev", + }, + dep1DevPath, + }, + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep2DrvPath), + .output = "out", + }, + dep2OutPath, + }, + }}, + [&] { + BasicDerivation expected; + expected.name = "with-inputs"; + expected.platform = "x86_64-linux"; + expected.builder = "/bin/bash"; + expected.outputs = multiOutputs; + expected.inputSrcs = {dep1OutPath, dep1DevPath, dep2OutPath}; + expected.env = { + {"DEP1_OUT", "prefix-" + store->printStorePath(dep1OutPath) + "-suffix"}, + {"DEP1_DEV", store->printStorePath(dep1DevPath)}, + {"DEP2", store->printStorePath(dep2OutPath)}, + }; + expected.structuredAttrs = StructuredAttrs{{ + {"dep1out", store->printStorePath(dep1OutPath)}, + {"nested", + nlohmann::json::object({{"dep2", "before " + store->printStorePath(dep2OutPath) + " after"}})}, + }}; + return expected; + }()); +} + +TEST_F(TryResolveTest, resolutionFailure) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + + Derivation drv; + drv.name = "resolution-failure"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + + BuildTrace buildTrace; + + checkpointJson(*this, "resolution-failure-before", drv); + checkpointJson(*this, "resolution-failure-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + EXPECT_FALSE(resolved); +} + +/** + * Test that `derivationOptionsFromStructuredAttrs` can parse + * `exportReferencesGraph` entries that reference a CA derivation output + * with a subpath appended (e.g. `${dep}/foo`), and that the parsed + * options resolve correctly. + * + * Regression test for #15003: the placeholder + subpath was not found + * in the placeholder map (exact-match only), and the fallback to + * `toStorePath()` failed because placeholders don't start with + * `/nix/store/`. + */ +void TryResolveTest::exportRefGraphSubpathTest( + std::string_view stem, const Derivation & drv, const StructuredAttrs * parsed) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + StorePath depOutPath{"f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out"}; + + nix::checkpointJson(*this, std::string{stem} + "-before", drv); + + auto options = derivationOptionsFromStructuredAttrs(*store, drv.inputDrvs, drv.env, parsed, true); + + nix::checkpointJson(*this, std::string{stem} + "-before-options", options); + + SingleDerivedPath expectedPath = SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(depDrvPath), + .output = "out", + }; + + ASSERT_EQ(options.exportReferencesGraph.size(), 1); + auto it = options.exportReferencesGraph.find("refs"); + ASSERT_NE(it, options.exportReferencesGraph.end()); + ASSERT_EQ(it->second.size(), 1); + EXPECT_EQ(*it->second.begin(), expectedPath); + + // Also test that resolution works + BuildTrace buildTrace{.dict{ + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(depDrvPath), + .output = "out", + }, + depOutPath, + }, + }}; + + nix::checkpointJson(*this, std::string{stem} + "-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + ASSERT_TRUE(resolved); + + nix::checkpointJson(*this, std::string{stem} + "-resolved", *resolved); + + // Re-parse options from the resolved derivation, where placeholders + // have been substituted with concrete store paths. + auto resolvedOptions = derivationOptionsFromStructuredAttrs( + *store, + /* inputDrvs */ {}, + resolved->env, + resolved->structuredAttrs ? &*resolved->structuredAttrs : nullptr, + true); + + nix::checkpointJson(*this, std::string{stem} + "-resolved-options", resolvedOptions); + + EXPECT_EQ( + resolvedOptions.exportReferencesGraph, + (decltype(resolvedOptions.exportReferencesGraph){ + {"refs", std::set{SingleDerivedPath::Opaque{depOutPath}}}})); +} + +TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); + + Derivation drv; + drv.name = "export-ref-subpath"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.env = { + {"exportReferencesGraph", "refs " + placeholder + "/foo"}, + }; + + exportRefGraphSubpathTest("export-ref-subpath", drv, nullptr); +} + +TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath_structuredAttrs) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); + + Derivation drv; + drv.name = "export-ref-subpath-sa"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.structuredAttrs = StructuredAttrs{{ + {"exportReferencesGraph", nlohmann::json::object({{"refs", nlohmann::json::array({placeholder + "/foo"})}})}, + }}; + drv.env = { + {std::string{StructuredAttrs::envVarName}, nlohmann::json(drv.structuredAttrs->structuredAttrs).dump()}, + }; + + exportRefGraphSubpathTest("export-ref-subpath-sa", drv, &*drv.structuredAttrs); +} + +} // namespace nix diff --git a/src/libstore-tests/derived-path.cc b/src/libstore-tests/derived-path.cc index 70e789c0c4fd..46297c9a30db 100644 --- a/src/libstore-tests/derived-path.cc +++ b/src/libstore-tests/derived-path.cc @@ -1,5 +1,3 @@ -#include - #include #include @@ -94,8 +92,6 @@ TEST_F(DerivedPathTest, built_built_xp) MissingExperimentalFeature); } -#ifndef COVERAGE - /* TODO: Disabled due to the following error: path '00000000000000000000000000000000-0^0' is not a valid store path: @@ -115,8 +111,6 @@ RC_GTEST_FIXTURE_PROP(DerivedPathTest, prop_round_rip, (const DerivedPath & o)) RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store), xpSettings)); } -#endif - /* ---------------------------------------------------------------------------- * JSON * --------------------------------------------------------------------------*/ diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index e3422fe585f8..4626fc31f28d 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -6,6 +6,7 @@ #include "nix/store/globals.hh" #include "nix/store/realisation.hh" +#include "nix/store/tests/libstore.hh" #include "nix/util/tests/json-characterization.hh" namespace nix { @@ -27,30 +28,67 @@ class DummyStoreTest : public virtual CharacterizationTest } }; +TEST(DummyStore, storeDir_default) +{ + DummyStoreConfig config{{}}; + EXPECT_EQ(config.storeDir, "/nix/store"); +} + +TEST(DummyStore, storeDir_absolutePath) +{ + DummyStoreConfig config{{{"store", "/my/store"}}}; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(DummyStore, storeDir_canonicalized) +{ + DummyStoreConfig config{{{"store", "/my//store/../store"}}}; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(DummyStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW((DummyStoreConfig{{{"store", "my/store"}}}), UsageError); +} + +TEST(DummyStore, storeDir_empty_rejected) +{ + EXPECT_THROW((DummyStoreConfig{{{"store", ""}}}), UsageError); +} + +TEST(DummyStore, getStateDir_default) +{ + // DummyStore uses the base StoreConfig::getStateDir which returns + // the global settings.nixStateDir + DummyStoreConfig config{{}}; + EXPECT_EQ(config.getStateDir(), settings.nixStateDir); +} + TEST(DummyStore, realisation_read) { initLibStore(/*loadConfig=*/false); + EnableExperimentalFeature enableCA{"ca-derivations"}; + auto store = [] { auto cfg = make_ref(StoreReference::Params{}); cfg->readOnly = false; return cfg->openDummyStore(); }(); - auto drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16); + StorePath drvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}; auto outputName = "foo"; - EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr); + EXPECT_EQ(store->queryRealisation({drvPath, outputName}), nullptr); UnkeyedRealisation value{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }; - store->buildTrace.insert({drvHash, {{outputName, value}}}); + store->buildTrace.insert({drvPath, {{outputName, value}}}); - auto value2 = store->queryRealisation({drvHash, outputName}); + auto value2 = store->queryRealisation({drvPath, outputName}); ASSERT_TRUE(value2); EXPECT_EQ(*value2, value); @@ -131,10 +169,7 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { [&] { auto store = writeCfg->openDummyStore(); store->buildTrace.insert_or_assign( - Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, std::map{ { "out", diff --git a/src/libstore-tests/filetransfer-request.cc b/src/libstore-tests/filetransfer-request.cc new file mode 100644 index 000000000000..b89bb7ed0bc1 --- /dev/null +++ b/src/libstore-tests/filetransfer-request.cc @@ -0,0 +1,19 @@ +#include + +#include "nix/store/filetransfer.hh" + +namespace nix { + +TEST(FileTransferRequest, displayUriStripsUserinfo) +{ + FileTransferRequest req(VerbatimURL{std::string{"https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"}}); + // uri itself is untouched (used for CURLOPT_URL, result.urls, cache keys). + EXPECT_EQ(req.uri.to_string(), "https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"); + // displayUri() drops the userinfo for diagnostics. + EXPECT_EQ(req.displayUri(), "https://example.org:8443/path/file.toml?x=1"); + + FileTransferRequest plain(VerbatimURL{std::string{"https://example.org/file"}}); + EXPECT_EQ(plain.displayUri(), "https://example.org/file"); +} + +} // namespace nix diff --git a/src/libstore-tests/filetransfer-retry.cc b/src/libstore-tests/filetransfer-retry.cc new file mode 100644 index 000000000000..4cd1d2dd0cd6 --- /dev/null +++ b/src/libstore-tests/filetransfer-retry.cc @@ -0,0 +1,155 @@ +#include + +#include + +#include "nix/store/filetransfer-impl.hh" + +namespace nix { + +// --------------------------------------------------------------------------- +// Deterministic (jitter=false) — table-driven +// --------------------------------------------------------------------------- + +struct RetryDelayCase +{ + std::string description; + uint32_t attempt; + uint32_t baseMs; + uint32_t ceilMs; + std::optional retryAfterMs; + long long expectedMs; // std::chrono::milliseconds::rep +}; + +class RetryDelayDeterministicTest : public ::testing::TestWithParam +{}; + +TEST_P(RetryDelayDeterministicTest, yieldsExpectedDelay) +{ + auto & p = GetParam(); + std::mt19937 rng{0}; + auto result = computeRetryDelayMs( + {.attempt = p.attempt, .baseMs = p.baseMs, .ceilMs = p.ceilMs, .retryAfterMs = p.retryAfterMs, .jitter = false}, + rng); + EXPECT_EQ(result.count(), p.expectedMs); +} + +INSTANTIATE_TEST_SUITE_P( + computeRetryDelayMs, + RetryDelayDeterministicTest, + ::testing::Values( + // --- exponential growth --- + RetryDelayCase{"exponential_attempt1", 1, 100, 60000, {}, 100}, + RetryDelayCase{"exponential_attempt2", 2, 100, 60000, {}, 200}, + RetryDelayCase{"exponential_attempt3", 3, 100, 60000, {}, 400}, + RetryDelayCase{"exponential_attempt4", 4, 100, 60000, {}, 800}, + RetryDelayCase{"exponential_attempt5", 5, 100, 60000, {}, 1600}, + + // --- ceiling --- + RetryDelayCase{"ceil_at_5000", 10, 100, 5000, {}, 5000}, // 100*2^9=51200 > 5000 + RetryDelayCase{"base_exceeds_ceil", 1, 10000, 5000, {}, 5000}, // base > ceil on attempt 1 + RetryDelayCase{"base_equals_ceil_a1", 1, 500, 500, {}, 500}, + RetryDelayCase{"base_equals_ceil_a5", 5, 500, 500, {}, 500}, + RetryDelayCase{"base_equals_ceil_a10", 10, 500, 500, {}, 500}, + RetryDelayCase{"ceil_zero_a1", 1, 100, 0, {}, 0}, + RetryDelayCase{"ceil_zero_a5", 5, 100, 0, {}, 0}, + + // --- Retry-After interaction --- + RetryDelayCase{"retry_after_exceeds_computed", 1, 100, 60000, 3000, 3000}, + RetryDelayCase{"retry_after_under_computed", 4, 100, 60000, 500, 800}, + RetryDelayCase{"retry_after_above_ceil", 1, 100, 60000, 120000, 120000}, + RetryDelayCase{"retry_after_zero", 1, 100, 60000, 0, 100}, + RetryDelayCase{"retry_after_equals_ceil", 1, 100, 60000, 60000, 60000}, + RetryDelayCase{"retry_after_equals_computed", 3, 100, 60000, 400, 400}, // max(400,400)=400 + + // --- rate-limit base (429/503 path) --- + RetryDelayCase{"rate_limit_a1", 1, 5000, 60000, {}, 5000}, + RetryDelayCase{"rate_limit_a2", 2, 5000, 60000, {}, 10000}, + RetryDelayCase{"rate_limit_a3", 3, 5000, 60000, {}, 20000}, + RetryDelayCase{"rate_limit_a4", 4, 5000, 60000, {}, 40000}, + RetryDelayCase{"rate_limit_a5", 5, 5000, 60000, {}, 60000}, // ceiled + RetryDelayCase{"rate_limit_retry_after_under", 2, 5000, 60000, 3000, 10000}, + + // --- attempt boundaries --- + RetryDelayCase{"attempt_zero", 0, 100, 60000, {}, 100}, // shift=0, same as attempt 1 + RetryDelayCase{"attempt_one", 1, 100, 60000, {}, 100}, + + // --- shift clamp boundary --- + // base=1: 1<<31 = 2147483648, ceiled at UINT32_MAX + RetryDelayCase{"shift_clamp_at_32", 32, 1, UINT32_MAX, {}, 2147483648LL}, + // attempt 33 → shift still 31 (clamped), same result + RetryDelayCase{"shift_clamp_at_33", 33, 1, UINT32_MAX, {}, 2147483648LL}, + + // --- zero / minimal base --- + RetryDelayCase{"zero_base", 1, 0, 60000, {}, 0}, + RetryDelayCase{"base_one_a1", 1, 1, 60000, {}, 1}, + RetryDelayCase{"base_one_a2", 2, 1, 60000, {}, 2}, + RetryDelayCase{"base_one_a11", 11, 1, 60000, {}, 1024}, + + // --- integer extremes --- + RetryDelayCase{"uint_max_base_small_ceil", 1, UINT32_MAX, 1000, {}, 1000}, + RetryDelayCase{"uint_max_ceil", 1, 100, UINT32_MAX, {}, 100}, + + // --- overflow guard --- + RetryDelayCase{"overflow_attempt100", 100, 1000, 60000, {}, 60000}, + RetryDelayCase{"overflow_attempt1M", 1000000, 100, 60000, {}, 60000}), + [](const auto & info) { return info.param.description; }); + +// --------------------------------------------------------------------------- +// Jitter — loop-based, kept as individual TEST()s +// --------------------------------------------------------------------------- + +TEST(computeRetryDelayMs, jitter_stays_in_bounds) +{ + std::mt19937 rng{42}; + for (int i = 0; i < 1000; i++) { + auto ms = computeRetryDelayMs({.attempt = 3, .baseMs = 100, .ceilMs = 60000, .jitter = true}, rng).count(); + EXPECT_LE(ms, 400); // 100 * 2^2 = 400 + } +} + +TEST(computeRetryDelayMs, jitter_with_retry_after_floor) +{ + // computed = 100, Retry-After = 5000 → jitter in [5000, 5100] + std::mt19937 rng{42}; + for (int i = 0; i < 1000; i++) { + auto ms = computeRetryDelayMs( + {.attempt = 1, .baseMs = 100, .ceilMs = 60000, .retryAfterMs = 5000, .jitter = true}, rng) + .count(); + EXPECT_GE(ms, 5000); + EXPECT_LE(ms, 5100); + } +} + +TEST(computeRetryDelayMs, zero_base_jitter_returns_zero) +{ + std::mt19937 rng{0}; + // ceiled == 0 → early return, no distribution created + EXPECT_EQ(computeRetryDelayMs({.attempt = 1, .baseMs = 0, .ceilMs = 60000, .jitter = true}, rng).count(), 0); +} + +TEST(computeRetryDelayMs, jitter_with_ceil_zero) +{ + std::mt19937 rng{42}; + // ceilMs=0 → backoff=0 → ceiling<=floor early return, rng untouched + EXPECT_EQ(computeRetryDelayMs({.attempt = 1, .baseMs = 100, .ceilMs = 0, .jitter = true}, rng).count(), 0); +} + +TEST(computeRetryDelayMs, jitter_ceiled_one) +{ + // Smallest non-trivial jitter range: [0, 1] + std::mt19937 rng{42}; + bool sawZero = false, sawOne = false; + for (int i = 0; i < 100; i++) { + auto ms = computeRetryDelayMs({.attempt = 1, .baseMs = 1, .ceilMs = 1, .jitter = true}, rng).count(); + EXPECT_LE(ms, 1); + EXPECT_GE(ms, 0); + if (ms == 0) + sawZero = true; + if (ms == 1) + sawOne = true; + } + EXPECT_TRUE(sawZero); + EXPECT_TRUE(sawOne); +} + +} // namespace nix diff --git a/src/libstore-tests/http-binary-cache-store.cc b/src/libstore-tests/http-binary-cache-store.cc index 8495db4b5abb..74f3b93cd3df 100644 --- a/src/libstore-tests/http-binary-cache-store.cc +++ b/src/libstore-tests/http-binary-cache-store.cc @@ -3,12 +3,22 @@ #include "nix/store/http-binary-cache-store.hh" #include "nix/store/tests/https-store.hh" -#include "nix/util/fs-sink.hh" namespace nix { using Authority = ParsedURL::Authority; +TEST(HttpBinaryCacheStore, storeDir_absolutePath) +{ + HttpBinaryCacheStoreConfig config{parseURL("https://example.com"), {{"store", "/my/store"}}}; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(HttpBinaryCacheStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW(HttpBinaryCacheStoreConfig(parseURL("https://example.com"), {{"store", "my/store"}}), UsageError); +} + TEST(HttpBinaryCacheStore, constructConfig) { HttpBinaryCacheStoreConfig config{ diff --git a/src/libstore-tests/legacy-ssh-store.cc b/src/libstore-tests/legacy-ssh-store.cc index 35543cf40064..926f4dc38741 100644 --- a/src/libstore-tests/legacy-ssh-store.cc +++ b/src/libstore-tests/legacy-ssh-store.cc @@ -4,6 +4,20 @@ namespace nix { +TEST(LegacySSHStore, storeDir_absolutePath) +{ + LegacySSHStoreConfig config{ + ParsedURL::Authority::parse("localhost"), + {{"store", "/my/store"}}, + }; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(LegacySSHStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW(LegacySSHStoreConfig(ParsedURL::Authority::parse("localhost"), {{"store", "my/store"}}), UsageError); +} + TEST(LegacySSHStore, constructConfig) { LegacySSHStoreConfig config( diff --git a/src/libstore-tests/local-binary-cache-store.cc b/src/libstore-tests/local-binary-cache-store.cc index 295976488b5f..9ae1519a6fc1 100644 --- a/src/libstore-tests/local-binary-cache-store.cc +++ b/src/libstore-tests/local-binary-cache-store.cc @@ -4,6 +4,18 @@ namespace nix { +TEST(LocalBinaryCacheStore, storeDir_absolutePath) +{ + LocalBinaryCacheStoreConfig config{std::filesystem::path("/foo/bar/baz"), {{"store", "/my/store"}}}; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(LocalBinaryCacheStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW( + LocalBinaryCacheStoreConfig(std::filesystem::path("/foo/bar/baz"), {{"store", "my/store"}}), UsageError); +} + TEST(LocalBinaryCacheStore, constructConfig) { LocalBinaryCacheStoreConfig config{std::filesystem::path("/foo/bar/baz"), {}}; diff --git a/src/libstore-tests/local-fs-store.cc b/src/libstore-tests/local-fs-store.cc new file mode 100644 index 000000000000..22f7ca4084f4 --- /dev/null +++ b/src/libstore-tests/local-fs-store.cc @@ -0,0 +1,72 @@ +#include + +#include "nix/store/local-fs-store.hh" +#include "nix/store/globals.hh" + +namespace nix { + +namespace { + +/** + * Concrete subclass of `LocalFSStoreConfig` for testing, since + * `LocalFSStoreConfig` is abstract (`openStore()` is pure virtual). + */ +struct TestLocalFSStoreConfig : LocalFSStoreConfig +{ + TestLocalFSStoreConfig(const Params & params) + : StoreConfig(params, FilePathType::Native) + , LocalFSStoreConfig(params) + { + } + + TestLocalFSStoreConfig(const std::filesystem::path & path, const Params & params) + : StoreConfig(params, FilePathType::Native) + , LocalFSStoreConfig(path, params) + { + } + + ref openStore() const override + { + unreachable(); + } +}; + +} // namespace + +TEST(LocalFSStoreConfig, getStateDir_default) +{ + TestLocalFSStoreConfig config{{}}; + // Default stateDir should be the global settings value + EXPECT_EQ(config.getStateDir(), settings.nixStateDir); +} + +TEST(LocalFSStoreConfig, getStateDir_withRoot) +{ + std::filesystem::path root = +#ifdef _WIN32 + "C:\\"; +#else + "/"; +#endif + root /= "foo"; + root /= "bar"; + auto expectedStateDir = root / "nix" / "var" / "nix"; + TestLocalFSStoreConfig config{root, {}}; + EXPECT_EQ(config.getStateDir(), expectedStateDir); +} + +TEST(LocalFSStoreConfig, getStateDir_explicitSetting) +{ + std::filesystem::path stateDir = +#ifdef _WIN32 + "C:\\"; +#else + "/"; +#endif + stateDir /= "custom"; + stateDir /= "state"; + TestLocalFSStoreConfig config{{{"state", stateDir.string()}}}; + EXPECT_EQ(config.getStateDir(), stateDir); +} + +} // namespace nix diff --git a/src/libstore-tests/local-overlay-store.cc b/src/libstore-tests/local-overlay-store.cc index c207564255f3..eb88d879ce20 100644 --- a/src/libstore-tests/local-overlay-store.cc +++ b/src/libstore-tests/local-overlay-store.cc @@ -6,24 +6,52 @@ namespace nix { TEST(LocalOverlayStore, constructConfig_rootQueryParam) { +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif LocalOverlayStoreConfig config{ "", { { "root", - "/foo/bar", + std::string{root}, }, }, }; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalOverlayStore, constructConfig_rootPath) { - LocalOverlayStoreConfig config{"/foo/bar", {}}; +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif + LocalOverlayStoreConfig config{std::string{root}, {}}; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); +} + +TEST(LocalOverlayStore, upperLayer_notOverridden) +{ + LocalOverlayStoreConfig config{"", {}}; + EXPECT_FALSE(config.upperLayer.isOverridden()); +} + +TEST(LocalOverlayStore, upperLayer_overridden) +{ + LocalOverlayStoreConfig config{ + "", + { + {"upper-layer", "/some/upper"}, + }, + }; + EXPECT_TRUE(config.upperLayer.isOverridden()); + EXPECT_EQ(config.upperLayer.get(), std::filesystem::path{"/some/upper"}); } } // namespace nix diff --git a/src/libstore-tests/local-store.cc b/src/libstore-tests/local-store.cc index 554d42efc811..06da1b008e07 100644 --- a/src/libstore-tests/local-store.cc +++ b/src/libstore-tests/local-store.cc @@ -10,26 +10,60 @@ namespace nix { +TEST(LocalStore, storeDir_absolutePath) +{ + std::filesystem::path storeDir = +#ifdef _WIN32 + "C:\\"; +#else + "/"; +#endif + storeDir /= "nix"; + storeDir /= "store"; + LocalStoreConfig config{"", {{"store", storeDir.string()}}}; + EXPECT_EQ(config.storeDir, storeDir.string()); +} + +TEST(LocalStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW(LocalStoreConfig("", {{"store", (std::filesystem::path{"nix"} / "store").string()}}), UsageError); +} + +TEST(LocalStore, storeDir_empty_rejected) +{ + EXPECT_THROW(LocalStoreConfig("", {{"store", ""}}), UsageError); +} + TEST(LocalStore, constructConfig_rootQueryParam) { +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif LocalStoreConfig config{ "", { { "root", - "/foo/bar", + std::string{root}, }, }, }; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalStore, constructConfig_rootPath) { - LocalStoreConfig config{"/foo/bar", {}}; +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif + LocalStoreConfig config{std::string{root}, {}}; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalStore, constructConfig_to_string) diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index f0a334c1b613..7c9a17ed6b00 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -2,18 +2,18 @@ #include "nix/util/file-system.hh" #include "nix/util/util.hh" -#include "nix/util/tests/characterization.hh" +#include "nix/util/tests/test-data.hh" #include #include -using testing::Contains; -using testing::ElementsAre; -using testing::Eq; -using testing::Field; -using testing::SizeIs; +using ::testing::Contains; +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::Field; +using ::testing::SizeIs; -using namespace nix; +namespace nix { TEST(machines, getMachinesWithEmptyBuilders) { @@ -27,7 +27,7 @@ TEST(machines, getMachinesUriOnly) ASSERT_THAT(actual, SizeIs(1)); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); - EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::filesystem::path{}))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::nullopt))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); @@ -49,7 +49,7 @@ TEST(machines, getMachinesDefaults) ASSERT_THAT(actual, SizeIs(1)); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); - EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::filesystem::path{}))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::nullopt))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); @@ -185,7 +185,10 @@ TEST(machines, getMachinesWithCorrectFileReference) TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { - std::filesystem::path path = "/dev/null"; + auto tmpDir = nix::createTempDir(); + AutoDelete delTmpDir(tmpDir); + auto path = tmpDir / "empty-machines"; + nix::writeFile(path, ""); ASSERT_TRUE(std::filesystem::exists(path)); auto actual = Machine::parseConfig({}, "@" + path.string()); @@ -207,3 +210,5 @@ TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) {}, "@" + std::filesystem::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()), FormatError); } + +} // namespace nix diff --git a/src/libstore-tests/main.cc b/src/libstore-tests/main.cc index c45e3a7f384a..93d93e4aa0ce 100644 --- a/src/libstore-tests/main.cc +++ b/src/libstore-tests/main.cc @@ -3,11 +3,9 @@ #include "nix/store/tests/test-main.hh" #include "nix/store/tests/libstore-network.hh" -using namespace nix; - int main(int argc, char ** argv) { - auto res = testMainForBuidingPre(argc, argv); + auto res = nix::testMainForBuidingPre(argc, argv); if (res) return res; diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 2d12b14d3cb5..a7353f1ec7dd 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -60,12 +60,16 @@ sources = files( 'derivation-advanced-attrs.cc', 'derivation/external-formats.cc', 'derivation/invariants.cc', + 'derivations.cc', 'derived-path.cc', 'downstream-placeholder.cc', 'dummy-store.cc', + 'filetransfer-request.cc', + 'filetransfer-retry.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', 'local-binary-cache-store.cc', + 'local-fs-store.cc', 'local-overlay-store.cc', 'local-store.cc', 'machines.cc', @@ -73,6 +77,7 @@ sources = files( 'nar-info-disk-cache.cc', 'nar-info.cc', 'nix_api_store.cc', + 'outputs-query.cc', 'outputs-spec.cc', 'path-info.cc', 'path.cc', @@ -82,6 +87,7 @@ sources = files( 's3-url.cc', 'serve-protocol.cc', 'ssh-store.cc', + 'store-open.cc', 'store-reference.cc', 'uds-remote-store.cc', 'worker-protocol.cc', @@ -105,14 +111,19 @@ this_exe = executable( cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [], ) +test_env = { + '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', +} + +if host_machine.system() != 'windows' + test_env += {'NIX_REMOTE' : meson.current_build_dir() / 'test-home' / 'store'} +endif + test( meson.project_name(), this_exe, - env : { - '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', - 'HOME' : meson.current_build_dir() / 'test-home', - 'NIX_REMOTE' : meson.current_build_dir() / 'test-home' / 'store', - }, + env : test_env, protocol : 'gtest', ) diff --git a/src/libstore-tests/nar-info-disk-cache.cc b/src/libstore-tests/nar-info-disk-cache.cc index c73d33b73038..b54c98fe5e7d 100644 --- a/src/libstore-tests/nar-info-disk-cache.cc +++ b/src/libstore-tests/nar-info-disk-cache.cc @@ -46,7 +46,6 @@ TEST(NarInfoDiskCacheImpl, create_and_read) // Check that the fields are saved and returned correctly. This does not test // the select statement yet, because of in-memory caching. savedId = cache->createCache("http://foo", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)}); - ; { auto r = cache->upToDateCacheExists("http://foo"); ASSERT_TRUE(r); diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index 9b0f6018cdeb..63e5908d8818 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -31,6 +31,16 @@ class NarInfoTestV2 : public CharacterizationTest, public LibStoreTest } }; +class NarInfoTestV3 : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "nar-info" / "json-3"; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / (testStem + ".json"); + } +}; + static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { auto info = NarInfo::makeFromCA( @@ -65,7 +75,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) }; info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; - info.compression = "xz"; + info.compression = CompressionAlgo::xz; info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="); info.fileSize = 4029176; } @@ -122,6 +132,31 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) JSON_READ_TEST_V2(STEM, PURE) \ JSON_WRITE_TEST_V2(STEM, PURE) +#define JSON_READ_TEST_V3(STEM, PURE) \ + TEST_F(NarInfoTestV3, NarInfo_##STEM##_from_json) \ + { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + auto got = UnkeyedNarInfo::fromJSON(nullptr, encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } + +#define JSON_WRITE_TEST_V3(STEM, PURE) \ + TEST_F(NarInfoTestV3, NarInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> json { return makeNarInfo(*store, PURE).toJSON(nullptr, PURE, PathInfoJsonFormat::V3); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +#define JSON_TEST_V3(STEM, PURE) \ + JSON_READ_TEST_V3(STEM, PURE) \ + JSON_WRITE_TEST_V3(STEM, PURE) + JSON_TEST_V1(pure, false) JSON_TEST_V1(impure, true) @@ -131,4 +166,19 @@ JSON_READ_TEST_V1(pure_noversion, false) JSON_TEST_V2(pure, false) JSON_TEST_V2(impure, true) +JSON_TEST_V3(pure, false) +JSON_TEST_V3(impure, true) + +#undef JSON_TEST_V1 +#undef JSON_READ_TEST_V1 +#undef JSON_WRITE_TEST_V1 + +#undef JSON_TEST_V2 +#undef JSON_READ_TEST_V2 +#undef JSON_WRITE_TEST_V2 + +#undef JSON_TEST_V3 +#undef JSON_READ_TEST_V3 +#undef JSON_WRITE_TEST_V3 + } // namespace nix diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index f35d6ddf36f3..0c4739bbc91f 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -5,6 +5,7 @@ #include "nix_api_util.h" #include "nix_api_store.h" +#include "nix/store/tests/libstore.hh" #include "nix/store/tests/nix_api_store.hh" #include "nix/store/globals.hh" #include "nix/util/tests/string_callback.hh" @@ -76,8 +77,8 @@ TEST_F(nix_api_store_test, ReturnsValidStorePath) { StorePath * result = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); ASSERT_NE(result, nullptr); - ASSERT_STREQ("name", result->path.name().data()); - ASSERT_STREQ(PATH_SUFFIX.substr(1).c_str(), result->path.to_string().data()); + ASSERT_EQ("name", result->path.name()); + ASSERT_EQ(PATH_SUFFIX.substr(1), result->path.to_string()); nix_store_path_free(result); } @@ -212,6 +213,9 @@ TEST_F(nix_api_store_test, nix_store_real_path) TEST_F(nix_api_util_context, nix_store_real_path_relocated) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine/Windows does not support symlinks needed for local store gcroots"; +#endif auto tmp = nix::createTempDir(); auto storeRoot = (tmp / "store").string(); auto stateDir = (tmp / "state").string(); @@ -251,8 +255,9 @@ TEST_F(nix_api_util_context, nix_store_real_path_relocated) TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) { - Store * store = - nix_store_open(ctx, nix::fmt("file://%s/binary-cache", nix::createTempDir().string()).c_str(), nullptr); + auto tmpDir = nix::createTempDir() / "binary-cache"; + auto url = nix::ParsedURL{.scheme = "file", .path = nix::pathToUrlPath(tmpDir)}; + Store * store = nix_store_open(ctx, url.to_string().c_str(), nullptr); assert_ctx_ok(); ASSERT_NE(store, nullptr); @@ -289,6 +294,7 @@ struct LambdaAdapter class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base { public: + std::optional enableCA; StorePath * drvPath = nullptr; nix_derivation * drv = nullptr; Store * store = nullptr; @@ -297,8 +303,11 @@ class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base void SetUp() override { nix_api_store_test_base::SetUp(); +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + enableCA.emplace("ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; store = open_local_store(); @@ -352,8 +361,11 @@ class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base TEST_F(nix_api_store_test_base, build_from_json) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // FIXME get rid of these - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::EnableExperimentalFeature enableCA{"ca-derivations"}; nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); @@ -399,8 +411,11 @@ TEST_F(nix_api_store_test_base, build_from_json) TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when the system is invalid - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::EnableExperimentalFeature enableCA{"ca-derivations"}; nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); @@ -444,8 +459,11 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system) TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when the builder fails - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::EnableExperimentalFeature enableCA{"ca-derivations"}; nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); @@ -489,8 +507,11 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails) TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when builder succeeds but produces no output - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::EnableExperimentalFeature enableCA{"ca-derivations"}; nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); @@ -686,7 +707,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering) // Test that nix_store_realise returns outputs in alphabetical order by output name. // This test uses a CA derivation with 10 outputs in randomized input order // to verify that the callback order is deterministic and alphabetical. - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::EnableExperimentalFeature enableCA{"ca-derivations"}; nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); diff --git a/src/libstore-tests/outputs-query.cc b/src/libstore-tests/outputs-query.cc new file mode 100644 index 000000000000..4be25e2ad3b8 --- /dev/null +++ b/src/libstore-tests/outputs-query.cc @@ -0,0 +1,114 @@ +// Regression tests for the functions in outputs-query.cc +// +// See https://github.com/NixOS/nix/issues/15713 + +#include + +#include "nix/store/outputs-query.hh" +#include "nix/store/derivations.hh" +#include "nix/store/dummy-store-impl.hh" +#include "nix/store/realisation.hh" +#include "nix/store/tests/libstore.hh" + +namespace nix { + +class OutputsQueryTest : public ::testing::Test +{ +public: + static void SetUpTestSuite() + { + initLibStore(false); + } + +protected: + EnableExperimentalFeature caFeature{"ca-derivations"}; + + ref store = [] { + auto cfg = make_ref(StoreReference::Params{}); + cfg->readOnly = false; + return cfg->openDummyStore(); + }(); + + static DerivationOutput caFloatingOutput() + { + return DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}; + } + + /** + * Build a simple floating CA derivation with a given name and no input + * derivations. + */ + Derivation makeLeafDrv(std::string name) + { + Derivation drv; + drv.name = std::move(name); + drv.platform = "x86_64-linux"; + drv.builder = "/bin/sh"; + drv.outputs = {{"out", caFloatingOutput()}}; + return drv; + } +}; + +/** + * Regression test for https://github.com/NixOS/nix/issues/15713 + * + * In a Fibonacci-style chain of floating CA derivations, the resolution + * algorithm used to call queryRealisation O(Fib(N)) times. + * This test verifies that memoization reduces this to O(N). + */ +TEST_F(OutputsQueryTest, fibonacciChainQueryCount) +{ + constexpr static size_t N = 10; + std::vector drvPaths; + + // d0, d1: leaf derivations + for (int i = 0; i < 2; ++i) { + drvPaths.push_back(store->writeDerivation(makeLeafDrv("d" + std::to_string(i)))); + } + + // d_i depends on d_{i-1} and d_{i-2} + for (size_t i = 2; i <= N; ++i) { + Derivation drv = makeLeafDrv("d" + std::to_string(i)); + drv.inputDrvs.map[drvPaths[i - 1]].value.insert("out"); + drv.inputDrvs.map[drvPaths[i - 2]].value.insert("out"); + drvPaths.push_back(store->writeDerivation(drv)); + } + + // Tracker for queryRealisation calls. + std::map callCounts; + std::map outPaths; + + QueryRealisationFun queryRealisation = [&](const DrvOutput & id) -> std::shared_ptr { + assert(id.outputName == "out"); + callCounts[id.drvPath]++; + + // Memoize mock output paths. + auto it = outPaths.find(id.drvPath); + if (it == outPaths.end()) { + auto hash = hashString(HashAlgorithm::SHA1, "mock-output-" + std::to_string(outPaths.size())); + it = outPaths.emplace(id.drvPath, StorePath(hash, "out")).first; + } + + return std::make_shared(UnkeyedRealisation{.outPath = it->second}); + }; + + auto result = deepQueryPartialDerivationOutput(*store, drvPaths[N], "out", nullptr, queryRealisation); + + ASSERT_TRUE(result); + + int totalCalls = 0; + for (auto & [path, count] : callCounts) { + totalCalls += count; + if (count > 1) + ADD_FAILURE() << "Derivation at " << store->printStorePath(path) << " was queried " << count + << " times (expected 1)"; + } + + // With full memoization (ResolveCache + RealisationCache), each derivation should be queried exactly once. + EXPECT_EQ(totalCalls, N + 1) << "queryRealisation called " << totalCalls << " times; expected exactly " << (N + 1); +} + +} // namespace nix diff --git a/src/libstore-tests/outputs-spec.cc b/src/libstore-tests/outputs-spec.cc index 1fac222fccba..ae0bb1789b2b 100644 --- a/src/libstore-tests/outputs-spec.cc +++ b/src/libstore-tests/outputs-spec.cc @@ -33,7 +33,16 @@ class ExtendedOutputsSpecTest : public virtual CharacterizationTest TEST_F(OutputsSpecTest, no_empty_names) { +#ifndef _WIN32 ASSERT_DEATH(OutputsSpec::Names{StringSet{}}, ""); +#else + // ASSERT_DEATH relies on CreateProcess, which under Wine leaks the + // child's stderr into the parent console. Test both sides of the + // invariant without a death test instead. + ASSERT_FALSE(OutputsSpec::parseOpt("")); + OutputsSpec::Names names{StringSet{"out"}}; + ASSERT_FALSE(names.empty()); +#endif } #define TEST_DONT_PARSE(NAME, STR) \ @@ -259,13 +268,9 @@ INSTANTIATE_TEST_SUITE_P( #undef TEST_JSON -#ifndef COVERAGE - RC_GTEST_PROP(OutputsSpec, prop_round_rip, (const OutputsSpec & o)) { RC_ASSERT(o == OutputsSpec::parse(o.to_string())); } -#endif - } // namespace nix diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index 97ad4b270ade..7734ee175561 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -30,6 +30,16 @@ class PathInfoTestV2 : public CharacterizationTest, public LibStoreTest } }; +class PathInfoTestV3 : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "path-info" / "json-3"; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / (testStem + ".json"); + } +}; + static UnkeyedValidPathInfo makeEmpty() { return { @@ -129,6 +139,31 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo JSON_READ_TEST_V2(STEM, OBJ) \ JSON_WRITE_TEST_V2(STEM, OBJ, PURE) +#define JSON_READ_TEST_V3(STEM, OBJ) \ + TEST_F(PathInfoTestV3, PathInfo_##STEM##_from_json) \ + { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON(nullptr, encoded); \ + auto expected = OBJ; \ + ASSERT_EQ(got, expected); \ + }); \ + } + +#define JSON_WRITE_TEST_V3(STEM, OBJ, PURE) \ + TEST_F(PathInfoTestV3, PathInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> json { return OBJ.toJSON(nullptr, PURE, PathInfoJsonFormat::V3); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +#define JSON_TEST_V3(STEM, OBJ, PURE) \ + JSON_READ_TEST_V3(STEM, OBJ) \ + JSON_WRITE_TEST_V3(STEM, OBJ, PURE) + JSON_TEST_V1(empty_pure, makeEmpty(), false) JSON_TEST_V1(empty_impure, makeEmpty(), true) JSON_TEST_V1(pure, makeFull(*store, false), false) @@ -142,6 +177,23 @@ JSON_TEST_V2(empty_impure, makeEmpty(), true) JSON_TEST_V2(pure, makeFull(*store, false), false) JSON_TEST_V2(impure, makeFull(*store, true), true) +JSON_TEST_V3(empty_pure, makeEmpty(), false) +JSON_TEST_V3(empty_impure, makeEmpty(), true) +JSON_TEST_V3(pure, makeFull(*store, false), false) +JSON_TEST_V3(impure, makeFull(*store, true), true) + +#undef JSON_TEST_V1 +#undef JSON_READ_TEST_V1 +#undef JSON_WRITE_TEST_V1 + +#undef JSON_TEST_V2 +#undef JSON_READ_TEST_V2 +#undef JSON_WRITE_TEST_V2 + +#undef JSON_TEST_V3 +#undef JSON_READ_TEST_V3 +#undef JSON_WRITE_TEST_V3 + TEST_F(PathInfoTestV2, PathInfo_full_shortRefs) { ValidPathInfo it = makeFullKeyed(*store, true); diff --git a/src/libstore-tests/path.cc b/src/libstore-tests/path.cc index eb860a34dab1..65755b7f9645 100644 --- a/src/libstore-tests/path.cc +++ b/src/libstore-tests/path.cc @@ -86,8 +86,6 @@ TEST_DO_PARSE(triple_dot, "...") #undef TEST_DO_PARSE -#ifndef COVERAGE - RC_GTEST_FIXTURE_PROP(StorePathTest, prop_regex_accept, (const StorePath & p)) { RC_ASSERT(std::regex_match(std::string{p.name()}, nameRegex)); @@ -141,8 +139,6 @@ RC_GTEST_FIXTURE_PROP(StorePathTest, prop_check_regex_eq_parse, ()) RC_ASSERT(parsed == std::regex_match(std::string{name}, nameRegex)); } -#endif - /* ---------------------------------------------------------------------------- * JSON * --------------------------------------------------------------------------*/ diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index d2d7df80f59a..cec9db9087ec 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -1,16 +1,57 @@ -#include - #include #include #include #include "nix/store/store-api.hh" +#include "nix/util/json-utils.hh" +#include "nix/util/signature/local-keys.hh" +#include "nix/util/signature/signer.hh" #include "nix/util/tests/json-characterization.hh" #include "nix/store/tests/libstore.hh" namespace nix { +using nlohmann::json; + +/* ---------------------------------------------------------------------------- + * Test data + * --------------------------------------------------------------------------*/ + +UnkeyedRealisation unkeyedSimple{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, +}; + +UnkeyedRealisation unkeyedWithSignature{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + }, +}; + +DrvOutput testDrvOutput{ + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, + .outputName = "foo", +}; + +auto testSecretKey = + "test-key:tU7tTvLcScf8pmz/eTV0BEtLmRsPpZfKaRcd0nCN+pysBZPHSeg61/u2oc7mIOewfuAY1V1BiX32homTaDJ2Jw=="; + +Realisation simple{ + unkeyedSimple, + testDrvOutput, +}; + +Realisation withSignature{ + unkeyedWithSignature, + testDrvOutput, +}; + +/* ---------------------------------------------------------------------------- + * Realisation JSON + * --------------------------------------------------------------------------*/ + class RealisationTest : public JsonCharacterizationTest, public LibStoreTest { std::filesystem::path unitTestData = getUnitTestData() / "realisation"; @@ -23,12 +64,6 @@ class RealisationTest : public JsonCharacterizationTest, public Lib } }; -/* ---------------------------------------------------------------------------- - * JSON - * --------------------------------------------------------------------------*/ - -using nlohmann::json; - struct RealisationJsonTest : RealisationTest, ::testing::WithParamInterface> {}; @@ -44,19 +79,6 @@ TEST_P(RealisationJsonTest, to_json) writeJsonTest(name, value); } -Realisation simple{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }, - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, -}; - INSTANTIATE_TEST_SUITE_P( RealisationJSON, RealisationJsonTest, @@ -67,24 +89,159 @@ INSTANTIATE_TEST_SUITE_P( }, std::pair{ "with-signature", - [&] { - auto r = simple; - // FIXME actually sign properly - r.signatures = { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - }; - return r; - }(), + withSignature, })); /** - * We no longer have a notion of "dependent realisations", but we still - * want to parse old realisation files. So make this just be a read test - * (no write direction), accordingly. + * Old signature format (string) should still be parseable. */ -TEST_F(RealisationTest, dependent_realisations_from_json) +TEST_F(RealisationTest, with_signature_from_json) +{ + readJsonTest("with-signature-unstructured", withSignature); +} + +/* ---------------------------------------------------------------------------- + * UnkeyedRealisation JSON + * --------------------------------------------------------------------------*/ + +class UnkeyedRealisationTest : public JsonCharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "realisation"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +struct UnkeyedRealisationJsonTest : UnkeyedRealisationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(UnkeyedRealisationJsonTest, from_json) +{ + const auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(UnkeyedRealisationJsonTest, to_json) +{ + const auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + UnkeyedRealisationJSON, + UnkeyedRealisationJsonTest, + ::testing::Values( + std::pair{ + "unkeyed-simple", + unkeyedSimple, + }, + std::pair{ + "unkeyed-with-signature", + unkeyedWithSignature, + })); + +/* ---------------------------------------------------------------------------- + * Signing and verification + * --------------------------------------------------------------------------*/ + +struct RealisationSigningTest : RealisationTest, ::testing::WithParamInterface> +{}; + +TEST_P(RealisationSigningTest, fingerprint) +{ + const auto & [name, realisation] = GetParam(); + writeTest(std::string{name} + "-fingerprint.txt", [&]() -> std::string { + return realisation.fingerprint(realisation.id); + }); +} + +TEST_P(RealisationSigningTest, sign) { - readJsonTest("with-dependent-realisations", simple); + const auto & [name, realisation] = GetParam(); + + LocalSigner signer(SecretKey::parse(testSecretKey)); + + auto sig = realisation.sign(realisation.id, signer); + + nix::writeJsonTest(*this, std::string{name} + "-sig", sig); } +TEST_P(RealisationSigningTest, verify) +{ + const auto & [name, realisation] = GetParam(); + + auto publicKey = SecretKey::parse(testSecretKey)->toPublicKey(); + PublicKeys publicKeys; + publicKeys.insert_or_assign(publicKey->name, std::move(publicKey)); + + readTest(std::string{name} + "-sig.json", [&](const auto & encoded) { + Signature sig = json::parse(encoded); + ASSERT_TRUE(realisation.checkSignature(realisation.id, publicKeys, sig)); + }); +} + +TEST_P(RealisationSigningTest, verify_rejects_wrong_key) +{ + const auto & [name, realisation] = GetParam(); + + auto wrongKey = SecretKey::generate("wrong-key", KeyType::Ed25519); + auto wrongPublicKey = wrongKey->toPublicKey(); + PublicKeys publicKeys; + publicKeys.insert_or_assign(wrongPublicKey->name, std::move(wrongPublicKey)); + + auto r = static_cast(realisation); + LocalSigner signer(SecretKey::parse(testSecretKey)); + r.sign(realisation.id, signer); + + ASSERT_EQ(r.checkSignatures(realisation.id, publicKeys), 0); +} + +TEST_P(RealisationSigningTest, verify_rejects_tampered_outpath) +{ + const auto & [name, realisation] = GetParam(); + + auto publicKey = SecretKey::parse(testSecretKey)->toPublicKey(); + PublicKeys publicKeys; + publicKeys.insert_or_assign(publicKey->name, std::move(publicKey)); + + auto r = static_cast(realisation); + LocalSigner signer(SecretKey::parse(testSecretKey)); + r.sign(realisation.id, signer); + + // Tamper with the output path after signing. + r.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}; + + ASSERT_EQ(r.checkSignatures(realisation.id, publicKeys), 0); +} + +TEST_P(RealisationSigningTest, signatures_stripped_from_fingerprint) +{ + const auto & [name, realisation] = GetParam(); + + auto fp = realisation.fingerprint(realisation.id); + auto parsed = json::parse(fp); + auto obj = getObject(parsed); + auto * value = optionalValueAt(obj, "value"); + ASSERT_NE(value, nullptr); + ASSERT_FALSE(getObject(*value).contains("signatures")); +} + +INSTANTIATE_TEST_SUITE_P( + RealisationSigning, + RealisationSigningTest, + ::testing::Values( + std::pair{ + "simple", + simple, + }, + std::pair{ + "with-signature", + withSignature, + })); + } // namespace nix diff --git a/src/libstore-tests/ref-scan-bench.cc b/src/libstore-tests/ref-scan-bench.cc index ff0aa181503a..05264fd29061 100644 --- a/src/libstore-tests/ref-scan-bench.cc +++ b/src/libstore-tests/ref-scan-bench.cc @@ -6,7 +6,7 @@ #include -using namespace nix; +namespace nix { template static void randomReference(std::mt19937 & urng, OIt outIter) @@ -91,3 +91,5 @@ static void BM_RefScanSinkRandom(benchmark::State & state) } BENCHMARK(BM_RefScanSinkRandom)->Arg(10'000)->Arg(100'000)->Arg(1'000'000)->Arg(5'000'000)->Arg(10'000'000); + +} // namespace nix diff --git a/src/libstore-tests/references.cc b/src/libstore-tests/references.cc index f2c6fb51e5ca..69b89b1460ee 100644 --- a/src/libstore-tests/references.cc +++ b/src/libstore-tests/references.cc @@ -4,6 +4,9 @@ #include +#include +#include + namespace nix { struct RewriteParams @@ -43,6 +46,100 @@ INSTANTIATE_TEST_CASE_P( RewriteParams{"foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}}, RewriteParams{"foooo", "foooo", {}})); +TEST(references, rewritingSinkChunking) +{ + std::mt19937 rng(42); + + /* Build a set of rewrites. Keys are random [a-z] strings of random + length in [8, 32]; values are same-length [A-Z] strings so a + replacement never produces a new match for any key. We also skip + any key that would be a substring of (or have as a substring) an + existing key, so that inserting one key into the input cannot also + produce a match for a different key. */ + StringMap rewrites; + std::vector keys; + { + std::uniform_int_distribution lowerDist('a', 'z'); + std::uniform_int_distribution upperDist('A', 'Z'); + std::uniform_int_distribution lenDist(8, 32); + + while (rewrites.size() < 8) { + std::string from(lenDist(rng), '\0'); + for (auto & c : from) + c = lowerDist(rng); + if (rewrites.count(from)) + continue; + bool overlap = false; + for (auto & other : keys) + if (from.find(other) != std::string::npos || other.find(from) != std::string::npos) { + overlap = true; + break; + } + if (overlap) + continue; + std::string to(from.size(), '\0'); + for (auto & c : to) + c = upperDist(rng); + rewrites[from] = to; + keys.push_back(from); + } + } + + /* Build a ~1 MB input mixing rewrite keys with [0-9] digits. Always + emit at least one digit between two consecutive keys so adjacency + cannot create spurious matches at key boundaries. Compute the + expected output string and matches vector at the same time. */ + std::string input; + std::string expectedOutput; + std::set expectedMatches; + input.reserve(1'000'000); + expectedOutput.reserve(1'000'000); + { + std::uniform_int_distribution digitDist('0', '9'); + std::uniform_int_distribution keyDist(0, keys.size() - 1); + std::bernoulli_distribution useKeyDist(0.2); + bool justInsertedKey = false; + + while (input.size() < 1'000'000) { + if (useKeyDist(rng) && !justInsertedKey) { + const auto & from = keys[keyDist(rng)]; + expectedMatches.insert(input.size()); + expectedOutput += rewrites.at(from); + input += from; + justInsertedKey = true; + } else { + char d = static_cast(digitDist(rng)); + input.push_back(d); + expectedOutput.push_back(d); + justInsertedKey = false; + } + } + } + + StringSink singleOut; + RewritingSink singleSink(rewrites, singleOut); + singleSink(input); + singleSink.flush(); + + StringSink chunkedOut; + RewritingSink chunkedSink(rewrites, chunkedOut); + { + std::uniform_int_distribution chunkDist(1, 128); + std::string_view remaining(input); + while (!remaining.empty()) { + auto n = std::min(chunkDist(rng), remaining.size()); + chunkedSink(remaining.substr(0, n)); + remaining = remaining.substr(n); + } + } + chunkedSink.flush(); + + ASSERT_EQ(singleOut.s, expectedOutput); + ASSERT_EQ(chunkedOut.s, expectedOutput); + ASSERT_EQ(singleSink.matches, expectedMatches); + ASSERT_EQ(chunkedSink.matches, expectedMatches); +} + TEST(references, scan) { std::string hash1 = "dc04vv14dak1c1r48qa0m23vr9jy8sm0"; diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 1d795818369e..ecea1c8010a4 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -5,14 +5,13 @@ #include "nix/store/store-open.hh" #include "nix/util/file-system.hh" #include "nix/util/hash.hh" -#include "nix/util/tests/test-data.hh" #ifndef _WIN32 # include # include -using namespace nix; +namespace nix { static void BM_RegisterValidPathsDerivations(benchmark::State & state) { @@ -23,7 +22,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) auto tmpRoot = createTempDir(); auto realStoreDir = tmpRoot / "nix/store"; - std::filesystem::create_directories(realStoreDir); + createDirs(realStoreDir); std::shared_ptr store = openStore(fmt("local?root=%s", tmpRoot.string())); auto localStore = std::dynamic_pointer_cast(store); @@ -67,7 +66,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) state.PauseTiming(); localStore.reset(); store.reset(); - std::filesystem::remove_all(tmpRoot); + deletePath(tmpRoot); state.ResumeTiming(); } @@ -76,4 +75,6 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) BENCHMARK(BM_RegisterValidPathsDerivations)->Arg(10); +} // namespace nix + #endif diff --git a/src/libstore-tests/s3-binary-cache-store.cc b/src/libstore-tests/s3-binary-cache-store.cc index 9aa9b2dd1a3e..126d7a522aaa 100644 --- a/src/libstore-tests/s3-binary-cache-store.cc +++ b/src/libstore-tests/s3-binary-cache-store.cc @@ -1,7 +1,6 @@ #include "nix/store/s3-binary-cache-store.hh" #include "nix/store/http-binary-cache-store.hh" #include "nix/store/filetransfer.hh" -#include "nix/store/s3-url.hh" #include diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 159abeb60827..e34f916cbb5c 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -1,5 +1,4 @@ #include -#include #include #include @@ -17,8 +16,6 @@ namespace nix { const char serveProtoDir[] = "serve-protocol"; -static constexpr std::string_view defaultStoreDir = "/nix/store"; - struct ServeProtoTest : VersionedProtoTest { /** @@ -78,16 +75,19 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - drvOutput, - "drv-output", - defaultVersion, + drvOutput_2_8, + "drv-output-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) @@ -96,54 +96,37 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, + unkeyedRealisation_2_8, + "unkeyed-realisation-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, + realisation_2_8, + "realisation-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -209,7 +192,10 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully write + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, buildResult_2_6, "build-result-2.6", @@ -242,27 +228,72 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_8, + "build-result-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -292,12 +323,12 @@ VERSIONED_CHARACTERIZATION_TEST( }), (std::tuple{ ({ - UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy}; + UnkeyedValidPathInfo info{"/nix/store", Hash::dummy}; info.narSize = 34878; info; }), ({ - UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy}; + UnkeyedValidPathInfo info{"/nix/store", Hash::dummy}; info.deriver = StorePath{ "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", }; @@ -322,7 +353,7 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple{ ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.deriver = StorePath{ @@ -513,12 +544,6 @@ TEST_F(ServeProtoTest, handshake_log) }); } -/// Has to be a `BufferedSink` for handshake. -struct NullBufferedSink : BufferedSink -{ - void writeUnbuffered(std::string_view data) override {} -}; - TEST_F(ServeProtoTest, handshake_client_replay) { CharacterizationTest::readTest("handshake-to-client.bin", [&](std::string toClientLog) { @@ -578,4 +603,62 @@ TEST_F(ServeProtoTest, handshake_client_corrupted_throws) }); } +/** + * The old-protocol fallback writes `builtOutputs` as a `StringMap` + * with a dummy hash so that old clients can still extract output + * paths. This round-trips because the read side only uses the + * `outputName` (from the key) and `outPath` (from the JSON value). + */ +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_7_compat, + "build-result-2.7-compat", + (ServeProto::Version{ + .major = 2, + .minor = 7, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + } // namespace nix diff --git a/src/libstore-tests/ssh-store.cc b/src/libstore-tests/ssh-store.cc index 2d4548d6ae21..813d58574ce5 100644 --- a/src/libstore-tests/ssh-store.cc +++ b/src/libstore-tests/ssh-store.cc @@ -6,6 +6,20 @@ namespace nix { +TEST(SSHStore, storeDir_absolutePath) +{ + SSHStoreConfig config{ + ParsedURL::Authority::parse("localhost"), + {{"store", "/my/store"}}, + }; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(SSHStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW(SSHStoreConfig(ParsedURL::Authority::parse("localhost"), {{"store", "my/store"}}), UsageError); +} + TEST(SSHStore, constructConfig) { SSHStoreConfig config{ @@ -31,6 +45,27 @@ TEST(SSHStore, constructConfig) EXPECT_EQ(config.getReference().render(/*withParams=*/true), "ssh-ng://me@localhost:2222"); } +TEST(MountedSSHStore, storeDir_absolutePath) +{ + std::filesystem::path storeDir = +#ifdef _WIN32 + "C:\\"; +#else + "/"; +#endif + storeDir /= "nix"; + storeDir /= "store"; + MountedSSHStoreConfig config{{.host = "localhost"}, {{"store", storeDir.string()}}}; + EXPECT_EQ(config.storeDir, storeDir.string()); +} + +TEST(MountedSSHStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW( + MountedSSHStoreConfig({.host = "localhost"}, {{"store", (std::filesystem::path{"nix"} / "store").string()}}), + UsageError); +} + TEST(MountedSSHStore, constructConfig) { MountedSSHStoreConfig config{ diff --git a/src/libstore-tests/store-open.cc b/src/libstore-tests/store-open.cc new file mode 100644 index 000000000000..cadeb3f040a4 --- /dev/null +++ b/src/libstore-tests/store-open.cc @@ -0,0 +1,62 @@ +#include + +#include "nix/store/store-open.hh" +#include "nix/store/store-reference.hh" +#include "nix/store/local-store.hh" +#include "nix/store/globals.hh" +#include "nix/util/file-system.hh" +#include "nix/util/finally.hh" + +namespace nix { + +TEST(StoreOpen, resolveStoreConfig_auto_default) +{ + // Save original settings + // + // TODO: resolveStoreConfig should not depend on global settings; + // the test should not have to override them. + auto originalStateDir = settings.nixStateDir; + Finally restoreStateDir([&]() { settings.nixStateDir = originalStateDir; }); + + // Set up a temporary writable state directory + auto tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + auto stateDir = tmpDir / "var/nix"; + createDirs(stateDir); + settings.nixStateDir = stateDir; + + StoreReference ref{ + .variant = StoreReference::Auto{}, + .params = {}, + }; + + auto config = resolveStoreConfig(std::move(ref)); + + // With a writable state directory and no daemon socket, "auto" should resolve to LocalStore + auto * localConfig = dynamic_cast(config.get()); + ASSERT_NE(localConfig, nullptr); + EXPECT_EQ(localConfig->getStateDir(), stateDir); +} + +TEST(StoreOpen, resolveStoreConfig_auto_withParams) +{ + // Create a temporary directory with a writable state directory + auto tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + auto stateDir = tmpDir / "var/nix"; + createDirs(stateDir); + + StoreReference ref{ + .variant = StoreReference::Auto{}, + .params = {{"state", stateDir.string()}}, + }; + + auto config = resolveStoreConfig(std::move(ref)); + + // With a writable state directory and no daemon socket, "auto" should resolve to LocalStore + auto * localConfig = dynamic_cast(config.get()); + ASSERT_NE(localConfig, nullptr); + EXPECT_EQ(localConfig->getStateDir(), stateDir); +} + +} // namespace nix diff --git a/src/libstore-tests/uds-remote-store.cc b/src/libstore-tests/uds-remote-store.cc index 88af22dbb22d..32122f5c2b86 100644 --- a/src/libstore-tests/uds-remote-store.cc +++ b/src/libstore-tests/uds-remote-store.cc @@ -4,6 +4,25 @@ namespace nix { +TEST(UDSRemoteStore, storeDir_absolutePath) +{ + std::filesystem::path storeDir = +#ifdef _WIN32 + "C:\\"; +#else + "/"; +#endif + storeDir /= "nix"; + storeDir /= "store"; + UDSRemoteStoreConfig config{"", {{"store", storeDir.string()}}}; + EXPECT_EQ(config.storeDir, storeDir.string()); +} + +TEST(UDSRemoteStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW(UDSRemoteStoreConfig("", {{"store", (std::filesystem::path{"nix"} / "store").string()}}), UsageError); +} + TEST(UDSRemoteStore, constructConfig) { UDSRemoteStoreConfig config{"/tmp/socket", {}}; diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index aac5822d01f8..91acde7380f6 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -1,4 +1,3 @@ -#include #include #include @@ -18,17 +17,17 @@ namespace nix { TEST(WorkerProtoVersionNumber, ordering) { using Number = WorkerProto::Version::Number; - EXPECT_LT((Number{1, 10}), (Number{1, 20})); - EXPECT_GT((Number{1, 30}), (Number{1, 20})); - EXPECT_EQ((Number{1, 10}), (Number{1, 10})); - EXPECT_LT((Number{0, 255}), (Number{1, 0})); + EXPECT_LT((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 20})); + EXPECT_GT((Number{.major = 1, .minor = 30}), (Number{.major = 1, .minor = 20})); + EXPECT_EQ((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 10})); + EXPECT_LT((Number{.major = 0, .minor = 255}), (Number{.major = 1, .minor = 0})); } TEST(WorkerProtoVersion, partialOrderingSameFeatures) { using V = WorkerProto::Version; - V v1{.number = {1, 20}, .features = {"a", "b"}}; - V v2{.number = {1, 30}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; + V v2{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}}; EXPECT_TRUE(v1 < v2); EXPECT_TRUE(v2 > v1); @@ -40,8 +39,8 @@ TEST(WorkerProtoVersion, partialOrderingSameFeatures) TEST(WorkerProtoVersion, partialOrderingSubsetFeatures) { using V = WorkerProto::Version; - V fewer{.number = {1, 30}, .features = {"a"}}; - V more{.number = {1, 30}, .features = {"a", "b"}}; + V fewer{.number = {.major = 1, .minor = 30}, .features = {"a"}}; + V more{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}}; // fewer <= more: JUST the features are a subset EXPECT_TRUE(fewer < more); @@ -54,8 +53,8 @@ TEST(WorkerProtoVersion, partialOrderingUnordered) { using V = WorkerProto::Version; // Same number but incomparable features - V v1{.number = {1, 20}, .features = {"a", "c"}}; - V v2{.number = {1, 20}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "c"}}; + V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; EXPECT_FALSE(v1 < v2); EXPECT_FALSE(v1 > v2); @@ -69,8 +68,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures) { using V = WorkerProto::Version; // Higher number but fewer features — unordered - V v1{.number = {1, 30}, .features = {"a"}}; - V v2{.number = {1, 20}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 30}, .features = {"a"}}; + V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; EXPECT_FALSE(v1 < v2); EXPECT_FALSE(v1 > v2); @@ -80,8 +79,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures) TEST(WorkerProtoVersion, partialOrderingEmptyFeatures) { using V = WorkerProto::Version; - V empty{.number = {1, 20}, .features = {}}; - V some{.number = {1, 30}, .features = {"a"}}; + V empty{.number = {.major = 1, .minor = 20}, .features = {}}; + V some{.number = {.major = 1, .minor = 30}, .features = {"a"}}; // empty features is a subset of everything EXPECT_TRUE(empty < some); @@ -91,8 +90,6 @@ TEST(WorkerProtoVersion, partialOrderingEmptyFeatures) const char workerProtoDir[] = "worker-protocol"; -static constexpr std::string_view defaultStoreDir = "/nix/store"; - struct WorkerProtoTest : VersionedProtoTest { /** @@ -223,69 +220,67 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, drvOutput, - "drv-output", - defaultVersion, + "drv-output-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 38, + }, + .features = {"realisation-with-path-not-hash"}, + }), (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, + unkeyedRealisation_realisation_with_path, + "unkeyed-realisation-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", + .major = 1, + .minor = 38, }, - }, + .features = {"realisation-with-path-not-hash"}, + }), + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, + realisation_realisation_with_path, + "realisation-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", + .major = 1, + .minor = 38, }, + .features = {"realisation-with-path-not-hash"}, + }), + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -318,7 +313,10 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully write + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_28, "build-result-1.28", @@ -347,25 +345,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -374,7 +360,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_29, "build-result-1.29", @@ -410,27 +397,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -443,7 +416,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_37, "build-result-1.37", @@ -479,27 +453,71 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_realisation_with_path, + "build-result-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 38, + }, + .features = {"realisation-with-path-not-hash"}, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -576,7 +594,7 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple{ ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.registrationTime = 23423; @@ -585,7 +603,7 @@ VERSIONED_CHARACTERIZATION_TEST( }), ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.deriver = StorePath{ @@ -620,7 +638,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -634,7 +652,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -675,7 +693,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -690,7 +708,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -920,7 +938,11 @@ TEST_F(WorkerProtoTest, handshake_features) out, in, WorkerProto::Version{ - .number = {.major = 1, .minor = 123}, + .number = + { + .major = 1, + .minor = 123, + }, .features = {"bar", "aap", "mies", "xyzzy"}, }); }); @@ -931,7 +953,11 @@ TEST_F(WorkerProtoTest, handshake_features) out, in, WorkerProto::Version{ - .number = {.major = 1, .minor = 200}, + .number = + { + .major = 1, + .minor = 200, + }, .features = {"foo", "bar", "xyzzy"}, }); @@ -950,12 +976,6 @@ TEST_F(WorkerProtoTest, handshake_features) })); } -/// Has to be a `BufferedSink` for handshake. -struct NullBufferedSink : BufferedSink -{ - void writeUnbuffered(std::string_view data) override {} -}; - TEST_F(WorkerProtoTest, handshake_client_replay) { CharacterizationTest::readTest("handshake-to-client.bin", [&](std::string toClientLog) { @@ -1013,4 +1033,65 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) }); } +/** + * The old-protocol fallback writes `builtOutputs` as a `StringMap` + * with a dummy hash so that old clients can still extract output + * paths. This round-trips because the read side only uses the + * `outputName` (from the key) and `outPath` (from the JSON value). + */ +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_29_compat, + "build-result-1.29-compat", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 29, + }, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + } // namespace nix diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index 3a049b7e7216..3534d44d0d8f 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -176,8 +176,7 @@ TEST_F(WorkerSubstitutionTest, singleRootStoreObjectWithSingleDepStoreObject) TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) { - // Enable CA derivations experimental feature - experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + EnableExperimentalFeature enableCA{"ca-derivations"}; // Create a CA floating output derivation Derivation drv; @@ -198,12 +197,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Snapshot the destination store before checkpointJson("ca-drv/store-before", dummyStore); - // Compute the hash modulo of the derivation - // For CA floating derivations, the kind is Deferred since outputs aren't known until build - auto hashModulo = hashDerivationModulo(*dummyStore, drv, true); - ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred); - auto drvHash = hashModulo.hashes.at("out"); - // Create the output store object auto outputPath = substituter->addToStore( "test-ca-drv-out", @@ -222,7 +215,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Add the realisation (build trace) to the substituter substituter->buildTrace.insert_or_assign( - drvHash, + drvPath, std::map{ { "out", @@ -236,7 +229,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) checkpointJson("ca-drv/substituter", substituter); // The realisation should not exist in the destination store yet - DrvOutput drvOutput{drvHash, "out"}; + DrvOutput drvOutput{drvPath, "out"}; ASSERT_FALSE(dummyStore->queryRealisation(drvOutput)); // Create a worker with our custom substituter @@ -248,7 +241,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Create a derivation goal for the CA derivation output // The worker should substitute the output rather than building - auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true); + auto goal = worker.makeDerivationGoal(drvPath, make_ref(drv), "out", bmNormal, true); // Run the worker Goals goals; @@ -268,9 +261,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Verify the goal succeeded ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); - - // Disable CA derivations experimental feature - experimentalFeatureSettings.set("extra-experimental-features", ""); } /** @@ -280,8 +270,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) */ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) { - // Enable CA derivations experimental feature - experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + EnableExperimentalFeature enableCA{"ca-derivations"}; // Create the dependency CA floating derivation Derivation depDrv; @@ -299,11 +288,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Write the dependency derivation to the destination store auto depDrvPath = dummyStore->writeDerivation(depDrv); - // Compute the hash modulo for the dependency derivation - auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true); - ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred); - auto depDrvHash = depHashModulo.hashes.at("out"); - // Create the output store object for the dependency in the substituter auto depOutputPath = substituter->addToStore( "dep-drv-out", @@ -322,7 +306,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Add the realisation for the dependency to the substituter substituter->buildTrace.insert_or_assign( - depDrvHash, + depDrvPath, std::map{ { "out", @@ -350,14 +334,22 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Write the root derivation to the destination store auto rootDrvPath = dummyStore->writeDerivation(rootDrv); + // Resolve the root derivation using a callback that returns the dep output path + auto resolvedRootDrv = rootDrv.tryResolve( + *substituter, + [&](ref drvPath, const std::string & outputName) -> std::optional { + EXPECT_EQ(*drvPath, SingleDerivedPath::Opaque{depDrvPath}); + EXPECT_EQ(outputName, "out"); + return depOutputPath; + }); + ASSERT_TRUE(resolvedRootDrv); + + // Write the resolved derivation to the substituter + auto resolvedRootDrvPath = substituter->writeDerivation(Derivation{*resolvedRootDrv}); + // Snapshot the destination store before checkpointJson("issue-11928/store-before", dummyStore); - // Compute the hash modulo for the root derivation - auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true); - ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred); - auto rootDrvHash = rootHashModulo.hashes.at("out"); - // Create the output store object for the root derivation // Note: it does NOT reference the dependency's output auto rootOutputPath = substituter->addToStore( @@ -378,12 +370,13 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) HashAlgorithm::SHA256); // The DrvOutputs for both derivations - DrvOutput depDrvOutput{depDrvHash, "out"}; - DrvOutput rootDrvOutput{rootDrvHash, "out"}; + DrvOutput depDrvOutput{depDrvPath, "out"}; + DrvOutput resolvedRootDrvOutput{resolvedRootDrvPath, "out"}; - // Add the realisation for the root derivation to the substituter + // Add the realisation for the *resolved* root derivation to the substituter + // (not the original root derivation - that would be an illegal "deep" realisation) substituter->buildTrace.insert_or_assign( - rootDrvHash, + resolvedRootDrvPath, std::map{ { "out", @@ -394,12 +387,12 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) }); // Snapshot the substituter - // Note: it has realisations for both drvs, but only the root's output store object + // Note: it has the dep realisation, resolved root drv, resolved root realisation, and root output checkpointJson("issue-11928/substituter", substituter); // The realisations should not exist in the destination store yet ASSERT_FALSE(dummyStore->queryRealisation(depDrvOutput)); - ASSERT_FALSE(dummyStore->queryRealisation(rootDrvOutput)); + ASSERT_FALSE(dummyStore->queryRealisation(resolvedRootDrvOutput)); // Create a worker with our custom substituter Worker worker{*dummyStore, *dummyStore}; @@ -410,7 +403,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Create a derivation goal for the root derivation output // The worker should substitute the output rather than building - auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false); + auto goal = worker.makeDerivationGoal(rootDrvPath, make_ref(rootDrv), "out", bmNormal, false); // Run the worker Goals goals; @@ -423,28 +416,22 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // The root output path should now exist in the destination store ASSERT_TRUE(dummyStore->isValidPath(rootOutputPath)); - // The root realisation should now exist in the destination store - auto rootRealisation = dummyStore->queryRealisation(rootDrvOutput); + // The resolved root realisation should now exist in the destination store + auto rootRealisation = dummyStore->queryRealisation(resolvedRootDrvOutput); ASSERT_TRUE(rootRealisation); ASSERT_EQ(rootRealisation->outPath, rootOutputPath); - // #11928: The dependency's REALISATION should be fetched, because - // it is needed to resolve the underlying derivation. Currently the - // realisation is not fetched (bug). Once fixed: Change - // depRealisation ASSERT_FALSE to ASSERT_TRUE and uncomment the - // ASSERT_EQ + // The dependency's REALISATION should have been fetched auto depRealisation = dummyStore->queryRealisation(depDrvOutput); - ASSERT_FALSE(depRealisation); - // ASSERT_EQ(depRealisation->outPath, depOutputPath); + ASSERT_TRUE(depRealisation); + ASSERT_EQ(depRealisation->outPath, depOutputPath); - // The dependency's OUTPUT is correctly not fetched (not referenced by root output) - ASSERT_FALSE(dummyStore->isValidPath(depOutputPath)); + // TODO #11928: The dependency's OUTPUT should NOT be fetched (not referenced + // by root output). Once #11928 is fixed, change ASSERT_TRUE to ASSERT_FALSE. + ASSERT_TRUE(dummyStore->isValidPath(depOutputPath)); // Verify the goal succeeded ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); - - // Disable CA derivations experimental feature - experimentalFeatureSettings.set("extra-experimental-features", ""); } } // namespace nix diff --git a/src/libstore-tests/write-derivation.cc b/src/libstore-tests/write-derivation.cc index c68753823d19..43060cb697b8 100644 --- a/src/libstore-tests/write-derivation.cc +++ b/src/libstore-tests/write-derivation.cc @@ -27,22 +27,19 @@ class WriteDerivationTest : public LibStoreTest ref config; }; -static Derivation makeSimpleDrv() -{ - Derivation drv; - drv.name = "simple-derivation"; - drv.platform = "system"; - drv.builder = "foo"; - drv.args = {"bar", "baz"}; - drv.env = StringPairs{{"BIG_BAD", "WOLF"}}; - return drv; -} - } // namespace TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce) { - auto drv = makeSimpleDrv(); + auto drv = []() { + Derivation drv; + drv.name = "simple-derivation"; + drv.platform = "system"; + drv.builder = "foo"; + drv.args = {"bar", "baz"}; + drv.env = StringPairs{{"BIG_BAD", "WOLF"}}; + return drv; + }(); auto path1 = store->writeDerivation(drv, NoRepair); config->readOnly = true; diff --git a/src/libstore/active-builds.cc b/src/libstore/active-builds.cc index d165b6d7d4eb..4b2688a9dd25 100644 --- a/src/libstore/active-builds.cc +++ b/src/libstore/active-builds.cc @@ -9,6 +9,10 @@ namespace nix { +void QueryActiveBuildsStore::anchor() {} + +void TrackActiveBuildsStore::anchor() {} + UserInfo UserInfo::fromUid(uid_t uid) { UserInfo info; diff --git a/src/libstore/async-path-writer.cc b/src/libstore/async-path-writer.cc index ede52a146aaf..f3271093a3c1 100644 --- a/src/libstore/async-path-writer.cc +++ b/src/libstore/async-path-writer.cc @@ -170,8 +170,14 @@ struct AsyncPathWriterImpl : AsyncPathWriter assert(storePath == item.storePath); } } + + void anchor() override; }; +void AsyncPathWriter::anchor() {} + +void AsyncPathWriterImpl::anchor() {} + ref AsyncPathWriter::make(ref store) { return make_ref(store); diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index b471962897bb..e4a909a4a554 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -4,6 +4,7 @@ # include # include "nix/store/s3-url.hh" +# include "nix/util/environment-variables.hh" # include "nix/util/logging.hh" # include @@ -27,12 +28,16 @@ namespace nix { +void AwsAuthError::anchor() {} + AwsAuthError::AwsAuthError(int errorCode) : CloneableError("AWS authentication error: '%s' (%d)", aws_error_str(errorCode), errorCode) , errorCode(errorCode) { } +AwsCredentialProvider::~AwsCredentialProvider() {} + namespace { /** @@ -170,15 +175,33 @@ static std::shared_ptr createSSOProvider( return createWrappedProvider(aws_credentials_provider_new_sso(allocator, &options), allocator); } +/** + * Check whether the AWS SDK can resolve a region from the standard + * environment variables. Mirrors aws_credentials_provider_resolve_region_from_env. + */ +static bool awsRegionSetInEnv() +{ + return getEnvNonEmpty("AWS_REGION") || getEnvNonEmpty("AWS_DEFAULT_REGION"); +} + /** * Create an STS WebIdentity credentials provider using the C library directly. * This reads AWS_WEB_IDENTITY_TOKEN_FILE, AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, * and AWS_REGION from the environment (falling back to the profile config). * Used by EKS IRSA, GitHub Actions OIDC, and other sts:AssumeRoleWithWebIdentity flows. * Returns nullptr if the required parameters can't be resolved. + * + * @param fallbackRegion Region to use when neither AWS_REGION nor + * AWS_DEFAULT_REGION is set — typically the ?region= from the S3 URL. + * Note: this also overrides any region from the profile config, since + * aws-c-auth gives options.region precedence over all implicit sources. + * Without this fallback the provider fails entirely in IRSA setups where + * the pod environment sets the token and role ARN but not the region + * (observed with AWS_CONFIG_FILE=/dev/null). */ static std::shared_ptr createSTSWebIdentityProvider( const std::string & profileName, + const std::string & fallbackRegion, Aws::Crt::Io::ClientBootstrap * bootstrap, Aws::Crt::Io::TlsContext * tlsContext, Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator()) @@ -192,6 +215,12 @@ static std::shared_ptr createSTSWebIdentit options.profile_name_override = aws_byte_cursor_from_c_str(profileName.c_str()); } + // aws-c-auth gives options.region precedence over env vars, so only set it + // when env vars are absent — otherwise we'd mask a user-supplied AWS_REGION. + if (!fallbackRegion.empty() && !awsRegionSetInEnv()) { + options.region = aws_byte_cursor_from_c_str(fallbackRegion.c_str()); + } + return createWrappedProvider(aws_credentials_provider_new_sts_web_identity(allocator, &options), allocator); } @@ -259,8 +288,6 @@ static AwsCredentials getCredentialsFromProvider(std::shared_ptr createProviderForProfile(const std::string & profile); + std::shared_ptr + createProviderForProfile(const std::string & profile, const std::string & region); private: Aws::Crt::ApiHandle apiHandle; std::shared_ptr tlsContext; Aws::Crt::Io::ClientBootstrap * bootstrap; - boost::concurrent_flat_map> - credentialProviderCache; + // Keyed by (profile, region). Region is part of the key because it is baked + // into the STS WebIdentity provider at construction time; two S3 URLs with + // different ?region= parameters need distinct chains. + boost:: + concurrent_flat_map, std::shared_ptr> + credentialProviderCache; }; std::shared_ptr -AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) +AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile, const std::string & region) { // profileDisplayName is only used for debug logging - SDK uses its default profile // when ProfileNameOverride is not set @@ -369,7 +402,7 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) bool ecsAdded = false; if (tlsContext) { addProviderToChain("STS WebIdentity", [&]() { - return createSTSWebIdentityProvider(profile, bootstrap, tlsContext.get(), allocator); + return createSTSWebIdentityProvider(profile, region, bootstrap, tlsContext.get(), allocator); }); ecsAdded = addProviderToChain("ECS", [&]() { return createECSProvider(bootstrap, tlsContext.get(), allocator); }); @@ -395,18 +428,19 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChain(chainConfig, allocator); } -AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile) +AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile, const std::string & region) { std::shared_ptr provider; + auto key = std::make_pair(profile, region); credentialProviderCache.try_emplace_and_cvisit( - profile, + key, nullptr, - [&](auto & kv) { provider = kv.second = createProviderForProfile(profile); }, + [&](auto & kv) { provider = kv.second = createProviderForProfile(profile, region); }, [&](const auto & kv) { provider = kv.second; }); if (!provider) { - credentialProviderCache.erase_if(profile, [](const auto & kv) { + credentialProviderCache.erase_if(key, [](const auto & kv) { [[maybe_unused]] auto [_, provider] = kv; return !provider; }); @@ -419,6 +453,8 @@ AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & return getCredentialsFromProvider(provider); } +} // anonymous namespace + ref makeAwsCredentialsProvider() { return make_ref(); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 92d743a16439..14bf76b87e5e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -3,11 +3,10 @@ #include "nix/util/compression.hh" #include "nix/store/derivations.hh" #include "nix/util/source-accessor.hh" -#include "nix/store/globals.hh" +#include "nix/store/nar-info-disk-cache.hh" #include "nix/store/nar-info.hh" #include "nix/util/sync.hh" #include "nix/store/remote-fs-accessor.hh" -#include "nix/store/nar-info-disk-cache.hh" #include "nix/util/nar-accessor.hh" #include "nix/util/thread-pool.hh" #include "nix/util/callback.hh" @@ -18,7 +17,6 @@ #include #include #include -#include #include #include @@ -26,11 +24,17 @@ namespace nix { +void BinaryCacheStoreConfig::anchor() {} + +void BinaryCacheStore::anchor() {} + +void NoSuchBinaryCacheFile::anchor() {} + BinaryCacheStore::BinaryCacheStore(Config & config) : config{config} { - if (!config.secretKeyFile.get().empty()) - signers.push_back(std::make_unique(SecretKey::parse(readFile(config.secretKeyFile.get())))); + if (auto & skf = config.secretKeyFile.get()) + signers.push_back(std::make_unique(SecretKey::parse(readFile(*skf)))); if (config.secretKeyFiles != "") { std::stringstream ss(config.secretKeyFiles); @@ -173,8 +177,10 @@ ref BinaryCacheStore::uploadData(Source & narSource, RepairFlag repair, { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed{fileSink, fileHashSink}; - auto compressionSink = makeCompressionSink( - config.compression, teeSinkCompressed, config.parallelCompression, config.compressionLevel); + bool parallel = config.parallelCompression.overridden ? config.parallelCompression.get() + : config.compression.get() == CompressionAlgo::zstd; + auto compressionSink = + makeCompressionSink(config.compression, teeSinkCompressed, parallel, config.compressionLevel); TeeSink teeSinkUncompressed{*compressionSink, narHashSink}; TeeSource teeSource{narSource, teeSinkUncompressed}; narAccessor = makeNarAccessor(parseNarListing(teeSource)); @@ -186,7 +192,7 @@ ref BinaryCacheStore::uploadData(Source & narSource, RepairFlag repair, auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); - narInfo->compression = config.compression.to_string(); // FIXME: Make NarInfo use CompressionAlgo + narInfo->compression = config.compression; auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; @@ -581,7 +587,13 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) stats.narReadBytes += narSize; }}; - auto decompressor = makeDecompressionSink(info->compression, uncompressedSink); + /* makeDecompressionSink used to treat empty strings as "none". It seems + impossible that it would actually end up here with an empty string though + (since an empty `Compression: ' is treated as bzip2 when parsed from a + .narinfo file and the narinfo disk cache wouldn't handle empty strings). + TODO: Revisit this and convert to an assert probably or even made + compression a non-optional field. */ + auto decompressor = makeDecompressionSink(info->compression.value_or(CompressionAlgo::none), uncompressedSink); try { getFile(info->url, *decompressor); @@ -677,7 +689,7 @@ StorePath BinaryCacheStore::addToStore( std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id) { - return realisationsPrefix + "/" + id.to_string() + ".doi"; + return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi"; } void BinaryCacheStore::queryRealisationUncached( @@ -698,7 +710,10 @@ void BinaryCacheStore::queryRealisationUncached( realisation = std::make_shared(nlohmann::json::parse(*data)); } catch (Error & e) { e.addTrace( - {}, "while parsing file '%s' as a realisation for key '%s'", outputInfoFilePath, id.to_string()); + {}, + "while parsing file '%s' as a build trace value for key '%s'", + outputInfoFilePath, + id.to_string()); throw; } return (*callbackPtr)(std::move(realisation)); @@ -714,7 +729,10 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info) { if (diskCache) diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); - upsertFile(makeRealisationPath(info.id), static_cast(info).dump(), "application/json"); + upsertFile( + makeRealisationPath(info.id), + static_cast(static_cast(info)).dump(), + "application/json"); } ref BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index 9b854f18cbad..dbbfa4af7995 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -6,6 +6,8 @@ namespace nix { +void BuildError::anchor() {} + void ExitStatusFlags::updateFromStatus(BuildResult::Failure::Status status) { // Allow selecting a subset of enum values @@ -22,7 +24,7 @@ void ExitStatusFlags::updateFromStatus(BuildResult::Failure::Status status) checkMismatch = true; break; case BuildResult::Failure::PermanentFailure: - // Also considered a permenant failure, it seems + // Also considered a permanent failure, it seems case BuildResult::Failure::InputRejected: permanentFailure = true; break; diff --git a/src/libstore/build/derivation-builder.cc b/src/libstore/build/derivation-builder.cc index 39ac40175f71..73404a08dcb8 100644 --- a/src/libstore/build/derivation-builder.cc +++ b/src/libstore/build/derivation-builder.cc @@ -25,3 +25,13 @@ void adl_serializer::to_json(json & json, const ExternalBuilder } } // namespace nlohmann + +namespace nix { + +void BuilderFailureError::anchor() {} + +void DerivationBuilder::anchor() {} + +DerivationBuilderCallbacks::~DerivationBuilderCallbacks() {} + +} // namespace nix diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 2321d7a84405..19f1cbadcc55 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -14,10 +14,11 @@ #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" #include "nix/store/local-store.hh" // TODO remove, along with remaining downcasts +#include "nix/store/outputs-query.hh" #include "nix/store/globals.hh" +#include "nix/util/current-process.hh" #include -#include #include #include #include @@ -29,10 +30,10 @@ namespace nix { DerivationBuildingGoal::DerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode, bool storeDerivation) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode, bool storeDerivation) : Goal(worker, gaveUpOnSubstitution(storeDerivation)) , drvPath(drvPath) - , drv{std::make_unique(drv)} + , drv{std::move(drv)} , buildMode(buildMode) { name = fmt("building derivation '%s'", worker.store.printStorePath(drvPath)); @@ -65,7 +66,76 @@ std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & dr return msg; } -static void runPostBuildHook( +namespace { + +struct LogSink : Sink +{ + Activity & act; + std::string currentLine; + + LogSink(Activity & act) + : act(act) + { + } + + void operator()(std::string_view data) override + { + for (auto c : data) { + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() + { + act.result(resPostBuildLogLine, currentLine); + currentLine.clear(); + } + + ~LogSink() + { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } +}; + +} // namespace + +struct PostBuildHookState +{ + const std::string hook; + Activity act; + std::unique_ptr sink; + std::unique_ptr out; + Pid pid; + + PostBuildHookState(Logger & logger, const std::string hook, const std::string drvPath) + : hook(hook) + , act(logger, + lvlTalkative, + actPostBuildHook, + fmt("running post-build-hook '%s'", hook), + Logger::Fields{drvPath}) + , out(std::make_unique()) + { + out->create(); + sink = std::make_unique(act); + } + + void complete() + { + if (int ret = pid.wait()) { + throw Error("program \"%s\" %s", hook, statusToString(ret)); + } + } +}; + +static std::unique_ptr runPostBuildHook( const WorkerSettings & workerSettings, const StoreDirConfig & store, Logger & logger, @@ -143,7 +213,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) auto outMap = [&] { for (auto * drvStore : {&worker.evalStore, &worker.store}) if (drvStore->isValidPath(depDrvPath)) - return worker.store.queryDerivationOutputMap(depDrvPath, drvStore); + return deepQueryDerivationOutputMap(worker.store, depDrvPath, drvStore); assert(false); }(); @@ -304,9 +374,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths) given this information by the downstream goal, that cannot happen anymore if the downstream goal only cares about one output, but we care about all outputs. */ - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - for (auto & [outputName, outputHash] : outputHashes) { - InitialOutput v{.outputHash = outputHash}; + for (auto & [outputName, _] : drv->outputs) { + InitialOutput v; /* TODO we might want to also allow randomizing the paths for regular CA derivations, e.g. for sake of checking @@ -699,9 +768,9 @@ Goal::Co DerivationBuildingGoal::buildWithHook( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { hook.reset(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } @@ -762,7 +831,21 @@ Goal::Co DerivationBuildingGoal::buildWithHook( StorePathSet outputPaths; for (auto & [_, output] : builtOutputs) outputPaths.insert(output.outPath); - runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + + if (worker.settings.postBuildHook.get() != "") { + auto hookState = runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + worker.childStarted(shared_from_this(), {hookState->out->readSide.get()}, false, false); + while (true) { + auto event = co_await WaitForChildEvent{}; + if (auto * output = std::get_if(&event)) { + (*hookState->sink)(output->data); + } else if (std::get_if(&event)) { + hookState->complete(); + worker.childTerminated(this); + break; + } + } + } /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will @@ -913,12 +996,12 @@ Goal::Co DerivationBuildingGoal::buildLocally( builder = localBuildCap.externalBuilder ? makeExternalDerivationBuilder( localBuildCap.localStore, - std::make_unique(*this, openLogFile, closeLogFile), + std::make_shared(*this, openLogFile, closeLogFile), std::move(params), *localBuildCap.externalBuilder) : makeDerivationBuilder( localBuildCap.localStore, - std::make_unique(*this, openLogFile, closeLogFile), + std::make_shared(*this, openLogFile, closeLogFile), std::move(params)); } @@ -962,9 +1045,9 @@ Goal::Co DerivationBuildingGoal::buildLocally( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { builder->killChild(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } @@ -998,7 +1081,21 @@ Goal::Co DerivationBuildingGoal::buildLocally( worker.markContentsGood(output.outPath); outputPaths.insert(output.outPath); } - runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + + if (worker.settings.postBuildHook.get() != "") { + auto hookState = runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + worker.childStarted(shared_from_this(), {hookState->out->readSide.get()}, false, false); + while (true) { + auto event = co_await WaitForChildEvent{}; + if (auto * output = std::get_if(&event)) { + (*hookState->sink)(output->data); + } else if (std::get_if(&event)) { + hookState->complete(); + worker.childTerminated(this); + break; + } + } + } /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will @@ -1011,24 +1108,21 @@ Goal::Co DerivationBuildingGoal::buildLocally( #endif } -static void runPostBuildHook( +static std::unique_ptr runPostBuildHook( const WorkerSettings & workerSettings, const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths) { - auto hook = workerSettings.postBuildHook; - if (hook == "") - return; +#ifdef _WIN32 + throw UnimplementedError("post-build-hook is not implemented on Windows"); +#else + auto state = + std::make_unique(logger, workerSettings.postBuildHook.get(), store.printStorePath(drvPath)); + + auto hook = workerSettings.postBuildHook.get(); - Activity act( - logger, - lvlTalkative, - actPostBuildHook, - fmt("running post-build-hook '%s'", workerSettings.postBuildHook), - Logger::Fields{store.printStorePath(drvPath)}); - PushActivity pact(act.id); OsStringMap hookEnvironment = getEnvOs(); hookEnvironment.emplace(OS_STR("DRV_PATH"), string_to_os_string(store.printStorePath(drvPath))); @@ -1036,50 +1130,34 @@ static void runPostBuildHook( OS_STR("OUT_PATHS"), string_to_os_string(chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))))); hookEnvironment.emplace(OS_STR("NIX_CONFIG"), string_to_os_string(globalConfig.toKeyValue())); - struct LogSink : Sink - { - Activity & act; - std::string currentLine; + ProcessOptions processOptions; + processOptions.allowVfork = false; - LogSink(Activity & act) - : act(act) - { - } + state->pid = startProcess( + [&] { + replaceEnv(hookEnvironment); + if (dup2(state->out->writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); - void operator()(std::string_view data) override - { - for (auto c : data) { - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } + Strings args_; + args_.push_front(hook); - void flushLine() - { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } + unix::closeExtraFDs(); - ~LogSink() - { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; + restoreProcessContext(); - LogSink sink(act); + execvp(hook.c_str(), stringsToCharPtrs(args_).data()); - runProgram2({ - .program = workerSettings.postBuildHook.get(), - .environment = hookEnvironment, - .standardOut = &sink, - .mergeStderrToStdout = true, - }); + throw SysError("executing %s", PathFmt(hook)); + }, + processOptions); + + state->out->writeSide.close(); + + return state; +#endif } BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailureError e, BuildLog & buildLog) @@ -1192,12 +1270,7 @@ LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings auto baseName = std::string(baseNameOf(store.printStorePath(drvPath))); - std::filesystem::path logDir; - if (auto localStore = dynamic_cast(&store)) - logDir = localStore->config->logDir.get(); - else - logDir = logSettings.nixLogDir; - auto dir = logDir / LocalFSStore::drvsLogDir / baseName.substr(0, 2); + auto dir = store.config.getLogDir() / LocalFSStore::drvsLogDir / baseName.substr(0, 2); createDirs(dir); auto logFileName = dir / (baseName.substr(2) + (logSettings.compressLog ? ".bz2" : "")); @@ -1282,12 +1355,12 @@ DerivationBuildingGoal::checkPathValidity(std::map & : PathStatus::Corrupt, }; } - auto drvOutput = DrvOutput{info.outputHash, i.first}; + auto drvOutput = DrvOutput{drvPath, i.first}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { .path = real->outPath, - .status = PathStatus::Valid, + .status = worker.store.isValidPath(real->outPath) ? PathStatus::Valid : PathStatus::Absent, }; } else if (info.known && info.known->isValid()) { // We know the output because it's a static output of the diff --git a/src/libstore/build/derivation-check.cc b/src/libstore/build/derivation-check.cc index d132b24649ec..8421642042b3 100644 --- a/src/libstore/build/derivation-check.cc +++ b/src/libstore/build/derivation-check.cc @@ -7,6 +7,48 @@ namespace nix { +void checkCAFixedOutput( + StoreDirConfig & store, + const StorePath & drvPath, + const DerivationOutput & outputSpec, + const ValidPathInfo & info, + Activity & act) +{ + if (const auto * dof = std::get_if(&outputSpec.raw)) { + auto & wanted = dof->ca.hash; + + /* Check wanted hash */ + assert(info.ca); + auto & got = info.ca->hash; + if (wanted != got) { + /* Throw an error after registering the path as + valid. */ + act.result( + resHashMismatch, + { + {"storePath", store.printStorePath(drvPath)}, + {"wanted", wanted}, + {"got", got}, + }); + throw BuildError( + BuildResult::Failure::HashMismatch, + "hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", + store.printStorePath(drvPath), + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true)); + } + if (!info.references.empty()) { + auto numViolations = info.references.size(); + throw BuildError( + BuildResult::Failure::HashMismatch, + "fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", + store.printStorePath(drvPath), + numViolations, + store.printStorePath(*info.references.begin())); + } + } +} + void checkOutputs( Store & store, const StorePath & drvPath, @@ -28,39 +70,7 @@ void checkOutputs( auto * outputSpec = get(drvOutputs, outputName); assert(outputSpec); - if (const auto * dof = std::get_if(&outputSpec->raw)) { - auto & wanted = dof->ca.hash; - - /* Check wanted hash */ - assert(info.ca); - auto & got = info.ca->hash; - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - act.result( - resHashMismatch, - { - {"storePath", store.printStorePath(drvPath)}, - {"wanted", wanted}, - {"got", got}, - }); - throw BuildError( - BuildResult::Failure::HashMismatch, - "hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", - store.printStorePath(drvPath), - wanted.to_string(HashFormat::SRI, true), - got.to_string(HashFormat::SRI, true)); - } - if (!info.references.empty()) { - auto numViolations = info.references.size(); - throw BuildError( - BuildResult::Failure::HashMismatch, - "fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", - store.printStorePath(drvPath), - numViolations, - store.printStorePath(*info.references.begin())); - } - } + checkCAFixedOutput(store, drvPath, *outputSpec, info, act); /* Compute the closure and closure size of some output. This is slightly tricky because some of its references (namely @@ -194,7 +204,8 @@ void checkOutputs( std::visit( overloaded{ [&](const DerivationOptions::OutputChecks & checks) { applyChecks(checks); }, - [&](const std::map::OutputChecks> & checksPerOutput) { + [&](const std::map::OutputChecks, std::less<>> & + checksPerOutput) { if (auto outputChecks = get(checksPerOutput, outputName)) applyChecks(*outputChecks); diff --git a/src/libstore/build/derivation-check.hh b/src/libstore/build/derivation-check.hh index ee2d01229524..1832264ea1a9 100644 --- a/src/libstore/build/derivation-check.hh +++ b/src/libstore/build/derivation-check.hh @@ -7,6 +7,18 @@ namespace nix { +/** + * If outputSpec is a CAFixed output, check that the actual output described in + * info meets the requirements for a CAFixed output. Do nothing if outputSpec is + * not a CAFixed output. + */ +void checkCAFixedOutput( + StoreDirConfig & store, + const StorePath & drvPath, + const DerivationOutput & outputSpec, + const ValidPathInfo & info, + Activity & act); + /** * Check that outputs meets the requirements specified by the * 'outputChecks' attribute (or the legacy diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index ba78a88343c9..55bf3923903b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2,33 +2,23 @@ #include "nix/store/build/drv-output-substitution-goal.hh" #include "nix/store/build/derivation-building-goal.hh" #include "nix/store/build/derivation-resolution-goal.hh" -#ifndef _WIN32 // TODO enable build hook on Windows -# include "nix/store/build/hook-instance.hh" -# include "nix/store/build/derivation-builder.hh" -#endif -#include "nix/util/processes.hh" -#include "nix/util/config-global.hh" #include "nix/store/build/worker.hh" #include "nix/util/util.hh" -#include "nix/util/compression.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" // Don't remove is actually needed -#include "nix/store/globals.hh" +#include "nix/store/outputs-query.hh" -#include #include #include #include #include -#include "nix/util/strings.hh" - namespace nix { DerivationGoal::DerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, Worker & worker, BuildMode buildMode, @@ -36,13 +26,7 @@ DerivationGoal::DerivationGoal( : Goal(worker, haveDerivation(storeDerivation)) , drvPath(drvPath) , wantedOutput(wantedOutput) - , drv{std::make_unique(drv)} - , outputHash{[&] { - auto outputHashes = staticOutputHashes(worker.evalStore, drv); - if (auto * mOutputHash = get(outputHashes, wantedOutput)) - return *mOutputHash; - throw Error("derivation '%s' does not have output '%s'", worker.store.printStorePath(drvPath), wantedOutput); - }()} + , drv{std::move(drv)} , buildMode(buildMode) { @@ -105,12 +89,14 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) them. */ if (worker.settings.useSubstitutes && drvOptions.substitutesAllowed(worker.settings)) { if (!checkResult) { - DrvOutput id{outputHash, wantedOutput}; + DrvOutput id{drvPath, wantedOutput}; auto g = worker.makeDrvOutputSubstitutionGoal(id); waitees.insert(g); co_await await(std::move(waitees)); if (nrFailed == 0) { + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath))); co_await await(std::move(waitees)); @@ -131,6 +117,8 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) } } + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) co_await await(std::move(waitees)); trace("all outputs substituted (maybe)"); @@ -164,13 +152,11 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) worker.store.printStorePath(drvPath)); } - auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, *drv, buildMode); + auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, drv, buildMode); /* We'll handle the error below. */ resolutionGoal->preserveFailure = true; - { - Goals waitees{resolutionGoal}; - co_await await(std::move(waitees)); - } + co_await await({resolutionGoal}); + if (nrFailed != 0) { auto * failure = resolutionGoal->buildResult.tryGetFailure(); assert(failure); @@ -180,8 +166,13 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) if (resolutionGoal->resolvedDrv) { auto & [pathResolved, drvResolved] = *resolutionGoal->resolvedDrv; - auto resolvedDrvGoal = - worker.makeDerivationGoal(pathResolved, drvResolved, wantedOutput, buildMode, /*storeDerivation=*/true); + auto resolvedDrvGoal = worker.makeDerivationGoal( + pathResolved, + make_ref(drvResolved), + wantedOutput, + buildMode, + /*storeDerivation=*/true); + { Goals waitees{resolvedDrvGoal}; co_await await(std::move(waitees)); @@ -194,12 +185,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) // No `std::visit` for coroutines yet if (auto * successP = resolvedResult.tryGetSuccess()) { auto & success = *successP; - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - auto resolvedHashes = staticOutputHashes(worker.store, drvResolved); - - auto outputHash = get(outputHashes, wantedOutput); - auto resolvedHash = get(resolvedHashes, wantedOutput); - if ((!outputHash) || (!resolvedHash)) + if (!drv->outputs.contains(wantedOutput)) throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", worker.store.printStorePath(drvPath), @@ -208,7 +194,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto realisation = [&] { auto take1 = get(success.builtOutputs, wantedOutput); if (take1) - return static_cast(*take1); + return *take1; /* The above `get` should work. But stateful tracking of outputs in resolvedResult, this can get out of sync with the @@ -216,7 +202,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) check the store directly if it fails. */ auto take2 = worker.evalStore.queryRealisation( DrvOutput{ - .drvHash = *resolvedHash, + .drvPath = pathResolved, .outputName = wantedOutput, }); if (take2) @@ -228,18 +214,6 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) wantedOutput); }(); - if (!drv->type().isImpure()) { - Realisation newRealisation{ - realisation, - { - .drvHash = *outputHash, - .outputName = wantedOutput, - }}; - newRealisation.signatures.clear(); - worker.store.signRealisation(newRealisation); - worker.store.registerDrvOutput(newRealisation); - } - auto status = success.status; if (status == BuildResult::Success::AlreadyValid) status = BuildResult::Success::ResolvesToAlreadyValid; @@ -255,18 +229,17 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) assert(false); } + /* We don't need it any more and don't want to hold on to it while suspended. */ + resolutionGoal.reset(); + /* Give up on substitution for the output we want, actually build this derivation */ - auto g = worker.makeDerivationBuildingGoal(drvPath, *drv, buildMode, storeDerivation); + auto g = worker.makeDerivationBuildingGoal(drvPath, drv, buildMode, storeDerivation); /* We will finish with it ourselves, as if we were the derivational goal. */ g->preserveFailure = true; - { - Goals waitees; - waitees.insert(g); - co_await await(std::move(waitees)); - } + co_await await({g}); trace("outer build done"); @@ -278,16 +251,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* In checking mode, the builder will not register any outputs. So we want to make sure the ones that we wanted to check are properly there. */ - success.builtOutputs = {{ - wantedOutput, - { - assertPathValidity(), - { - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}; + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; } else { /* Otherwise the builder will give us info for out output, but also for other outputs. Filter down to just our output so as @@ -306,16 +270,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) if (success.builtOutputs.count(wantedOutput) == 0) { debug( "BUG! wanted output '%s' not in builtOutputs, working around by adding it manually", wantedOutput); - success.builtOutputs = {{ - wantedOutput, - { - assertPathValidity(), - { - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}; + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; } } } @@ -336,7 +291,7 @@ Goal::Co DerivationGoal::repairClosure() auto outputs = [&] { for (auto * drvStore : {&worker.evalStore, &worker.store}) if (drvStore->isValidPath(drvPath)) - return worker.store.queryDerivationOutputMap(drvPath, drvStore); + return deepQueryDerivationOutputMap(worker.store, drvPath, drvStore); OutputPathMap res; for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) @@ -412,7 +367,7 @@ std::optional> DerivationGoal::checkPa if (drv->type().isImpure()) return std::nullopt; - auto drvOutput = DrvOutput{outputHash, wantedOutput}; + auto drvOutput = DrvOutput{drvPath, wantedOutput}; std::optional mRealisation; @@ -452,7 +407,7 @@ std::optional> DerivationGoal::checkPa Realisation{ *mRealisation, { - .drvHash = outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }, }); @@ -483,16 +438,7 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke auto res = Goal::doneSuccess( BuildResult::Success{ .status = status, - .builtOutputs = {{ - wantedOutput, - { - std::move(builtOutput), - DrvOutput{ - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}, + .builtOutputs = {{wantedOutput, std::move(builtOutput)}}, }); logger->result( diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index 81c698e18563..6f55ea29d96e 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -7,10 +7,10 @@ namespace nix { DerivationResolutionGoal::DerivationResolutionGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode) : Goal(worker, resolveDerivation()) , drvPath(drvPath) - , drv{std::make_unique(drv)} + , drv(std::move(drv)) , buildMode{buildMode} { name = fmt("resolving derivation '%s'", worker.store.printStorePath(drvPath)); @@ -22,60 +22,49 @@ std::string DerivationResolutionGoal::key() return "dc$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } -/** - * Used for `inputGoals` local variable below - */ -struct value_comparison -{ - template - bool operator()(const ref & lhs, const ref & rhs) const - { - return *lhs < *rhs; - } -}; - Goal::Co DerivationResolutionGoal::resolveDerivation() { Goals waitees; - std::map, GoalPtr, value_comparison> inputGoals; - - { - std::function, const DerivedPathMap::ChildNode &)> - addWaiteeDerivedPath; - - addWaiteeDerivedPath = [&](ref inputDrv, - const DerivedPathMap::ChildNode & inputNode) { - if (!inputNode.value.empty()) { - auto g = worker.makeGoal( - DerivedPath::Built{ - .drvPath = inputDrv, - .outputs = inputNode.value, - }, - buildMode == bmRepair ? bmRepair : bmNormal); - inputGoals.insert_or_assign(inputDrv, g); - waitees.insert(std::move(g)); - } - for (const auto & [outputName, childNode] : inputNode.childMap) - addWaiteeDerivedPath( - make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); - }; - - for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { - /* Ensure that pure, non-fixed-output derivations don't - depend on impure derivations. */ - if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() - && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); - if (inputDrv.type().isImpure()) - throw Error( - "pure derivation '%s' depends on impure derivation '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(inputDrvPath)); - } - - addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); + using ValueComparison = decltype([](const ref & lhs, const ref & rhs) { + /* Compare the values, not the pointers themselves. */ + return *lhs < *rhs; + }); + + std::map, GoalPtr, ValueComparison> inputGoals; + + auto addWaiteeDerivedPath = [&worker = worker, buildMode = buildMode, &waitees, &inputGoals]( + this const auto & self, + ref inputDrv, + const DerivedPathMap::ChildNode & inputNode) -> void { + if (!inputNode.value.empty()) { + auto g = worker.makeGoal( + DerivedPath::Built{ + .drvPath = inputDrv, + .outputs = inputNode.value, + }, + buildMode == bmRepair ? bmRepair : bmNormal); + inputGoals.insert_or_assign(inputDrv, g); + waitees.insert(std::move(g)); } + for (const auto & [outputName, childNode] : inputNode.childMap) + self(make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); + }; + + for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { + /* Ensure that pure, non-fixed-output derivations don't + depend on impure derivations. */ + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() + && !drv->type().isFixed()) { + auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); + if (inputDrv.type().isImpure()) + throw Error( + "pure derivation '%s' depends on impure derivation '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(inputDrvPath)); + } + + addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); } co_await await(std::move(waitees)); diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index 7621268c1574..14864ed052f8 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -49,18 +49,17 @@ void DerivationTrampolineGoal::commonInit() DerivationTrampolineGoal::~DerivationTrampolineGoal() {} -static StorePath pathPartOfReq(const SingleDerivedPath & req) -{ - return std::visit( - overloaded{ - [&](const SingleDerivedPath::Opaque & bo) { return bo.path; }, - [&](const SingleDerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); }, - }, - req.raw()); -} - std::string DerivationTrampolineGoal::key() { + auto pathPartOfReq = [](this const auto & self, const SingleDerivedPath & req) -> StorePath { + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) { return bo.path; }, + [&](const SingleDerivedPath::Built & bfd) { return self(*bfd.drvPath); }, + }, + req.raw()); + }; + return "da$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + DerivedPath::Built{ .drvPath = drvReq, .outputs = wantedOutputs, @@ -155,8 +154,10 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation /* Build this step! */ + auto sharedDrv = make_ref(std::move(drv)); + for (auto & output : resolvedWantedOutputs) { - auto g = upcast_goal(worker.makeDerivationGoal(drvPath, drv, output, buildMode, false)); + auto g = upcast_goal(worker.makeDerivationGoal(drvPath, sharedDrv, output, buildMode, false)); g->preserveFailure = true; /* We will finish with it ourselves, as if we were the derivational goal. */ concreteDrvGoals.insert(std::move(g)); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 71f0ae1e7db4..ed653fed7555 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -1,7 +1,7 @@ +#include "goal-impl.hh" + #include "nix/store/build/drv-output-substitution-goal.hh" -#include "nix/util/finally.hh" #include "nix/store/build/worker.hh" -#include "nix/store/build/substitution-goal.hh" #include "nix/util/callback.hh" namespace nix { @@ -10,7 +10,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput & id, Worke : Goal(worker, init()) , id(id) { - name = fmt("substitution of '%s'", id.to_string()); + name = fmt("substitution of '%s'", id.render(worker.store)); trace("created"); } @@ -30,55 +30,9 @@ Goal::Co DrvOutputSubstitutionGoal::init() for (const auto & sub : subs) { trace("trying next substituter"); - /* The callback of the curl download below can outlive `this` (if - some other error occurs), so it must not touch `this`. So put - the shared state in a separate refcounted object. */ - auto outPipe = std::make_shared(); -#ifndef _WIN32 - outPipe->create(); -#else - outPipe->createAsyncPipe(worker.ioport.get()); -#endif - - auto promise = std::make_shared>>(); - - sub->queryRealisation( - id, {[outPipe(outPipe), promise(promise)](std::future> res) { - try { - Finally updateStats([&]() { outPipe->writeSide.close(); }); - promise->set_value(res.get()); - } catch (...) { - promise->set_exception(std::current_exception()); - } - }}); - - worker.childStarted( - shared_from_this(), - { -#ifndef _WIN32 - outPipe->readSide.get() -#else - &*outPipe -#endif - }, - true, - false); - - while (true) { - auto event = co_await WaitForChildEvent{}; - if (std::get_if(&event)) { - // Doesn't process child output - } else if (std::get_if(&event)) { - break; - } else if (std::get_if(&event)) { - unreachable(); - } - } - - worker.childTerminated(this); - try { - outputInfo = promise->get_future().get(); + outputInfo = co_await AsyncCallback>( + [sub, id = this->id](auto cb) { sub->queryRealisation(id, std::move(cb)); }); } catch (std::exception & e) { printError(e.what()); substituterFailed = true; @@ -93,7 +47,8 @@ Goal::Co DrvOutputSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); + debug( + "derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store)); if (substituterFailed) { worker.failedSubstitutions++; @@ -108,7 +63,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() std::string DrvOutputSubstitutionGoal::key() { - return "a$" + std::string(id.to_string()); + return "a$" + std::string(id.render(worker.store)); } } // namespace nix diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 385d61b8cec0..dbc688cb033b 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -2,7 +2,6 @@ #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/build/derivation-trampoline-goal.hh" -#include "nix/store/local-store.hh" #include "nix/util/strings.hh" namespace nix { diff --git a/src/libstore/build/goal-impl.hh b/src/libstore/build/goal-impl.hh new file mode 100644 index 000000000000..b35d2b5df632 --- /dev/null +++ b/src/libstore/build/goal-impl.hh @@ -0,0 +1,52 @@ +#pragma once +#include "nix/store/build/goal.hh" +#include "nix/store/build/worker.hh" + +namespace nix { + +template +auto Goal::promise_type::await_transform(AsyncCallback && ac) +{ + struct Awaiter + { + fun)> fn; + std::shared_ptr> promise; + + bool await_ready() + { + return false; + } + + void await_suspend(handle_type h) + { + auto goal = h.promise().goal; + promise = std::make_shared>(); + + fn(Callback([promise = promise, + maybeWaker = goal->worker.getCrossThreadWaker(), + goalWeak = goal->weak_from_this()](std::future res) { + try { + promise->set_value(res.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + + /* The Worker might have already died (and the waker with it) by + the time the callback fired. */ + if (auto waker = maybeWaker.lock()) + waker->enqueue(goalWeak); + })); + + goal->worker.waitForCompletion(goal->shared_from_this()); + } + + T await_resume() + { + return promise->get_future().get(); + } + }; + + return Awaiter{.fn = std::move(ac.fn)}; +} + +} // namespace nix diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index ef3501b00916..5b1be405d246 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -4,24 +4,29 @@ namespace nix { +void WorkerSettings::anchor() {} + TimedOut::TimedOut(time_t maxDuration) : CloneableError(BuildResult::Failure::TimedOut, "timed out after %1% seconds", maxDuration) , maxDuration(maxDuration) { } +void TimedOut::anchor() {} + +void Goal::anchor() {} + using Co = nix::Goal::Co; using promise_type = nix::Goal::promise_type; -using ChildEvents = decltype(promise_type::childEvents); -void ChildEvents::pushChildEvent(ChildOutput event) +void Goal::ChildEvents::pushChildEvent(ChildOutput event) { if (childTimeout) return; // Already timed out, ignore childOutputs.push(std::move(event)); } -void ChildEvents::pushChildEvent(ChildEOF event) +void Goal::ChildEvents::pushChildEvent(ChildEOF event) { if (childTimeout) return; // Already timed out, ignore @@ -29,20 +34,20 @@ void ChildEvents::pushChildEvent(ChildEOF event) childEOF = std::move(event); } -void ChildEvents::pushChildEvent(TimedOut event) +void Goal::ChildEvents::pushChildEvent(TimedOut event) { // Timeout is immediate - flush pending events childOutputs = {}; childEOF.reset(); - childTimeout = std::move(event); + childTimeout = std::make_unique(std::move(event)); } -bool ChildEvents::hasChildEvent() const +bool Goal::ChildEvents::hasChildEvent() const { return !childOutputs.empty() || childEOF || childTimeout; } -Goal::ChildEvent ChildEvents::popChildEvent() +Goal::ChildEvent Goal::ChildEvents::popChildEvent() { if (!childOutputs.empty()) { auto event = std::move(childOutputs.front()); @@ -52,23 +57,28 @@ Goal::ChildEvent ChildEvents::popChildEvent() if (childEOF) return *std::exchange(childEOF, std::nullopt); if (childTimeout) - return *std::exchange(childTimeout, std::nullopt); + return std::exchange(childTimeout, nullptr); unreachable(); } using handle_type = nix::Goal::handle_type; using Suspend = nix::Goal::Suspend; -Co::Co(Co && rhs) +Co::Co(Co && rhs) noexcept { this->handle = rhs.handle; rhs.handle = nullptr; } -void Co::operator=(Co && rhs) +Co & Co::operator=(Co && rhs) noexcept { - this->handle = rhs.handle; + if (handle) { + handle.promise().alive = false; + handle.destroy(); + } + handle = rhs.handle; rhs.handle = nullptr; + return *this; } Co::~Co() @@ -157,9 +167,7 @@ std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller) bool CompareGoalPtrs::operator()(const GoalPtr & a, const GoalPtr & b) const { - std::string s1 = a->key(); - std::string s2 = b->key(); - return s1 < s2; + return a->keyCached() < b->keyCached(); } void addToWeakGoals(WeakGoals & goals, GoalPtr p) @@ -275,22 +283,19 @@ void Goal::work() void Goal::handleChildOutput(Descriptor fd, std::string_view data) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); + childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); worker.wakeUp(shared_from_this()); } void Goal::handleEOF(Descriptor fd) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildEOF{fd}); + childEvents.pushChildEvent(ChildEOF{fd}); worker.wakeUp(shared_from_this()); } void Goal::timedOut(TimedOut && ex) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(std::move(ex)); + childEvents.pushChildEvent(std::move(ex)); worker.wakeUp(shared_from_this()); } @@ -308,6 +313,13 @@ Goal::Co Goal::waitForAWhile() co_return Return{}; } +Goal::Co Goal::waitUntilWoken() +{ + worker.waitForCompletion(shared_from_this()); + co_await Suspend{}; + co_return Return{}; +} + Goal::Co Goal::waitForBuildSlot() { worker.waitForBuildSlot(shared_from_this()); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index a76f6e8d4dd9..de49582cd4c9 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -1,11 +1,11 @@ +#include "goal-impl.hh" + #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/nar-info.hh" -#include "nix/util/finally.hh" +#include "nix/store/worker-settings.hh" #include "nix/util/signals.hh" -#include "nix/store/globals.hh" - -#include +#include "nix/util/callback.hh" #include @@ -91,9 +91,9 @@ Goal::Co PathSubstitutionGoal::init() } try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath & e) { + info = co_await AsyncCallback>( + [sub, path = subPath.value_or(storePath)](auto cb) { sub->queryPathInfo(path, std::move(cb)); }); + } catch (InvalidPath &) { continue; } catch (SubstituterDisabled & e) { continue; @@ -165,7 +165,7 @@ Goal::Co PathSubstitutionGoal::init() } if (lastStoresException.has_value()) { if (!worker.settings.tryFallback) { - throw *lastStoresException; + throw std::move(*lastStoresException); } else logError(lastStoresException->info()); } @@ -223,56 +223,53 @@ Goal::Co PathSubstitutionGoal::tryToRun( auto maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); -#ifndef _WIN32 - outPipe.create(); -#else - outPipe.createAsyncPipe(worker.ioport.get()); -#endif - auto promise = std::promise>(); - - thr = std::thread([this, &promise, &subPath, &sub]() { + auto future = promise.get_future(); + + /* Be careful with ownership. cleanup() doesn't signal the worker thread + to cleanly shutdown, so the worker can die while the thread is still + running. That's why we use weak_ptr for everything that is owned by the + Worker. */ + thr = std::thread([weakGoal = weak_from_this(), + promise = std::move(promise), + subPath, + storePath = storePath, + repair = repair, + sub, + maybeWaker = worker.getCrossThreadWaker(), + maybeWorkerStore = worker.store.weak_from_this()]() mutable { try { ReceiveInterrupts receiveInterrupts; - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide.close(); }); + /* The Worker might have died while we were starting up. */ + auto workerStore = maybeWorkerStore.lock(); + if (!workerStore) + return; Activity act( *logger, actSubstitute, - Logger::Fields{worker.store.printStorePath(storePath), sub->config.getHumanReadableURI()}); + Logger::Fields{workerStore->printStorePath(storePath), sub->config.getHumanReadableURI()}); PushActivity pact(act.id); promise.set_value( - copyStorePath(*sub, worker.store, subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs)); + copyStorePath(*sub, *workerStore, subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs)); } catch (...) { promise.set_exception(std::current_exception()); } + + /* The Worker might have already died (and the waker with it) by the + time we finished. N.B. if enqueueing to the waker throws, we better + std::terminate, since something has gone very wrong. This intentionally + lets the thread crash on exceptions for that reason. */ + if (auto waker = maybeWaker.lock()) + waker->enqueue(weakGoal); }); - worker.childStarted( - shared_from_this(), - { -#ifndef _WIN32 - outPipe.readSide.get() -#else - &outPipe -#endif - }, - true, - false); - - while (true) { - auto event = co_await WaitForChildEvent{}; - if (std::get_if(&event)) { - // Substitution doesn't process child output - } else if (std::get_if(&event)) { - break; - } else if (std::get_if(&event)) { - unreachable(); // Substitution doesn't use timeouts - } - } + /* Use up the substitution slot. */ + worker.childStarted(shared_from_this(), /*channels=*/{}, /*inBuildSlot=*/true, /*respectTimeouts=*/false); + /* Suspend until the thread finishes. */ + co_await waitUntilWoken(); trace("substitute finished"); @@ -282,7 +279,7 @@ Goal::Co PathSubstitutionGoal::tryToRun( std::shared_ptr provenance; try { - auto info = promise.get_future().get(); + auto info = future.get(); if (info) provenance = info->provenance; } catch (std::exception & e) { @@ -341,8 +338,6 @@ void PathSubstitutionGoal::cleanup() thr.join(); worker.childTerminated(this, JobCategory::Substitution); } - - outPipe.close(); } catch (...) { ignoreExceptionInDestructor(); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 6bcf897a61bb..2229df9f2f23 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -17,7 +17,9 @@ namespace nix { Worker::Worker(Store & store, Store & evalStore) - : act(*logger, actRealise) + /* Can't use make_ref, because the constructor is private. */ + : wakerState(ref(new Waker{})) + , act(*logger, actRealise) , actDerivations(*logger, actBuilds) , actSubstitutions(*logger, actCopyPaths) #ifdef _WIN32 @@ -33,6 +35,7 @@ Worker::Worker(Store & store, Store & evalStore) #ifdef _WIN32 if (!ioport) throw windows::WinError("CreateIoCompletionPort"); + wakerState->ioport = ioport.get(); #endif nrLocalBuilds = 0; nrSubstitutions = 0; @@ -58,7 +61,7 @@ std::shared_ptr Worker::initGoalIfNeeded(std::weak_ptr & goal_weak, Args & if (auto goal = goal_weak.lock()) return goal; - auto goal = std::make_shared(args...); + auto goal = std::make_shared(std::forward(args)...); goal_weak = goal; wakeUp(goal); return goal; @@ -85,25 +88,32 @@ std::shared_ptr Worker::makeDerivationTrampolineGoal( std::shared_ptr Worker::makeDerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, BuildMode buildMode, bool storeDerivation) { return initGoalIfNeeded( - derivationGoals[drvPath][wantedOutput], drvPath, drv, wantedOutput, *this, buildMode, storeDerivation); + derivationGoals[drvPath][wantedOutput], + drvPath, + std::move(drv), + wantedOutput, + *this, + buildMode, + storeDerivation); } std::shared_ptr -Worker::makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode) +Worker::makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode) { return initGoalIfNeeded(derivationResolutionGoals[drvPath], drvPath, drv, *this, buildMode); } std::shared_ptr Worker::makeDerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, BuildMode buildMode, bool storeDerivation) + const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation) { - return initGoalIfNeeded(derivationBuildingGoals[drvPath], drvPath, drv, *this, buildMode, storeDerivation); + return initGoalIfNeeded( + derivationBuildingGoals[drvPath], drvPath, std::move(drv), *this, buildMode, storeDerivation); } std::shared_ptr Worker::makePathSubstitutionGoal( @@ -131,64 +141,31 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) req.raw()); } -/** - * This function is polymorphic (both via type parameters and - * overloading) and recursive in order to work on a various types of - * trees - * - * @return Whether the tree node we are processing is not empty / should - * be kept alive. In the case of this overloading the node in question - * is the leaf, the weak reference itself. If the weak reference points - * to the goal we are looking for, our caller can delete it. In the - * inductive case where the node is an interior node, we'll likewise - * return whether the interior node is non-empty. If it is empty - * (because we just deleted its last child), then our caller can - * likewise delete it. - */ -template -static bool removeGoal(std::shared_ptr goal, std::weak_ptr & gp) -{ - return gp.lock() != goal; -} - -template -static bool removeGoal(std::shared_ptr goal, std::map & goalMap) -{ - /* !!! inefficient */ - for (auto i = goalMap.begin(); i != goalMap.end();) { - if (!removeGoal(goal, i->second)) - i = goalMap.erase(i); - else - ++i; - } - return !goalMap.empty(); -} - -template -static bool -removeGoal(std::shared_ptr goal, typename DerivedPathMap>>::ChildNode & node) -{ - bool valueKeep = removeGoal(goal, node.value); - bool childMapKeep = removeGoal(goal, node.childMap); - return valueKeep || childMapKeep; -} - void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, derivationTrampolineGoals.map); - else if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, derivationGoals); - else if (auto drvResolutionGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvResolutionGoal, derivationResolutionGoals); - else if (auto drvBuildingGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvBuildingGoal, derivationBuildingGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, substitutionGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, drvOutputSubstitutionGoals); - else - assert(false); + if (auto drvGoal = std::dynamic_pointer_cast(goal)) { + derivationTrampolineGoals.removeSlot(*drvGoal->drvReq, [&](auto & node) { + node.value.erase(drvGoal->wantedOutputs); + /* Return true if ancestors don't need to be pruned. */ + return !node.value.empty(); + }); + } else if (auto drvGoal = std::dynamic_pointer_cast(goal)) { + if (auto it = derivationGoals.find(drvGoal->drvPath); it != derivationGoals.end()) { + it->second.erase(drvGoal->wantedOutput); + if (it->second.empty()) + derivationGoals.erase(it); + } + } else if (auto drvResolutionGoal = std::dynamic_pointer_cast(goal)) { + derivationResolutionGoals.erase(drvResolutionGoal->drvPath); + } else if (auto drvBuildingGoal = std::dynamic_pointer_cast(goal)) { + derivationBuildingGoals.erase(drvBuildingGoal->drvPath); + } else if (auto subGoal = std::dynamic_pointer_cast(goal)) { + substitutionGoals.erase(subGoal->storePath); + } else if (auto subGoal = std::dynamic_pointer_cast(goal)) { + drvOutputSubstitutionGoals.erase(subGoal->id); + } else { + unreachable(); + } if (topGoals.find(goal) != topGoals.end()) { topGoals.erase(goal); @@ -197,15 +174,6 @@ void Worker::removeGoal(GoalPtr goal) if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) topGoals.clear(); } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) - wakeUp(goal); - } - - waitingForAnyGoal.clear(); } void Worker::wakeUp(GoalPtr goal) @@ -244,21 +212,21 @@ void Worker::childStarted( nrLocalBuilds++; break; case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; default: + /* Doesn't make sense, since there are only building and substitution slots. */ unreachable(); } } } -void Worker::childTerminated(Goal * goal, bool wakeSleepers) +void Worker::childTerminated(Goal * goal) { - childTerminated(goal, goal->jobCategory(), wakeSleepers); + childTerminated(goal, goal->jobCategory()); } -void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers) +void Worker::childTerminated(Goal * goal, JobCategory jobCategory) { + // FIXME: Inefficient. Make children a map from Goal -> Child instead. auto i = std::find_if(children.begin(), children.end(), [&](const Child & child) { return child.goal2 == goal; }); if (i == children.end()) return; @@ -274,75 +242,64 @@ void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSlee nrLocalBuilds--; break; case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; default: + /* Doesn't make sense, since there are only building and substitution slots. */ unreachable(); } } children.erase(i); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) - wakeUp(goal); + auto & waiting = jobCategory == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild; + + /* Wake up goals waiting for a build slot. Wake at most one waiter to avoid + starting unnecessary work (that is accompanied by coroutine frame allocation). */ + auto it = waiting.begin(); + while (it != waiting.end()) { + if (auto goal = it->lock()) { + waiting.erase(it); + wakeUp(goal); + break; } - - wantingToBuild.clear(); + it = waiting.erase(it); } } void Worker::waitForBuildSlot(GoalPtr goal) { goal->trace("wait for build slot"); - bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution; - if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) - || (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs)) - wakeUp(goal); /* we can do it right away */ + + bool slotAvailable = [&] { + if (goal->jobCategory() == JobCategory::Substitution) + return getNrSubstitutions() < settings.maxSubstitutionJobs; + else + return getNrLocalBuilds() < settings.maxBuildJobs; + }(); + + if (slotAvailable) + wakeUp(goal); /* Can do it right away. */ else - addToWeakGoals(wantingToBuild, goal); + addToWeakGoals(goal->jobCategory() == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild, goal); } -void Worker::waitForAnyGoal(GoalPtr goal) +void Worker::waitForAWhile(GoalPtr goal) { - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); + goal->trace("wait for a while"); + addToWeakGoals(waitingForAWhile, goal); } -void Worker::waitForAWhile(GoalPtr goal) +void Worker::waitForCompletion(GoalPtr goal) { - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); + goal->trace("waiting for completion callback"); + addToWeakGoals(waitingForCompletion, goal); } void Worker::run(const Goals & _topGoals) { - std::vector topPaths; - - for (auto & i : _topGoals) { - topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back( - DerivedPath::Built{ - .drvPath = goal->drvReq, - .outputs = goal->wantedOutputs, - }); - } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Opaque{goal->storePath}); - } - } - - /* Call queryMissing() to efficiently query substitutes. */ - store.queryMissing(topPaths); - debug("entered goal loop"); + for (std::shared_ptr goal : _topGoals) + topGoals.insert(std::move(goal)); while (1) { - checkInterrupt(); // TODO GC interface? @@ -359,9 +316,24 @@ void Worker::run(const Goals & _topGoals) awake2.insert(goal); } awake.clear(); + for (auto & goal : awake2) { checkInterrupt(); + + std::chrono::time_point startTime; + if (verbosity >= lvlVomit) + startTime = std::chrono::steady_clock::now(); + goal->work(); + + /* Useful for tracing which goals hod the event loop. */ + vomit( + "worker event loop worked goal '%1%' for %2$.3fms", + goal->name, + std::chrono::duration_cast>( + std::chrono::steady_clock::now() - startTime) + .count()); + if (topGoals.empty()) break; // stuff may have been cancelled } @@ -370,8 +342,8 @@ void Worker::run(const Goals & _topGoals) if (topGoals.empty()) break; - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) + /* Wait for input or completion callbacks. */ + if (!children.empty() || !waitingForAWhile.empty() || !waitingForCompletion.empty()) waitForInput(); else if (awake.empty() && 0U == settings.maxBuildJobs) { if (Machine::parseConfig({nix::settings.thisSystem}, nix::settings.getWorkerSettings().builders).empty()) @@ -463,6 +435,17 @@ void Worker::waitForInput() state.fdToPollStatus[j] = state.pollStatus.size() - 1; } } + + { + auto wakeupPipeFd = wakerState->wakeupPipe.pipe.readSide.get(); + state.pollStatus.push_back( + pollfd{ + .fd = wakeupPipeFd, + .events = POLLIN, + }); + + state.fdToPollStatus[wakeupPipeFd] = state.pollStatus.size() - 1; + } #endif state.poll( @@ -508,6 +491,17 @@ void Worker::waitForInput() } } +#ifndef _WIN32 + std::set wakerChannels{wakerState->wakeupPipe.pipe.readSide.get()}; + state.iterate( + wakerChannels, + [&](Descriptor k, std::string_view data) { wakerState->wakeAll(*this); }, + [](Descriptor fd) { unreachable(); }); +#else + /* Slightly less optimal on windows. We don't use a wakeup pipe and signal the ioport directly. */ + wakerState->wakeAll(*this); +#endif + if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { lastWokenUp = after; for (auto & i : waitingForAWhile) { @@ -519,6 +513,39 @@ void Worker::waitForInput() } } +std::weak_ptr Worker::getCrossThreadWaker() +{ + return wakerState.get_ptr(); +} + +void Worker::Waker::wakeAll(Worker & worker) +{ + /* Wake up all goals that have been enqueued by asynchronous completion callbacks. */ + auto wakeupQueue(wakeupQueue_.lock()); +#ifndef _WIN32 + wakeupPipe.drain(); +#endif + while (!wakeupQueue->empty()) { + auto ptr = wakeupQueue->front().lock(); + wakeupQueue->pop(); + if (ptr) { + worker.waitingForCompletion.erase(ptr); + worker.wakeUp(ptr); + } + } +} + +void Worker::Waker::enqueue(WeakGoalPtr goal) +{ + wakeupQueue_.lock()->push(goal); +#ifdef _WIN32 + PostQueuedCompletionStatus( + ioport, /*dwNumberOfBytesTransferred=*/0, /*dwCompletionKey=*/0, /*lpOverlapped=*/nullptr); +#else + wakeupPipe.notify(); +#endif +} + bool Worker::pathContentsGood(const StorePath & path) { auto i = pathContentsGoodCache.find(path); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 3dd66be2ccae..2ad98fa92741 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -10,6 +10,8 @@ namespace nix { +void BuildEnvFileConflictError::anchor() {} + RegisterBuiltinBuilder::BuiltinBuilders & RegisterBuiltinBuilder::builtinBuilders() { static RegisterBuiltinBuilder::BuiltinBuilders builders; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1ff66eb89fa4..fd38b5a4afd8 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -1,7 +1,6 @@ #include "nix/store/builtins.hh" #include "nix/store/filetransfer.hh" #include "nix/store/store-api.hh" -#include "nix/store/globals.hh" #include "nix/util/archive.hh" #include "nix/util/compression.hh" #include "nix/util/file-system.hh" @@ -14,12 +13,13 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) this to be stored in a file. It would be nice if we could just pass a pointer to the data. */ if (ctx.netrcData != "") { - fileTransferSettings.netrcFile = "netrc"; + fileTransferSettings.netrcFile = ctx.tmpDirInSandbox / "netrc"; writeFile(fileTransferSettings.netrcFile.get(), ctx.netrcData, 0600); } - fileTransferSettings.caFile = "ca-certificates.crt"; - writeFile(*fileTransferSettings.caFile.get(), ctx.caFileData, 0600); + auto caFilePath = ctx.tmpDirInSandbox / "ca-certificates.crt"; + fileTransferSettings.caFile = std::optional{caFilePath}; + writeFile(caFilePath, ctx.caFileData, 0600); auto out = get(ctx.drv.outputs, "out"); if (!out) @@ -54,7 +54,8 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) } #endif - auto decompressor = makeDecompressionSink(unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); + auto decompressor = makeDecompressionSink( + unpack && hasSuffix(mainUrl, ".xz") ? CompressionAlgo::xz : CompressionAlgo::none, sink); fileTransfer->download(std::move(request), *decompressor); decompressor->finish(); }); diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index c5e4e3897992..931d7ab7d959 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -2,40 +2,21 @@ -- Won't be loaded unless the experimental feature `ca-derivations` -- is enabled -create table if not exists Realisations ( +-- Why the `*V` tables +-- +-- We are trying to keep different versions of the experiment to have +-- completely independent extra schemas from one another. This will +-- enable people to switch between versions of the experiment (including +-- newer to older) without migrating between them, but at the cost +-- of having many abandoned tables lying around. Closer to the end of +-- the experiment, we'll provide guidance on how to clean this up. + +create table if not exists BuildTraceV3 ( id integer primary key autoincrement not null, drvPath text not null, outputName text not null, -- symbolic output id, usually "out" - outputPath integer not null, - signatures text, -- space-separated list - foreign key (outputPath) references ValidPaths(id) on delete cascade -); - -create index if not exists IndexRealisations on Realisations(drvPath, outputName); - --- We can end-up in a weird edge-case where a path depends on itself because --- it’s an output of a CA derivation, that happens to be the same as one of its --- dependencies. --- In that case we have a dependency loop (path -> realisation1 -> realisation2 --- -> path) that we need to break by removing the dependencies between the --- realisations -create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths - begin - delete from RealisationsRefs where realisationReference in ( - select id from Realisations where outputPath = old.id - ); - end; - -create table if not exists RealisationsRefs ( - referrer integer not null, - realisationReference integer, - foreign key (referrer) references Realisations(id) on delete cascade, - foreign key (realisationReference) references Realisations(id) on delete restrict + outputPath text not null, + signatures text -- space-separated list ); --- used by deletion trigger -create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); --- used by QueryRealisationReferences -create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); --- used by cascade deletion when ValidPaths is deleted -create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); +create index if not exists IndexBuildTraceV3 on BuildTraceV3(drvPath, outputName); diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index 6e3a1921dc04..e9bd5fa56459 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -1,11 +1,10 @@ #include "nix/util/serialise.hh" #include "nix/store/path-with-outputs.hh" -#include "nix/store/store-api.hh" #include "nix/store/build-result.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" -#include "nix/util/archive.hh" #include "nix/store/derivations.hh" +#include "nix/store/store-dir-config.hh" #include "nix/util/signature/local-keys.hh" #include @@ -47,34 +46,6 @@ void CommonProto::Serialise::write( conn.to << renderContentAddress(ca); } -Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - try { - return nlohmann::json::parse(rawInput); - } catch (Error & e) { - e.addTrace({}, "while parsing a realisation object in the remote protocol"); - throw; - } -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << static_cast(realisation).dump(); -} - -DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); -} - std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index fd61b12efeee..db3677151416 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -4,11 +4,13 @@ namespace nix { CommonSSHStoreConfig::CommonSSHStoreConfig(const ParsedURL::Authority & authority, const Params & params) - : StoreConfig(params) + : StoreConfig(params, FilePathType::Unix) , authority(authority) { } +void CommonSSHStoreConfig::anchor() {} + SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const { return { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 17aa41008522..24cb9207113a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -3,7 +3,6 @@ #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-connection.hh" #include "nix/store/worker-protocol-impl.hh" -#include "nix/store/build-result.hh" #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" #include "nix/store/filetransfer.hh" @@ -16,13 +15,14 @@ #include "nix/util/archive.hh" #include "nix/store/derivations.hh" #include "nix/util/args.hh" -#include "nix/util/git.hh" #include "nix/util/logging.hh" #include "nix/store/globals.hh" #include "nix/store/active-builds.hh" #include "nix/util/provenance.hh" #include "nix/util/async.hh" +#include + #ifndef _WIN32 // TODO need graceful async exit support on Windows? # include "nix/util/monitor-fd.hh" #endif @@ -46,6 +46,8 @@ Sink & operator<<(Sink & sink, const Logger::Fields & fields) return sink; } +namespace { + /* Logger that forwards log messages to the client, *if* we're in a state where the protocol allows it (i.e., when canSendStderr is true). */ @@ -186,22 +188,6 @@ struct TunnelLogger : public Logger } }; -struct TunnelSink : Sink -{ - Sink & to; - - TunnelSink(Sink & to) - : to(to) - { - } - - void operator()(std::string_view data) override - { - to << STDERR_WRITE; - writeString(data, to); - } -}; - struct TunnelSource : BufferedSource { Source & from; @@ -224,6 +210,8 @@ struct TunnelSource : BufferedSource } }; +} // namespace + struct ClientSettings { bool keepFailed; @@ -303,10 +291,9 @@ struct ClientSettings trusted || name == settings.getWorkerSettings().buildTimeout.name || name == settings.getWorkerSettings().maxSilentTime.name || name == settings.getWorkerSettings().pollInterval.name || name == "connect-timeout" - || name == loggerSettings.sessionId.name || (name == "builders" && value == "")) { + || name == loggerSettings.sessionId.name || (name == "builders" && value == "")) settings.set(name, value); - fileTransferSettings.set(name, value); - } else if (setSubstituters(settings.getWorkerSettings().substituters)) + else if (setSubstituters(settings.getWorkerSettings().substituters)) ; else warn( @@ -767,7 +754,18 @@ static void performOp( case WorkerProto::Op::CollectGarbage: { GCOptions options; options.action = WorkerProto::Serialise::read(*store, rconn); - options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); + if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { + options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); + } else { + auto paths = WorkerProto::Serialise::read(*store, rconn); + if (options.action != GCAction::gcDeleteSpecific && paths.empty()) + options.pathsToDelete = GCOptions::WholeStore{}; + else + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = paths, + .deleteReferrers = false, + }; + } conn.from >> options.ignoreLiveness >> options.maxFreed; options.censor = !trusted; // obsolete fields @@ -775,6 +773,13 @@ static void performOp( readInt(conn.from); readInt(conn.from); + if (options.action == GCAction::gcDeleteDead + && std::holds_alternative(options.pathsToDelete) + && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { + throw Error( + "Garbage collecting specific paths requested but it is not supported by the negotiated protocol"); + } + GCResults results; logger->startWork(); @@ -1025,14 +1030,8 @@ static void performOp( case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { - auto outputId = WorkerProto::Serialise::read(*store, rconn); - auto outputPath = StorePath(readString(conn.from)); - store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId}); - } else { - auto realisation = WorkerProto::Serialise::read(*store, rconn); - store->registerDrvOutput(realisation); - } + auto realisation = WorkerProto::Serialise::read(*store, rconn); + store->registerDrvOutput(realisation); logger->stopWork(); break; } @@ -1040,19 +1039,16 @@ static void performOp( case WorkerProto::Op::QueryRealisation: { logger->startWork(); auto outputId = WorkerProto::Serialise::read(*store, rconn); - auto info = store->queryRealisation(outputId); + auto ptr = store->queryRealisation(outputId); + std::optional info; + if (ptr) + info = *ptr; logger->stopWork(); - if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { - std::set outPaths; - if (info) - outPaths.insert(info->outPath); - WorkerProto::write(*store, wconn, outPaths); - } else { - std::set realisations; - if (info) - realisations.insert({*info, outputId}); - WorkerProto::write(*store, wconn, realisations); - } + /* Only return the new format because if we got past + `DrvOutput` serialization, we know that is what we're using. + */ + assert(conn.protoVersion.features.contains(WorkerProto::featureRealisationWithPath)); + WorkerProto::write(*store, wconn, info); break; } @@ -1106,11 +1102,14 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted #endif /* Exchange the greeting. */ - WorkerProto::BasicServerConnection conn; - auto version = WorkerProto::latest; + auto localVersion = WorkerProto::latest; + if (recursive) + localVersion.features.insert(std::string{WorkerProto::featureDisableSetOptions}); if (!experimentalFeatureSettings.isEnabled(Xp::Provenance)) - version.features.erase(std::string(WorkerProto::featureProvenance)); - conn.protoVersion = WorkerProto::BasicServerConnection::handshake(to, from, version); + localVersion.features.erase(std::string(WorkerProto::featureProvenance)); + + WorkerProto::BasicServerConnection conn; + conn.protoVersion = WorkerProto::BasicServerConnection::handshake(to, from, localVersion); if (conn.protoVersion.number < WorkerProto::minimum.number) throw Error("the Nix client version is too old"); diff --git a/src/libstore/darwin/build/darwin-derivation-builder.cc b/src/libstore/darwin/build/darwin-derivation-builder.cc new file mode 100644 index 000000000000..2e05f5858963 --- /dev/null +++ b/src/libstore/darwin/build/darwin-derivation-builder.cc @@ -0,0 +1,313 @@ +#include "derivation-builder-impl.hh" +#include "darwin-derivation-builder.hh" + +#include +#include +#include +#include +#include +#include +#include + +/* This definition is undocumented but depended upon by all major browsers. */ +extern "C" int +sandbox_init_with_parameters(const char * profile, uint64_t flags, const char * const parameters[], char ** errorbuf); + +/* Darwin IPC structures and constants */ +#define IPCS_MAGIC 0x00000001 +#define IPCS_SHM_ITER 0x00000002 +#define IPCS_SEM_ITER 0x00000020 +#define IPCS_MSG_ITER 0x00000200 +#define IPCS_SHM_SYSCTL "kern.sysv.ipcs.shm" +#define IPCS_MSG_SYSCTL "kern.sysv.ipcs.msg" +#define IPCS_SEM_SYSCTL "kern.sysv.ipcs.sem" + +struct IpcsCommand +{ + uint32_t ipcs_magic; + uint32_t ipcs_op; + uint32_t ipcs_cursor; + uint32_t ipcs_datalen; + void * ipcs_data; +}; + +namespace nix { + +void DarwinDerivationBuilder::prepareSandbox() +{ + pathsInChroot = getPathsInSandbox(); +} + +void DarwinDerivationBuilder::setUser() +{ + DerivationBuilderImpl::setUser(); + + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; + + if (useSandbox) { + + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + StringSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : pathsInChroot) { + std::filesystem::path cur = i.first; + while (cur != "/") { + cur = cur.parent_path(); + ancestry.insert(cur.native()); + } + } + + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + std::filesystem::path cur = store.storeDir; + while (cur != "/") { + ancestry.insert(cur.native()); + cur = cur.parent_path(); + } + + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) { + auto p = store.printStorePath(i); + pathsInChroot.insert_or_assign(p, ChrootPath{.source = p}); + } + + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be + * configurable */ + if (store.config->getLocalSettings().darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; + } + + sandboxProfile += +#include "sandbox-defaults.sb" + ; + + if (!derivationType.isSandboxed()) + sandboxProfile += +#include "sandbox-network.sb" + ; + + /* Add the output paths we'll use at build-time to the chroot */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & [_, path] : scratchOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", store.printStorePath(path)); + + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed above) + without file-write* allowed, access() incorrectly returns EPERM */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + + // We create multiple allow lists, to avoid exceeding a limit in the darwin sandbox interpreter. + // See https://github.com/NixOS/nix/issues/4119 + // We split our allow groups approximately at half the actual limit, 1 << 16 + const size_t breakpoint = sandboxProfile.length() + (1 << 14); + for (auto & i : pathsInChroot) { + + if (sandboxProfile.length() >= breakpoint) { + debug("Sandbox break: %d %d", sandboxProfile.length(), breakpoint); + sandboxProfile += ")\n(allow file-read* file-write* process-exec\n"; + } + + if (i.first != i.second.source) + throw Error( + "can't map %1% to %2%: mismatched impure paths not supported on Darwin", + PathFmt(i.first), + PathFmt(i.second.source)); + + std::string path = i.first; + auto optSt = maybeLstat(path.c_str()); + if (!optSt) { + if (i.second.optional) + continue; + throw SysError("getting attributes of required path '%s", path); + } + if (S_ISDIR(optSt->st_mode)) + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); + else + sandboxProfile += fmt("\t(literal \"%s\")\n", path); + } + sandboxProfile += ")\n"; + + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto & i : ancestry) { + sandboxProfile += fmt("\t(literal \"%s\")\n", i); + } + sandboxProfile += ")\n"; + + sandboxProfile += drvOptions.additionalSandboxProfile; + } else + sandboxProfile += +#include "sandbox-minimal.sb" + ; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different + mechanisms to find temporary directories, so we want to open up a broader place for them to put their files, + if needed. */ + std::filesystem::path globalTmpDir = canonPath(defaultTempDir().native(), true); + + /* They don't like trailing slashes on subpath directives */ + std::string globalTmpDirStr = globalTmpDir.native(); + while (!globalTmpDirStr.empty() && globalTmpDirStr.back() == '/') + globalTmpDirStr.pop_back(); + + if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { + Strings sandboxArgs; + sandboxArgs.push_back("_NIX_BUILD_TOP"); + sandboxArgs.push_back(tmpDir.native()); + sandboxArgs.push_back("_GLOBAL_TMP_DIR"); + sandboxArgs.push_back(globalTmpDirStr); + if (drvOptions.allowLocalNetworking) { + sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); + sandboxArgs.push_back("1"); + } + char * sandbox_errbuf = nullptr; + if (sandbox_init_with_parameters( + sandboxProfile.c_str(), 0, stringsToCharPtrs(sandboxArgs).data(), &sandbox_errbuf)) { + writeFull( + STDERR_FILENO, fmt("failed to configure sandbox: %s\n", sandbox_errbuf ? sandbox_errbuf : "(null)")); + _exit(1); + } + } +} + +void DarwinDerivationBuilder::execBuilder(const Strings & args, const Strings & envStrs) +{ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv.platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv.platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn( + NULL, drv.builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +} + +void DarwinDerivationBuilder::cleanupSysVIPCForUser(uid_t uid) +{ + struct IpcsCommand ic; + size_t ic_size = sizeof(ic); + // IPC ids to cleanup + std::vector shm_ids, msg_ids, sem_ids; + + { + struct shmid_ds shm_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_SHM_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &shm_ds; + ic.ipcs_datalen = sizeof(shm_ds); + + while (true) { + memset(&shm_ds, 0, sizeof(shm_ds)); + + if (sysctlbyname(IPCS_SHM_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (shm_ds.shm_perm.uid == uid) { + int shmid = shmget(shm_ds.shm_perm._key, 0, 0); + if (shmid != -1) { + shm_ids.push_back(shmid); + } + } + } + } + + for (auto id : shm_ids) { + if (shmctl(id, IPC_RMID, NULL) == 0) + debug("removed shared memory segment with shmid %d", id); + } + + { + struct msqid_ds msg_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_MSG_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &msg_ds; + ic.ipcs_datalen = sizeof(msg_ds); + + while (true) { + memset(&msg_ds, 0, sizeof(msg_ds)); + + if (sysctlbyname(IPCS_MSG_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (msg_ds.msg_perm.uid == uid) { + int msgid = msgget(msg_ds.msg_perm._key, 0); + if (msgid != -1) { + msg_ids.push_back(msgid); + } + } + } + } + + for (auto id : msg_ids) { + if (msgctl(id, IPC_RMID, NULL) == 0) + debug("removed message queue with msgid %d", id); + } + + { + struct semid_ds sem_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_SEM_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &sem_ds; + ic.ipcs_datalen = sizeof(sem_ds); + + while (true) { + memset(&sem_ds, 0, sizeof(sem_ds)); + + if (sysctlbyname(IPCS_SEM_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (sem_ds.sem_perm.uid == uid) { + int semid = semget(sem_ds.sem_perm._key, 0, 0); + if (semid != -1) { + sem_ids.push_back(semid); + } + } + } + } + + for (auto id : sem_ids) { + if (semctl(id, 0, IPC_RMID) == 0) + debug("removed semaphore with semid %d", id); + } +} + +void DarwinDerivationBuilder::killSandbox(bool getStats) +{ + DerivationBuilderImpl::killSandbox(getStats); + if (buildUser) { + auto uid = buildUser->getUID(); + cleanupSysVIPCForUser(uid); + } +} + +} // namespace nix diff --git a/src/libstore/darwin/build/darwin-derivation-builder.hh b/src/libstore/darwin/build/darwin-derivation-builder.hh new file mode 100644 index 000000000000..260339c5c577 --- /dev/null +++ b/src/libstore/darwin/build/darwin-derivation-builder.hh @@ -0,0 +1,47 @@ +#pragma once + +#include "derivation-builder-impl.hh" + +namespace nix { + +struct DarwinDerivationBuilder : DerivationBuilderImpl +{ + PathsInChroot pathsInChroot; + + /** + * Whether full sandboxing is enabled. Note that macOS builds + * always have *some* sandboxing (see sandbox-minimal.sb). + */ + bool useSandbox; + + DarwinDerivationBuilder( + LocalStore & store, + std::shared_ptr miscMethods, + DerivationBuilderParams params, + bool useSandbox) + : DerivationBuilderImpl(store, miscMethods, std::move(params)) + , useSandbox(useSandbox) + { + } + + void prepareSandbox() override; + + void setUser() override; + + void execBuilder(const Strings & args, const Strings & envStrs) override; + + /** + * Cleans up all System V IPC objects owned by the specified user. + * + * On Darwin, IPC objects (shared memory segments, message queues, and semaphore) + * can persist after the build user's processes are killed, since there are no IPC namespaces + * like on Linux. This can exhaust kernel IPC limits over time. + * + * Uses sysctl to enumerate and remove all IPC objects owned by the given UID. + */ + void cleanupSysVIPCForUser(uid_t uid); + + void killSandbox(bool getStats) override; +}; + +} // namespace nix diff --git a/src/libstore/unix/build/sandbox-defaults.sb b/src/libstore/darwin/build/sandbox-defaults.sb similarity index 100% rename from src/libstore/unix/build/sandbox-defaults.sb rename to src/libstore/darwin/build/sandbox-defaults.sb diff --git a/src/libstore/unix/build/sandbox-minimal.sb b/src/libstore/darwin/build/sandbox-minimal.sb similarity index 100% rename from src/libstore/unix/build/sandbox-minimal.sb rename to src/libstore/darwin/build/sandbox-minimal.sb diff --git a/src/libstore/unix/build/sandbox-network.sb b/src/libstore/darwin/build/sandbox-network.sb similarity index 100% rename from src/libstore/unix/build/sandbox-network.sb rename to src/libstore/darwin/build/sandbox-network.sb diff --git a/src/libstore/darwin/meson.build b/src/libstore/darwin/meson.build new file mode 100644 index 000000000000..72d94baac2be --- /dev/null +++ b/src/libstore/darwin/meson.build @@ -0,0 +1,3 @@ +include_dirs += [ include_directories('build') ] + +sources += files('build/darwin-derivation-builder.cc') diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 5208440c7d94..b3d4261b5464 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -6,7 +6,6 @@ #include "nix/store/store-api.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" -#include "nix/util/variant-wrapper.hh" #include #include @@ -73,7 +72,8 @@ template using OutputChecks = DerivationOptions::OutputChecks; template -using OutputChecksVariant = std::variant, std::map>>; +using OutputChecksVariant = + std::variant, std::map, std::less<>>>; DerivationOptions derivationOptionsFromStructuredAttrs( const StoreDirConfig & store, @@ -124,7 +124,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( { DerivationOptions defaults = {}; - std::map placeholders; + std::map> placeholders; if (mockXpSettings.isEnabled(Xp::CaDerivations)) { /* Initialize placeholder map from inputDrvs */ auto initPlaceholders = [&](this const auto & initPlaceholders, @@ -156,16 +156,27 @@ DerivationOptions derivationOptionsFromStructuredAttrs( } } + /* Extract the placeholder key from a path that may have a subpath + appended (e.g. `/HASH/foo` → `/HASH`), mirroring how + `StoreDirConfig::toStorePath` strips subpaths from store paths. */ + auto findPlaceholder = [&](std::string_view pathS) -> const SingleDerivedPath::Built * { + auto slash = pathS.find('/', 1); + auto key = pathS.substr(0, slash); + if (auto it = placeholders.find(key); it != placeholders.end()) + return &it->second; + return nullptr; + }; + auto parseSingleDerivedPath = [&](const std::string & pathS) -> SingleDerivedPath { - if (auto it = placeholders.find(pathS); it != placeholders.end()) - return it->second; + if (auto * built = findPlaceholder(pathS)) + return *built; else return SingleDerivedPath::Opaque{store.toStorePath(pathS).first}; }; auto parseRef = [&](const std::string & pathS) -> DrvRef { - if (auto it = placeholders.find(pathS); it != placeholders.end()) - return it->second; + if (auto * built = findPlaceholder(pathS)) + return *built; if (store.isStorePath(pathS)) return SingleDerivedPath::Opaque{store.toStorePath(pathS).first}; else @@ -206,7 +217,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( if (parsed) { auto & structuredAttrs = parsed->structuredAttrs; - std::map> res; + std::map, std::less<>> res; if (auto * outputChecks = get(structuredAttrs, "outputChecks")) { for (auto & [outputName, output_] : getObject(*outputChecks)) { auto & output = getObject(output_); @@ -265,7 +276,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( }(), .unsafeDiscardReferences = [&] { - std::map res; + std::map> res; if (parsed) { if (auto * udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) { @@ -298,7 +309,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( }(), .exportReferencesGraph = [&] { - std::map> ret; + std::map, std::less<>> ret; if (parsed) { auto * e = get(parsed->structuredAttrs, "exportReferencesGraph"); @@ -444,9 +455,10 @@ std::optional> tryResolve( }; // Helper function to resolve exportReferencesGraph using functional style - auto tryResolveExportReferencesGraph = [&](const std::map> & exportGraph) - -> std::optional>> { - std::map> resolved; + auto tryResolveExportReferencesGraph = + [&](const std::map, std::less<>> & exportGraph) + -> std::optional, std::less<>>> { + std::map, std::less<>> resolved; for (const auto & [name, inputPaths] : exportGraph) { std::set resolvedPaths; for (const auto & inputPath : inputPaths) { @@ -466,19 +478,20 @@ std::optional> tryResolve( [&](const DerivationOptions::OutputChecks & checks) -> std::optional::OutputChecks, - std::map::OutputChecks>>> { + std::map::OutputChecks, std::less<>>>> { auto resolved = tryResolveOutputChecks(checks); if (!resolved) return std::nullopt; return std::variant< DerivationOptions::OutputChecks, - std::map::OutputChecks>>(*resolved); + std::map::OutputChecks, std::less<>>>(*resolved); }, - [&](const std::map::OutputChecks> & checksMap) + [&](const std::map::OutputChecks, std::less<>> & + checksMap) -> std::optional::OutputChecks, - std::map::OutputChecks>>> { - std::map::OutputChecks> resolvedMap; + std::map::OutputChecks, std::less<>>>> { + std::map::OutputChecks, std::less<>> resolvedMap; for (const auto & [outputName, checks] : checksMap) { auto resolved = tryResolveOutputChecks(checks); if (!resolved) @@ -487,7 +500,7 @@ std::optional> tryResolve( } return std::variant< DerivationOptions::OutputChecks, - std::map::OutputChecks>>(resolvedMap); + std::map::OutputChecks, std::less<>>>(resolvedMap); }}, drvOptions.outputChecks); @@ -525,21 +538,22 @@ namespace nlohmann { using namespace nix; -DerivationOptions adl_serializer>::from_json(const json & json_) +template +static DerivationOptions derivationOptionsFromJson(const nlohmann::json & json_) { auto & json = getObject(json_); return { - .outputChecks = [&]() -> OutputChecksVariant { + .outputChecks = [&]() -> OutputChecksVariant { auto outputChecks = getObject(valueAt(json, "outputChecks")); auto forAllOutputsOpt = get(outputChecks, "forAllOutputs"); auto perOutputOpt = get(outputChecks, "perOutput"); if (forAllOutputsOpt && !perOutputOpt) { - return static_cast>(*forAllOutputsOpt); + return static_cast>(*forAllOutputsOpt); } else if (perOutputOpt && !forAllOutputsOpt) { - return static_cast>>(*perOutputOpt); + return static_cast, std::less<>>>(*perOutputOpt); } else { throw Error("Exactly one of 'perOutput' or 'forAllOutputs' is required"); } @@ -561,17 +575,17 @@ DerivationOptions adl_serializer>::to_json( - json & json, const DerivationOptions & o) +template +static void derivationOptionsToJson(nlohmann::json & json, const DerivationOptions & o) { json["outputChecks"] = std::visit( overloaded{ - [&](const OutputChecks & checks) { + [&](const OutputChecks & checks) { nlohmann::json outputChecks; outputChecks["forAllOutputs"] = checks; return outputChecks; }, - [&](const std::map> & checksPerOutput) { + [&](const std::map, std::less<>> & checksPerOutput) { nlohmann::json outputChecks; outputChecks["perOutput"] = checksPerOutput; return outputChecks; @@ -594,7 +608,8 @@ void adl_serializer>::to_json( json["allowSubstitutes"] = o.allowSubstitutes; } -OutputChecks adl_serializer>::from_json(const json & json_) +template +static OutputChecks outputChecksFromJson(const nlohmann::json & json_) { auto & json = getObject(json_); @@ -602,16 +617,15 @@ OutputChecks adl_serializer>: .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")), .maxSize = ptrToOwned(getNullable(valueAt(json, "maxSize"))), .maxClosureSize = ptrToOwned(getNullable(valueAt(json, "maxClosureSize"))), - .allowedReferences = - ptrToOwned>>(getNullable(valueAt(json, "allowedReferences"))), + .allowedReferences = ptrToOwned>>(getNullable(valueAt(json, "allowedReferences"))), .disallowedReferences = valueAt(json, "disallowedReferences"), - .allowedRequisites = - ptrToOwned>>(getNullable(valueAt(json, "allowedRequisites"))), + .allowedRequisites = ptrToOwned>>(getNullable(valueAt(json, "allowedRequisites"))), .disallowedRequisites = valueAt(json, "disallowedRequisites"), }; } -void adl_serializer>::to_json(json & json, const OutputChecks & c) +template +static void outputChecksToJson(nlohmann::json & json, const OutputChecks & c) { json["ignoreSelfRefs"] = c.ignoreSelfRefs; json["maxSize"] = c.maxSize; @@ -622,4 +636,45 @@ void adl_serializer>::to_json(json & json, const json["disallowedRequisites"] = c.disallowedRequisites; } +DerivationOptions adl_serializer>::from_json(const json & json_) +{ + return derivationOptionsFromJson(json_); +} + +void adl_serializer>::to_json( + json & json, const DerivationOptions & o) +{ + derivationOptionsToJson(json, o); +} + +DerivationOptions adl_serializer>::from_json(const json & json_) +{ + return derivationOptionsFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const DerivationOptions & o) +{ + derivationOptionsToJson(json, o); +} + +OutputChecks adl_serializer>::from_json(const json & json_) +{ + return outputChecksFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const OutputChecks & c) +{ + outputChecksToJson(json, c); +} + +OutputChecks adl_serializer>::from_json(const json & json_) +{ + return outputChecksFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const OutputChecks & c) +{ + outputChecksToJson(json, c); +} + } // namespace nlohmann diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index bfea6e992c19..b4f1f5a27fa2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1,10 +1,8 @@ #include "nix/store/derivations.hh" #include "nix/store/downstream-placeholder.hh" #include "nix/store/store-api.hh" -#include "nix/store/globals.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" -#include "nix/util/split.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" #include "nix/util/strings-inline.hh" @@ -20,6 +18,10 @@ namespace nix { using namespace std::literals::string_view_literals; +BasicDerivation::~BasicDerivation() {} + +Derivation::~Derivation() {} + std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { @@ -768,7 +770,7 @@ std::string Derivation::unparse( s += ",["sv; first = true; - auto unparseEnv = [&](const StringPairs atermEnv) { + auto unparseEnv = [&](const StringPairs & atermEnv) { for (auto & i : atermEnv) { if (first) first = false; @@ -881,13 +883,14 @@ DrvHashes drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) +static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) { - std::optional hash; + std::optional hash; if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) { return *hash; } auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false); + // Cache it drvHashes.insert_or_assign(drvPath, h); return h; @@ -910,12 +913,10 @@ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPa don't leak the provenance of fixed outputs, reducing pointless cache misses as the build itself won't know this. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { - auto type = drv.type(); - /* Return a fixed hash for fixed-output derivations. */ - if (type.isFixed()) { + if (drv.type().isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw); @@ -925,54 +926,66 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } - return DrvHash{ - .hashes = outputHashes, - .kind = DrvHash::Kind::Regular, - }; + return outputHashes; } - auto kind = std::visit( - overloaded{ - [](const DerivationType::InputAddressed & ia) { - /* This might be a "pesimistically" deferred output, so we don't - "taint" the kind yet. */ - return DrvHash::Kind::Regular; - }, - [](const DerivationType::ContentAddressed & ca) { - return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; - }, - [](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }}, - drv.type().raw); + if (std::visit( + overloaded{ + [](const DerivationType::InputAddressed & ia) { + /* This might be a "pesimistically" deferred output, so we don't + "taint" the kind yet. */ + return false; + }, + [](const DerivationType::ContentAddressed & ca) { + // Already covered + assert(!ca.fixed); + return true; + }, + [](const DerivationType::Impure &) { return true; }}, + drv.type().raw)) { + return DrvHashModulo::DeferredDrv{}; + } + /* For other derivations, replace the inputs paths with recursive + calls to this function. */ DerivedPathMap::ChildNode::Map inputs2; for (auto & [drvPath, node] : drv.inputDrvs.map) { - const auto & res = pathDerivationModulo(store, drvPath); - if (res.kind == DrvHash::Kind::Deferred) - kind = DrvHash::Kind::Deferred; - for (auto & outputName : node.value) { - const auto h = get(res.hashes, outputName); - if (!h) - throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); + /* Need to build and resolve dynamic derivations first */ + if (!node.childMap.empty()) { + return DrvHashModulo::DeferredDrv{}; } - } - - auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); - std::map outputHashes; - for (const auto & [outputName, _] : drv.outputs) { - outputHashes.insert_or_assign(outputName, hash); + const auto & res = pathDerivationModulo(store, drvPath); + if (std::visit( + overloaded{ + [&](const DrvHashModulo::DeferredDrv &) { return true; }, + // Regular non-CA derivation, replace derivation + [&](const DrvHashModulo::DrvHash & drvHash) { + inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node); + return false; + }, + // CA derivation's output hashes + [&](const DrvHashModulo::CaOutputHashes & outputHashes) { + for (auto & outputName : node.value) { + /* Put each one in with a single "out" output.. */ + const auto h = get(outputHashes, outputName); + if (!h) + throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); + inputs2.insert_or_assign( + h->to_string(HashFormat::Base16, false), + DerivedPathMap::ChildNode{ + .value = {"out"}, + }); + } + return false; + }, + }, + res.raw)) { + return DrvHashModulo::DeferredDrv{}; + } } - return DrvHash{ - .hashes = outputHashes, - .kind = kind, - }; -} - -std::map staticOutputHashes(Store & store, const Derivation & drv) -{ - return hashDerivationModulo(store, drv, true).hashes; + return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); } static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) @@ -1122,26 +1135,6 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites) } } -static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) -{ - drv.applyRewrites(rewrites); - - auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); - for (auto & [outputName, output] : drv.outputs) { - if (std::holds_alternative(output.raw)) { - auto h = get(hashModulo.hashes, outputName); - if (!h) - throw Error( - "derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", drv.name, outputName); - auto outPath = store.makeOutputPath(outputName, *h, drv.name); - drv.env[outputName] = store.printStorePath(outPath); - output = DerivationOutput::InputAddressed{ - .path = std::move(outPath), - }; - } - } -} - bool Derivation::shouldResolve() const { /* No input drvs means nothing to resolve. */ @@ -1188,7 +1181,7 @@ std::optional Derivation::tryResolve(Store & store, Store * eva } static bool tryResolveInput( - Store & store, + const StoreDirConfig & store, StorePathSet & inputSrcs, StringMap & inputRewrites, const DownstreamPlaceholder * placeholderOpt, @@ -1253,9 +1246,13 @@ std::optional Derivation::tryResolve( queryResolutionChain)) return std::nullopt; - rewriteDerivation(store, resolved, inputRewrites); + resolved.applyRewrites(inputRewrites); + + Derivation resolved2{std::move(resolved)}; + + resolved2.fillInOutputPaths(store); - return resolved; + return resolved2; } /** @@ -1282,7 +1279,15 @@ std::optional Derivation::tryResolve( template static void processDerivationOutputPaths(Store & store, auto && drv, std::string_view drvName) { - std::optional hashesModulo; + std::optional hashModulo_; + + auto hashModulo = [&]() -> const auto & { + if (!hashModulo_) { + // somewhat expensive so we do lazily + hashModulo_ = hashDerivationModulo(store, drv, true); + } + return *hashModulo_; + }; for (auto & [outputName, output] : drv.outputs) { auto envHasRightPath = [&](const StorePath & actual, bool isDeferred = false) { @@ -1319,65 +1324,64 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string } }; auto hash = [&](const Output & outputVariant) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(store, drv, true); - } - switch (hashesModulo->kind) { - case DrvHash::Kind::Regular: { - auto h = get(hashesModulo->hashes, outputName); - if (!h) - throw Error("derivation produced no hash for output '%s'", outputName); - auto outPath = store.makeOutputPath(outputName, *h, drvName); - - if constexpr (std::is_same_v) { - if (outputVariant.path == outPath) { - return; // Correct case - } - /* Error case, an explicitly wrong path is - always an error. */ - throw Error( - "derivation has incorrect output '%s', should be '%s'", - store.printStorePath(outputVariant.path), - store.printStorePath(outPath)); - } else if constexpr (std::is_same_v) { - if constexpr (fillIn) - /* Fill in output path for Deferred - outputs */ - output = DerivationOutput::InputAddressed{ - .path = outPath, - }; - else - /* Validation mode: deferred outputs - should have been filled in */ - warn( - "derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.", - store.printStorePath(outPath)); - } else { - /* Will never happen, based on where - `hash` is called. */ - static_assert(false); - } - envHasRightPath(outPath); - break; - } - case DrvHash::Kind::Deferred: - if constexpr (std::is_same_v) { - /* Error case, an explicitly wrong path is - always an error. */ - throw Error( - "derivation has incorrect output '%s', should be deferred", - store.printStorePath(outputVariant.path)); - } else if constexpr (std::is_same_v) { - /* Correct: Deferred output with Deferred - hash kind. */ - } else { - /* Will never happen, based on where - `hash` is called. */ - static_assert(false); - } - break; - } + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + auto outPath = store.makeOutputPath(outputName, drvHash, drvName); + + if constexpr (std::is_same_v) { + if (outputVariant.path == outPath) { + envHasRightPath(outPath); + return; // Correct case + } + /* Error case, an explicitly wrong path is + always an error. */ + throw Error( + "derivation has incorrect output '%s', should be '%s'", + store.printStorePath(outputVariant.path), + store.printStorePath(outPath)); + } else if constexpr (std::is_same_v) { + if constexpr (fillIn) { + /* Fill in output path for Deferred outputs */ + output = DerivationOutput::InputAddressed{ + .path = outPath, + }; + envHasRightPath(outPath); + } else { + /* Validation mode: deferred outputs + should have been filled in */ + warn( + "derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.", + store.printStorePath(outPath)); + } + } else { + /* Will never happen, based on where + `hash` is called. */ + static_assert(false); + } + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + input-addressed (or deferred waiting to be). */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + if constexpr (std::is_same_v) { + /* Error case, an explicitly wrong path is + always an error. */ + throw Error( + "derivation has incorrect output '%s', should be deferred", + store.printStorePath(outputVariant.path)); + } else if constexpr (std::is_same_v) { + /* Correct: Deferred output with Deferred hash kind. */ + } else { + /* Will never happen, based on where + `hash` is called. */ + static_assert(false); + } + }, + }, + hashModulo().raw); }; std::visit( overloaded{ @@ -1544,32 +1548,25 @@ adl_serializer::from_json(const json & _json, const Experiment } } -void adl_serializer::to_json(json & res, const BasicDerivation & d) +static void inputSrcsToJson(json & res, const StorePathSet & inputSrcs) +{ + res = nlohmann::json::array(); + for (auto & input : inputSrcs) + res.emplace_back(input); +} + +static void basicDerivationToJson(json & res, const BasicDerivation & d) { res = nlohmann::json::object(); res["name"] = d.name; - res["version"] = expectedJsonVersionDerivation; { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); - for (auto & [outputName, output] : d.outputs) { + for (auto & [outputName, output] : d.outputs) outputsObj[outputName] = output; - } - } - - { - auto & inputsObj = res["inputs"]; - inputsObj = nlohmann::json::object(); - - { - auto & inputsList = inputsObj["srcs"]; - inputsList = nlohmann::json::array(); - for (auto & input : d.inputSrcs) - inputsList.emplace_back(input); - } } res["system"] = d.platform; @@ -1581,36 +1578,52 @@ void adl_serializer::to_json(json & res, const BasicDerivation res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; } -void adl_serializer::to_json(json & res, const Derivation & d) +void adl_serializer::to_json(json & res, const BasicDerivation & d) { - adl_serializer::to_json(res, static_cast(d)); - - auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { - auto value = nlohmann::json::object(); - value["outputs"] = inputNode.value; - { - auto next = nlohmann::json::object(); - for (auto & [outputId, childNode] : inputNode.childMap) - next[outputId] = doInput(childNode); - value["dynamicOutputs"] = std::move(next); - } - return value; - }; + basicDerivationToJson(res, d); - auto & inputDrvsObj = res["inputs"]["drvs"]; - inputDrvsObj = nlohmann::json::object(); - for (auto & [inputDrv, inputNode] : d.inputDrvs.map) - inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); + inputSrcsToJson(res["inputs"], d.inputSrcs); } -Derivation adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +void adl_serializer::to_json(json & res, const Derivation & d) { - using nlohmann::detail::value_t; + basicDerivationToJson(res, d); - Derivation res; + { + auto & inputsObj = res["inputs"]; + inputsObj = nlohmann::json::object(); - auto & json = getObject(_json); + inputSrcsToJson(inputsObj["srcs"], d.inputSrcs); + + auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { + auto value = nlohmann::json::object(); + value["outputs"] = inputNode.value; + { + auto next = nlohmann::json::object(); + for (auto & [outputId, childNode] : inputNode.childMap) + next[outputId] = doInput(childNode); + value["dynamicOutputs"] = std::move(next); + } + return value; + }; + + auto & inputDrvsObj = inputsObj["drvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : d.inputDrvs.map) + inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); + } +} + +static void inputSrcsFromJson(const json & inputSrcsJson, StorePathSet & inputSrcs) +{ + auto arr = getArray(inputSrcsJson); + for (auto & input : arr) + inputSrcs.insert(input); +} +static void basicDerivationFromJson( + const json::object_t & json, BasicDerivation & res, const ExperimentalFeatureSettings & xpSettings) +{ res.name = getString(valueAt(json, "name")); { @@ -1632,13 +1645,50 @@ Derivation adl_serializer::from_json(const json & _json, const Exper throw; } + res.platform = getString(valueAt(json, "system")); + res.builder = getString(valueAt(json, "builder")); + res.args = getStringList(valueAt(json, "args")); + + auto envJson = valueAt(json, "env"); + try { + res.env = getStringMap(envJson); + } catch (Error & e) { + e.addTrace({}, "while reading key 'env'"); + throw; + } + + if (auto structuredAttrs = get(json, "structuredAttrs")) + res.structuredAttrs = StructuredAttrs{*structuredAttrs}; +} + +BasicDerivation +adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +{ + BasicDerivation res; + auto & json = getObject(_json); + basicDerivationFromJson(json, res, xpSettings); + + try { + inputSrcsFromJson(valueAt(json, "inputs"), res.inputSrcs); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputs'"); + throw; + } + + return res; +} + +Derivation adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +{ + Derivation res; + auto & json = getObject(_json); + basicDerivationFromJson(json, res, xpSettings); + try { auto inputsObj = getObject(valueAt(json, "inputs")); try { - auto inputSrcs = getArray(valueAt(inputsObj, "srcs")); - for (auto & input : inputSrcs) - res.inputSrcs.insert(input); + inputSrcsFromJson(valueAt(inputsObj, "srcs"), res.inputSrcs); } catch (Error & e) { e.addTrace({}, "while reading key 'srcs'"); throw; @@ -1669,21 +1719,6 @@ Derivation adl_serializer::from_json(const json & _json, const Exper throw; } - res.platform = getString(valueAt(json, "system")); - res.builder = getString(valueAt(json, "builder")); - res.args = getStringList(valueAt(json, "args")); - - auto envJson = valueAt(json, "env"); - try { - res.env = getStringMap(envJson); - } catch (Error & e) { - e.addTrace({}, "while reading key 'env'"); - throw; - } - - if (auto structuredAttrs = get(json, "structuredAttrs")) - res.structuredAttrs = StructuredAttrs{*structuredAttrs}; - return res; } diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 3d799ab1c583..684c4462fbf8 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -48,6 +48,34 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single return initIter(k); } +template +void DerivedPathMap::removeSlot(const SingleDerivedPath & k, fun callback) +{ + auto removeIter = [&map = + map](this auto & self, const SingleDerivedPath & k, fun onNode) -> void { + std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) { + if (auto it = map.find(bo.path); it != map.end() && !onNode(it->second)) + map.erase(it); + }, + [&](const SingleDerivedPath::Built & bfd) { + self(*bfd.drvPath, [&](ChildNode & parent) -> bool { + auto it = parent.childMap.find(bfd.output); + if (it == parent.childMap.end()) + return !parent.value.empty() || !parent.childMap.empty(); + if (!onNode(it->second)) + parent.childMap.erase(it); + return !parent.value.empty() || !parent.childMap.empty(); + }); + }, + }, + k.raw()); + }; + + removeIter(k, [&](ChildNode & node) -> bool { return callback(node) || !node.childMap.empty(); }); +} + } // namespace nix // instantiations diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 251e112514e1..131674aa5595 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,11 +1,9 @@ #include "nix/store/derived-path.hh" #include "nix/store/derivations.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-dir-config.hh" #include "nix/util/comparator.hh" #include "nix/util/json-utils.hh" -#include - namespace nix { // Custom implementation to avoid `ref` ptr equality diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index dde799ba528c..3c4de14edecd 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -10,6 +10,10 @@ namespace nix { +void DummyStoreConfig::anchor() {} + +void DummyStore::anchor() {} + std::string DummyStoreConfig::doc() { return @@ -31,6 +35,8 @@ namespace { class WholeStoreViewAccessor : public SourceAccessor { + void anchor() override {}; + using BaseName = std::string; /** @@ -63,6 +69,8 @@ class WholeStoreViewAccessor : public SourceAccessor }); if (!res) + /* The accessor is truly empty, i.e. without any file at root so + any subsequent operation with it will fail. */ res = &emptyAccessor; return callback(*res, path); @@ -101,6 +109,7 @@ class WholeStoreViewAccessor : public SourceAccessor DirEntries readDirectory(const CanonPath & path) override { + /* FIXME: Special-case the root directory to read the whole store, not just an empty root. */ return callWithAccessorForPath( path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.readDirectory(path); }); } @@ -126,6 +135,10 @@ bool DummyStoreConfig::getReadOnly() const struct DummyStoreImpl : DummyStore { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; /** @@ -336,7 +349,7 @@ struct DummyStoreImpl : DummyStore void registerDrvOutput(const Realisation & output) override { - buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, output}}}, [&](auto & kv) { + buildTrace.insert_or_visit({output.id.drvPath, {{output.id.outputName, output}}}, [&](auto & kv) { kv.second.insert_or_assign(output.id.outputName, output); }); } @@ -345,7 +358,7 @@ struct DummyStoreImpl : DummyStore const DrvOutput & drvOutput, Callback> callback) noexcept override { bool visited = false; - buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) { + buildTrace.cvisit(drvOutput.drvPath, [&](const auto & kv) { if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) { visited = true; callback(std::make_shared(it->second)); @@ -384,6 +397,8 @@ struct DummyStoreImpl : DummyStore } }; +void DummyStoreImpl::anchor() {} + ref DummyStore::Config::openDummyStore() const { return make_ref(ref{shared_from_this()}); @@ -442,11 +457,7 @@ ref adl_serializer>::from_json(const json & json) for (auto & [k1, v2] : getObject(v)) { UnkeyedRealisation realisation = v2; res->buildTrace.insert_or_visit( - { - Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), - {{k1, realisation}}, - }, - [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); }); + {StorePath{k0}, {{k1, realisation}}}, [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); }); } } return res; @@ -479,7 +490,7 @@ void adl_serializer::to_json(json & json, const DummyStore & val) auto obj = json::object(); val.buildTrace.cvisit_all([&](const auto & kv) { auto & [k, v] = kv; - auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); + auto & obj2 = obj[k.to_string()] = json::object(); for (auto & [k2, v2] : kv.second) obj2[k2] = v2; }); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 599600b9e8ce..80f6a6be0735 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,12 +1,12 @@ #include "nix/store/filetransfer.hh" +#include "nix/store/filetransfer-impl.hh" #include "nix/store/globals.hh" #include "nix/util/config-global.hh" -#include "nix/store/store-api.hh" #include "nix/util/finally.hh" #include "nix/util/callback.hh" #include "nix/util/signals.hh" +#include "nix/util/util.hh" -#include "store-config-private.hh" #include "nix/store/s3-url.hh" #include #if NIX_WITH_AWS_AUTH @@ -23,38 +23,102 @@ #include #include -#include +#include +#include #include #include #include #include +#include #include namespace nix { -const unsigned int RETRY_TIME_MS_DEFAULT = 250; -const unsigned int RETRY_TIME_MS_TOO_MANY_REQUESTS = 60000; +namespace { + +enum struct HttpStatus : long { + Ok = 200, + Created = 201, + NoContent = 204, + PartialContent = 206, + NotModified = 304, + Unauthorized = 401, + Forbidden = 403, + NotFound = 404, + ProxyAuthRequired = 407, + RequestTimeout = 408, + Gone = 410, + TooManyRequests = 429, + NotImplemented = 501, + ServiceUnavailable = 503, + HttpVersionNotSupported = 505, + NetworkAuthRequired = 511, +}; + +constexpr bool operator==(long lhs, HttpStatus rhs) noexcept +{ + return lhs == static_cast(rhs); +} -std::filesystem::path FileTransferSettings::getDefaultSSLCertFile() +} // namespace + +std::chrono::milliseconds computeRetryDelayMs(const RetryDelayParams & p, std::mt19937 & rng) { + uint32_t backoff = clampedExponential(p.baseMs, p.attempt, p.ceilMs); + + // Retry-After is a hard minimum — the server explicitly asked us to wait + // at least this long. ceilMs caps the backoff algorithm, not the server's + // signal: retrying before the server is ready just burns an attempt. + uint32_t floor = p.retryAfterMs.value_or(0); + + if (!p.jitter) + return std::chrono::milliseconds(std::max(floor, backoff)); + + // Jitter spreads retries over [floor, floor+backoff] so that concurrent + // clients receiving the same Retry-After don't all retry simultaneously. + // Saturating add: clamp if floor + backoff would overflow uint32_t. + constexpr auto u32max = std::numeric_limits::max(); + uint32_t ceiling = (backoff > u32max - floor) ? u32max : floor + backoff; + if (ceiling <= floor) + return std::chrono::milliseconds(floor); + + return std::chrono::milliseconds(std::uniform_int_distribution(floor, ceiling)(rng)); +} + +std::optional FileTransferSettings::getDefaultSSLCertFile() +{ + /* Windows has no notion of a default location for the certificate bundles. + Instead we use CURLSSLOPT_NATIVE_CA by default. */ +#ifndef _WIN32 for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) if (pathAccessible(fn)) return fn; - return ""; +#endif + return std::nullopt; } +void FileTransferSettings::anchor() {} + FileTransferSettings::FileTransferSettings() { - auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); - if (sslOverride != "") - caFile = sslOverride; + std::optional sslOverride = + getEnvOs(OS_STR("NIX_SSL_CERT_FILE")) + .or_else([] { return getEnvOs(OS_STR("SSL_CERT_FILE")); }) + .and_then([](OsString s) -> std::optional { + return s.empty() ? std::nullopt : std::optional{std::move(s)}; + }) + .transform([](OsString s) { return AbsolutePath{std::filesystem::path{std::move(s)}}; }); + if (sslOverride) + caFile = *sslOverride; } FileTransferSettings fileTransferSettings; static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); +FileTransfer::~FileTransfer() {} + namespace { using curlSList = std::unique_ptr<::curl_slist, decltype([](::curl_slist * list) { ::curl_slist_free_all(list); })>; @@ -71,12 +135,21 @@ struct curlMultiError final : CloneableError } }; +/* Check if the linked libcurl was built with HTTP3 support. */ +bool curlSupportsHttp3() +{ + const auto * info = ::curl_version_info(CURLVERSION_NOW); + return info && (info->features & CURL_VERSION_HTTP3); +} + } // namespace struct curlFileTransfer : public FileTransfer { const FileTransferSettings & settings; + const bool http3Supported = curlSupportsHttp3(); + curlMulti curlm; std::random_device rd; @@ -94,7 +167,7 @@ struct curlFileTransfer : public FileTransfer char errbuf[CURL_ERROR_SIZE]; std::string statusMsg; - unsigned int attempt = 0; + uint32_t attempt = 0; /* Don't start this download until the specified time point has been reached. */ @@ -137,12 +210,26 @@ struct curlFileTransfer : public FileTransfer */ bool hasContentEncoding:1 = false; + /** + * Server-provided minimum retry delay, parsed from the `Retry-After` + * response header. Reset on each new HTTP status line, and consumed + * (cleared) by maybeRetry() so it applies to at most one retry attempt. + */ + std::optional retryAfterMs; + curl_off_t bytesReceived = 0; + curl_off_t writtenToSink = 0; std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now(); - inline static const std::set successfulStatuses{200, 201, 204, 206, 304, 0 /* other protocol */}; + inline static const std::set successfulStatuses{ + static_cast(HttpStatus::Ok), + static_cast(HttpStatus::Created), + static_cast(HttpStatus::NoContent), + static_cast(HttpStatus::PartialContent), + static_cast(HttpStatus::NotModified), + 0 /* other protocol */}; /* Get the HTTP status code, or 0 for other protocols. */ long getHTTPStatus() @@ -226,8 +313,7 @@ struct curlFileTransfer : public FileTransfer curl_easy_cleanup(req); } if (!done && enqueued) - fail(FileTransferError( - Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri)); + failInterruptedOrCancelled(); } void failEx(std::exception_ptr ex) noexcept @@ -240,7 +326,7 @@ struct curlFileTransfer : public FileTransfer /* Already descriptive enough. */ } catch (nix::Error & e) { /* Add more context to the error message. */ - e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.uri.to_string()); + e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.displayUri()); } catch (...) { /* Can't add more context to the error. */ } @@ -253,6 +339,18 @@ struct curlFileTransfer : public FileTransfer failEx(std::make_exception_ptr(std::forward(e))); } + void failInterruptedOrCancelled() + { + HintFmt fmt("%s of '%s' was interrupted", Uncolored(request.noun()), request.displayUri()); + + /* Technically, we don't really have per-transfer cancellation currently, + but it's nice to distinguish between the two in the future. */ + if (getInterrupted()) + fail(nix::Interrupted(std::move(fmt))); + else + fail(nix::Cancelled(std::move(fmt))); + } + LambdaSink finalSink; std::optional errorSink; @@ -304,7 +402,7 @@ struct curlFileTransfer : public FileTransfer try { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); + printMsg(lvlVomit, "got header for '%s': %s", request.displayUri(), trim(line)); static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); if (std::smatch match; std::regex_match(line, match, statusLine)) { @@ -314,6 +412,7 @@ struct curlFileTransfer : public FileTransfer statusMsg = trim(match.str(1)); acceptRanges = false; hasContentEncoding = false; + retryAfterMs = std::nullopt; bytesReceived = 0; appendCurrentUrl(); } else { @@ -331,7 +430,7 @@ struct curlFileTransfer : public FileTransfer data. */ long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - if (result.etag == request.expectedETag && httpStatus == 200) { + if (result.etag == request.expectedETag && httpStatus == HttpStatus::Ok) { debug("shutting down on 200 HTTP response with expected ETag"); return 0; } @@ -354,6 +453,19 @@ struct curlFileTransfer : public FileTransfer else debug("got invalid link header '%s'", value); } + + else if (name == "retry-after") { + auto value = trim(line.substr(i + 1)); + // RFC 7231 §7.1.3: Retry-After = HTTP-date / delay-seconds. + if (auto seconds = string2Int(value)) { + retryAfterMs = saturateMs(std::chrono::seconds{*seconds}); + } else if (time_t date = curl_getdate(requireCString(value), nullptr); date != -1) { + time_t now = time(nullptr); + retryAfterMs = saturateMs(std::chrono::seconds{date > now ? date - now : 0}); + } else { + debug("ignoring unparseable Retry-After header: '%s'", value); + } + } } } return realSize; @@ -380,8 +492,8 @@ struct curlFileTransfer : public FileTransfer *logger, lvlTalkative, actFileTransfer, - fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri), - Logger::Fields{request.uri.to_string()}, + fmt("%s '%s'", request.verb(/*continuous=*/true), request.displayUri()), + Logger::Fields{request.displayUri()}, request.parentAct); // Reset the start time to when we actually started the download. startTime = std::chrono::steady_clock::now(); @@ -522,7 +634,10 @@ struct curlFileTransfer : public FileTransfer : "")) .c_str()); curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); - if (fileTransfer.settings.enableHttp2) + /* Enable HTTP3 only on user config and linked libcurl have support, o.w. fall back. */ + if (fileTransfer.settings.enableHttp3 && fileTransfer.http3Supported) + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_3); + else if (fileTransfer.settings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); @@ -568,8 +683,15 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_SEEKDATA, this); } + /* Note: libcurl copies string arguments, so temporaries from + .string().c_str() are safe. See the comment near CURLOPT_SSLKEY below. */ if (auto & caFile = fileTransfer.settings.caFile.get()) - curl_easy_setopt(req, CURLOPT_CAINFO, caFile->c_str()); + curl_easy_setopt(req, CURLOPT_CAINFO, caFile->string().c_str()); +#ifdef _WIN32 + /* Use native windows certificate store when the option is not specified explicitly. */ + else + curl_easy_setopt(req, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA); +#endif #if !defined(_WIN32) curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback); @@ -589,7 +711,7 @@ struct curlFileTransfer : public FileTransfer /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ - curl_easy_setopt(req, CURLOPT_NETRC_FILE, fileTransfer.settings.netrcFile.get().c_str()); + curl_easy_setopt(req, CURLOPT_NETRC_FILE, fileTransfer.settings.netrcFile.get().string().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); if (requestRange) @@ -644,14 +766,12 @@ struct curlFileTransfer : public FileTransfer { auto finishTime = std::chrono::steady_clock::now(); - auto retryTimeMs = request.baseRetryTimeMs; - auto httpStatus = getHTTPStatus(); debug( "finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", Uncolored(request.noun()), - request.uri, + request.displayUri(), code, httpStatus, result.bodySize, @@ -661,19 +781,19 @@ struct curlFileTransfer : public FileTransfer if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { code = CURLE_OK; - httpStatus = 304; + httpStatus = std::to_underlying(HttpStatus::NotModified); } if (callbackException) failEx(callbackException); else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) { - result.cached = httpStatus == 304; + result.cached = (httpStatus == HttpStatus::NotModified); // In 2021, GitHub responds to If-None-Match with 304, // but omits ETag. We just use the If-None-Match etag // since 304 implies they are the same. - if (httpStatus == 304 && result.etag == "") + if (httpStatus == HttpStatus::NotModified && result.etag == "") result.etag = request.expectedETag; curl_off_t dlSize = 0; @@ -717,24 +837,27 @@ struct curlFileTransfer : public FileTransfer if (std::find(s3RetryableErrors.begin(), s3RetryableErrors.end(), s3ErrorCode) != s3RetryableErrors.end()) { debug("S3 error '%s', will retry", s3ErrorCode); - } else if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) { + } else if ( + httpStatus == HttpStatus::NotFound || httpStatus == HttpStatus::Gone + || code == CURLE_FILE_COULDNT_READ_FILE) { // The file is definitely not there err = NotFound; - } else if (httpStatus == 401 || httpStatus == 407) { + } else if (httpStatus == HttpStatus::Unauthorized || httpStatus == HttpStatus::ProxyAuthRequired) { err = Unauthorized; - } else if (httpStatus == 403) { + } else if (httpStatus == HttpStatus::Forbidden) { // Don't retry on authentication/authorization failures. // Note: the only reason we treat this differently from 401/407 is S3 returns 403 if a file doesn't // exist and the bucket is unlistable. err = Forbidden; - } else if (httpStatus == 429) { - // 429 means too many requests, so we retry (with a substantially longer delay) - retryTimeMs = RETRY_TIME_MS_TOO_MANY_REQUESTS; - } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408) { + } else if ( + httpStatus >= 400 && httpStatus < 500 && httpStatus != HttpStatus::RequestTimeout + && httpStatus != HttpStatus::TooManyRequests) { // Most 4xx errors are client errors and are probably not worth retrying: // * 408 means the server timed out waiting for us, so we try again err = Misc; - } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) { + } else if ( + httpStatus == HttpStatus::NotImplemented || httpStatus == HttpStatus::HttpVersionNotSupported + || httpStatus == HttpStatus::NetworkAuthRequired) { // Let's treat most 5xx (server) errors as transient, except for a handful: // * 501 not implemented // * 505 http version not supported @@ -775,19 +898,20 @@ struct curlFileTransfer : public FileTransfer std::optional response; if (errorSink) response = std::move(errorSink->s); - auto exc = code == CURLE_ABORTED_BY_CALLBACK && getInterrupted() ? FileTransferError( - Interrupted, - std::move(response), - "%s of '%s' was interrupted", - Uncolored(request.noun()), - request.uri) - : httpStatus != 0 + + /* TODO: Also support per-transfer cancellations. */ + if (code == CURLE_ABORTED_BY_CALLBACK && getInterrupted()) { + failInterruptedOrCancelled(); + return; + } + + auto exc = httpStatus != 0 ? FileTransferError( err, std::move(response), "unable to %s '%s': HTTP error %d%s", Uncolored(request.verb()), - request.uri, + request.displayUri(), httpStatus, code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError( @@ -795,52 +919,88 @@ struct curlFileTransfer : public FileTransfer std::move(response), "unable to %s '%s': %s (%d) %s", Uncolored(request.verb()), - request.uri, + request.displayUri(), curl_easy_strerror(code), code, errbuf); - /* If this is a transient error, then maybe retry the - download after a while. If we're writing to a - sink, we can only retry if the server supports - ranged requests. */ - if (err == Transient && attempt < fileTransfer.settings.tries - && !(this->request.dataCallback && hasContentEncoding)) { - int ms = retryTimeMs - * std::pow( - 2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); - - if (writtenToSink) { - if (acceptRanges) - requestRange = true; - warn( - "%s; retrying from offset %d in %d ms (attempt %d/%d)", - exc.message(), - writtenToSink, - ms, - attempt, - fileTransfer.settings.tries); - } else { - warn( - "%s; retrying in %d ms (attempt %d/%d)", - exc.message(), - ms, - attempt, - fileTransfer.settings.tries); - } + maybeRetry(err, httpStatus, std::move(exc)); + } + } - errorSink.reset(); - embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - try { - fileTransfer.enqueueItem(ref{shared_from_this()}); - } catch (const nix::Error & e) { - // If enqueue fails (e.g., during shutdown), fail the transfer properly - // instead of letting the exception propagate, which would leave done=false - // and cause the destructor to attempt a second callback invocation - fail(std::move(exc)); - } - } else - fail(std::move(exc)); + /* If this is a transient error, then maybe retry the download + after a while. If we're writing to a sink, we can only retry + if the server supports ranged requests. */ + void maybeRetry(FileTransfer::Error err, long httpStatus, FileTransferError && exc) + { + // Resolve effective retry config (request overrides > global settings) + auto effAttempts = request.retryAttempts.value_or(fileTransfer.settings.tries); + auto effBaseMs = request.retryDelayMs.value_or(fileTransfer.settings.retryDelayMs); + auto effRateLimitMs = + request.retryDelayRateLimitedMs.value_or(fileTransfer.settings.retryDelayRateLimitedMs); + auto effMaxMs = request.retryMaxDelayMs.value_or(fileTransfer.settings.retryMaxDelayMs); + + // Pick base delay by error class: 429/503 indicate the server is + // rate-limiting or overloaded, so use the longer rate-limit delay. + uint32_t baseMs = + (httpStatus == HttpStatus::TooManyRequests || httpStatus == HttpStatus::ServiceUnavailable) + ? effRateLimitMs + : effBaseMs; + + auto canRetry = [&] { + if (err != Transient) + return false; + if (attempt >= effAttempts) + return false; + // If we've already streamed bytes to the callback, we can + // resume via a Range request (if the server accepts byte + // ranges), or start over and discard the data we've already + // received. Neither works if the response is compressed (the + // Range applies to the encoded stream, but the sink saw + // decoded bytes). + if (request.dataCallback && writtenToSink != 0) + return !hasContentEncoding; + return true; + }(); + + if (!canRetry) { + fail(std::move(exc)); + return; + } + + auto delay = computeRetryDelayMs( + { + .attempt = attempt, + .baseMs = baseMs, + .ceilMs = effMaxMs, + .retryAfterMs = std::exchange(retryAfterMs, std::nullopt), + .jitter = fileTransfer.settings.retryJitter, + }, + fileTransfer.mt19937); + + if (writtenToSink) { + if (acceptRanges) + requestRange = true; + warn( + "%s; retrying from offset %d in %d ms (attempt %d/%d)", + exc.message(), + writtenToSink, + delay.count(), + attempt, + effAttempts); + } else { + warn("%s; retrying in %d ms (attempt %d/%d)", exc.message(), delay.count(), attempt, effAttempts); + } + + errorSink.reset(); + embargo = std::chrono::steady_clock::now() + delay; + try { + fileTransfer.enqueueItem(ref{shared_from_this()}); + } catch (const nix::Error & e) { + // If enqueue fails (e.g., during shutdown), fail the transfer properly + // instead of letting the exception propagate, which would leave done=false + // and cause the destructor to attempt a second callback invocation + fail(std::move(exc)); } } }; @@ -904,15 +1064,7 @@ struct curlFileTransfer : public FileTransfer workerThread = std::thread([&]() { workerThreadEntry(); }); } - ~curlFileTransfer() - { - try { - stopWorkerThread(); - } catch (...) { - ignoreExceptionInDestructor(); - } - workerThread.join(); - } + ~curlFileTransfer() override; void stopWorkerThread() { @@ -1012,7 +1164,7 @@ struct curlFileTransfer : public FileTransfer } for (auto & item : incoming) { - debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.uri); + debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.displayUri()); item->init(); curl_multi_add_handle(curlm.get(), item->req); item->active = true; @@ -1080,7 +1232,7 @@ struct curlFileTransfer : public FileTransfer { if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https" && item->request.uri.scheme() != "s3") - throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string()); + throw nix::Error("uploading to '%s' is not supported", item->request.displayUri()); { auto state(state_.lock()); @@ -1138,6 +1290,16 @@ struct curlFileTransfer : public FileTransfer } }; +curlFileTransfer::~curlFileTransfer() +{ + try { + stopWorkerThread(); + } catch (...) { + ignoreExceptionInDestructor(); + } + workerThread.join(); +} + ref makeCurlFileTransfer(const FileTransferSettings & settings = fileTransferSettings) { return make_ref(settings); @@ -1168,6 +1330,20 @@ ref makeFileTransfer(const FileTransferSettings & settings) return makeCurlFileTransfer(settings); } +std::string FileTransferRequest::displayUri() const +{ + try { + auto parsed = uri.parsed(); + if (parsed.authority && parsed.authority->user) { + parsed.authority->user.reset(); + parsed.authority->password.reset(); + return parsed.to_string(); + } + } catch (BadURL &) { + } + return uri.to_string(); +} + void FileTransferRequest::setupForS3() { auto parsedS3 = ParsedS3URL::parse(uri.parsed()); @@ -1244,7 +1420,7 @@ void FileTransfer::download( bool paused = false; std::exception_ptr exc; std::string data; - std::condition_variable avail, request; + std::condition_variable avail; }; auto _state = std::make_shared>(); @@ -1254,10 +1430,9 @@ void FileTransfer::download( Finally finally([&]() { auto state(_state->lock()); state->quit = true; - state->request.notify_one(); }); - request.dataCallback = [_state, uri = request.uri.to_string()](std::string_view data) -> PauseTransfer { + request.dataCallback = [_state, uri = request.displayUri()](std::string_view data) -> PauseTransfer { auto state(_state->lock()); if (state->quit) @@ -1300,7 +1475,6 @@ void FileTransfer::download( state->exc = std::current_exception(); } state->avail.notify_one(); - state->request.notify_one(); }}); while (true) { @@ -1334,8 +1508,6 @@ void FileTransfer::download( chunk = std::move(state->data); /* Reset state->data after the move, since we check data.empty() */ state->data = ""; - - state->request.notify_one(); } /* Flush the data to the sink and wake up the download thread @@ -1346,6 +1518,8 @@ void FileTransfer::download( } } +void FileTransferError::anchor() {} + template FileTransferError::FileTransferError( FileTransfer::Error error, std::optional response, const Args &... args) diff --git a/src/libstore/freebsd/build/chroot-freebsd-derivation-builder.hh b/src/libstore/freebsd/build/chroot-freebsd-derivation-builder.hh new file mode 100644 index 000000000000..7ff6d1fb4615 --- /dev/null +++ b/src/libstore/freebsd/build/chroot-freebsd-derivation-builder.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "chroot-derivation-builder.hh" +#include "freebsd-derivation-builder.hh" + +#include "nix/util/freebsd-jail.hh" + +namespace nix { + +struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivationBuilder +{ + std::shared_ptr autoDelJail = std::make_shared(); + + ChrootFreeBSDDerivationBuilder( + LocalStore & store, std::shared_ptr miscMethods, DerivationBuilderParams params) + : DerivationBuilderImpl{store, miscMethods, params} + , ChrootDerivationBuilder{store, miscMethods, params} + , FreeBSDDerivationBuilder{store, miscMethods, params} + { + } + + virtual void cleanupBuild(bool force) override; + + void prepareSandbox() override; + + void startChild() override; + + void enterChroot() override; + + void addDependencyImpl(const StorePath & path) override; +}; + +} // namespace nix diff --git a/src/libstore/freebsd/build/freebsd-derivation-builder.cc b/src/libstore/freebsd/build/freebsd-derivation-builder.cc new file mode 100644 index 000000000000..68ccdafa1ae5 --- /dev/null +++ b/src/libstore/freebsd/build/freebsd-derivation-builder.cc @@ -0,0 +1,484 @@ +#include "derivation-builder-impl.hh" +#include "freebsd-derivation-builder.hh" +#include "chroot-derivation-builder.hh" +#include "chroot-freebsd-derivation-builder.hh" + +#include "nix/util/freebsd-jail.hh" +#include "nix/util/util.hh" +#include "nix/store/filetransfer.hh" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nix { + +FreeBSDDerivationBuilder::~FreeBSDDerivationBuilder() {} + +namespace { + +struct PasswordEntry +{ + std::string name; + uid_t uid; + gid_t gid; + std::string description; + std::filesystem::path home; + std::filesystem::path shell; +}; + +using UniqueDB = std::unique_ptr<::DB, decltype([](::DB * db) { + if (db) + (db->close)(db); + })>; + +// Database open flags from FreeBSD, in case they're necessary for compatibility +static constexpr HASHINFO dbFlags = { + .bsize = 4096, + .ffactor = 32, + .nelem = 256, + .cachesize = 2 * 1024 * 1024, + .hash = nullptr, + .lorder = BIG_ENDIAN, +}; + +// Password database version +// Version 4 has been current since 2003 +static const uint8_t dbVersion = 4; + +static void serializeString(std::vector & buf, std::string const & str) +{ + buf.insert(buf.end(), str.begin(), str.end()); + buf.push_back(0); +} + +static void serializeInt(std::vector & buf, uint32_t num) +{ + // Always big endian + buf.push_back((num >> 24) & 0xff); + buf.push_back((num >> 16) & 0xff); + buf.push_back((num >> 8) & 0xff); + buf.push_back((num >> 0) & 0xff); +} + +static std::vector byNameKey(std::string const & name) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYNAME, dbVersion)}; + buf.reserve(1 + name.size()); + // We can't use serializeString since that's null terminated + buf.insert(buf.end(), name.begin(), name.end()); + + return buf; +} + +static std::vector byNumKey(uint32_t num) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYNUM, dbVersion)}; + serializeInt(buf, num); + + return buf; +} + +static std::vector byUidKey(uid_t uid) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYUID, dbVersion)}; + serializeInt(buf, uid); + + return buf; +} + +static void createPasswordFiles(std::filesystem::path & chrootRootDir, std::vector & users) +{ + auto db = + UniqueDB(::dbopen((chrootRootDir / "etc/pwd.db").c_str(), O_CREAT | O_RDWR | O_EXCL, 0644, DB_HASH, &dbFlags)); + + if (!db) + throw SysError("could not create password database"); + + auto dbInsert = [&db](std::vector keyBuf, std::vector & valueBuf) { + DBT key = {keyBuf.data(), keyBuf.size()}; + DBT value = {valueBuf.data(), valueBuf.size()}; + + if ((db->put)(db.get(), &key, &value, R_NOOVERWRITE) == -1) { + throw SysError("could not write to password database"); + } + }; + + // Annoyingly DBT doesn't have const pointers so we need this whole shuffle + std::string versionKeyStr(_PWD_VERSION_KEY); + std::vector versionKey(versionKeyStr.begin(), versionKeyStr.end()); + std::vector versionValue{dbVersion}; + dbInsert(versionKey, versionValue); + + for (const auto & [i, user] : enumerate(users)) { + // flags for non-empty fields + uint32_t fields = _PWF_NAME | _PWF_PASSWD | _PWF_UID | _PWF_GID | _PWF_GECOS | _PWF_DIR | _PWF_SHELL; + + std::vector buf; + serializeString(buf, user.name); + // pw_password is always "*" in the insecure database + serializeString(buf, std::string("*")); + serializeInt(buf, user.uid); + serializeInt(buf, user.gid); + // pw_change = 0 means no requirement to change password + serializeInt(buf, 0); + // pw_class is empty since we don't make a class database + serializeString(buf, std::string("")); + serializeString(buf, user.description); + serializeString(buf, user.home); + serializeString(buf, user.shell); + // pw_expire = 0 means password does not expire + serializeInt(buf, 0); + serializeInt(buf, fields); + + dbInsert(byNameKey(user.name), buf); + // _PW_KEYBYNUM is 1-indexed + dbInsert(byNumKey(i + 1), buf); + dbInsert(byUidKey(user.uid), buf); + } + + // FreeBSD libc doesn't use /etc/passwd, but some software might + std::string passwdContent; + for (const auto & user : users) { + passwdContent.append( + fmt("%s:*:%d:%d:%s:%s:%s\n", + user.name, + user.uid, + user.gid, + user.description, + user.home.native(), + user.shell.native())); + } + + writeFile(chrootRootDir / "etc/passwd", passwdContent); + + // No need to make /etc/master.passwd or /etc/spwd.db, + // our build user wouldn't be able to read them anyway +} + +} // namespace + +template +struct iovec iovFromMutableBuffer(std::array & array) +{ + return { + .iov_base = static_cast(array.data()), + .iov_len = N, + }; +} + +template +struct iovec iovFromStaticSizedString(const char (&array)[N]) +{ + return { + .iov_base = const_cast(static_cast(array)), + .iov_len = N, + }; +} + +struct iovec iovFromDynamicSizeString(const std::string & s) +{ + return { + .iov_base = const_cast(static_cast(s.c_str())), + .iov_len = s.length() + 1, + }; +} + +void ChrootFreeBSDDerivationBuilder::cleanupBuild(bool force) +{ + autoDelJail->remove(); + ChrootDerivationBuilder::cleanupBuild(force); +} + +void ChrootFreeBSDDerivationBuilder::prepareSandbox() +{ + ChrootDerivationBuilder::prepareSandbox(); + + std::vector users{ + { + .name = "root", + .uid = 0, + .gid = 0, + .description = "Nix build user", + .home = store.config->getLocalSettings().sandboxBuildDir, + .shell = "/noshell", + }, + { + .name = "nixbld", + .uid = buildUser->getUID(), + .gid = sandboxGid(), + .description = "Nix build user", + .home = store.config->getLocalSettings().sandboxBuildDir, + .shell = "/noshell", + }, + { + .name = "nobody", + .uid = 65534, + .gid = 65534, + .description = "Nobody", + .home = "/", + .shell = "/noshell", + }, + }; + + createPasswordFiles(chrootRootDir, users); + + // FreeBSD doesn't have a group database, just write a text file + writeFile( + chrootRootDir / "etc/group", + fmt("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n", + sandboxGid())); + + // Linux waits until after entering the child to start mounting so it doesn't + // pollute the root mount namespace. + // FreeBSD doesn't have mount namespaces, so there's no reason to wait. + + auto devpath = chrootRootDir / "dev"; + createDir(devpath, 0555); + createDir(chrootRootDir / "bin", 0555); + + std::array errmsg{}; + std::array<::iovec, 8> iov = { + iovFromStaticSizedString("fstype"), + iovFromStaticSizedString("devfs"), + iovFromStaticSizedString("fspath"), + iovFromDynamicSizeString(devpath.native()), + iovFromStaticSizedString("ruleset"), + iovFromStaticSizedString("4"), + iovFromStaticSizedString("errmsg"), + iovFromMutableBuffer(errmsg), + }; + + if (nmount(iov.data(), iov.size(), 0) < 0) + throw SysError("failed to mount jail /dev: %1%", std::string_view(errmsg.data())); + + autoDelJail->childrenMounts.emplace_back(devpath); + + for (const auto & [target, chrootPath] : pathsInChroot) { + std::filesystem::path path = chrootRootDir / target.relative_path(); + + auto maybeSt = maybeLstat(chrootPath.source); + if (!maybeSt) { + if (chrootPath.optional) + continue; /* Skip mounting this path. */ + else + throw SysError("getting attributes of path %1%", PathFmt(chrootPath.source)); + } + + /* Mount points must exist and be the right type. */ + if (S_ISDIR(maybeSt->st_mode)) { + createDirs(path); + } else if (S_ISLNK(maybeSt->st_mode)) { + createDirs(path.parent_path()); + copyFile(chrootPath.source, path, /*andDelete=*/false, /*contents=*/false); + continue; + } else { + createDirs(path.parent_path()); + writeFile(path, ""); + } + + std::array<::iovec, 8> iov = { + iovFromStaticSizedString("fstype"), + iovFromStaticSizedString("nullfs"), + iovFromStaticSizedString("fspath"), + iovFromDynamicSizeString(path.native()), + iovFromStaticSizedString("target"), + iovFromDynamicSizeString(chrootPath.source.native()), + iovFromStaticSizedString("errmsg"), + iovFromMutableBuffer(errmsg), + }; + + debug("setting up a nullfs mount from %1% to %2%", PathFmt(chrootPath.source), PathFmt(path)); + + int flags = 0; + if (store.isInStore(target.native())) + /* While we are at it, enforce invariants about store paths. Anything located at the "logical" store + location must be readonly (file permission canonicalisation enforces this on the host filesystem). + Also the store must never contain setuid binaries for the same reason. This is just defense-in-depth. */ + flags = MNT_RDONLY | MNT_NOSUID; + + if (nmount(iov.data(), iov.size(), flags) < 0) + throw SysError("failed to mount nullfs for %1%: %2%", PathFmt(path), std::string_view(errmsg.data())); + + autoDelJail->childrenMounts.emplace_back(path); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (!derivationType.isSandboxed()) { + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed-outputs. + writeFile(chrootRootDir / "etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + + /* N.B. it is realistic that these paths might not exist. It + happens when testing Nix building fixed-output derivations + within a pure derivation. */ + for (std::filesystem::path path : {"/etc/resolv.conf", "/etc/services", "/etc/hosts"}) { + if (pathExists(path)) { + // This means if your network config changes during a FOD build, + // the DNS in the sandbox will be wrong. However, this is pretty unlikely + // to actually be a problem, because FODs are generally pretty fast, + // and machines with often-changing network configurations probably + // want to run resolved or some other local resolver anyway. + // + // There's also just no simple way to do this correctly, you have to manually + // inotify watch the files for changes on the outside and update the sandbox + // while the build is running (or at least that's what Flatpak does). + // + // I also just generally feel icky about modifying sandbox state under a build, + // even though it really shouldn't be a big deal. -K900 + copyFile(path, chrootRootDir / path.relative_path(), false, true); + } + } + + if (fileTransferSettings.caFile.get() && pathExists(fileTransferSettings.caFile.get().value())) { + // For the same reasons as above, copy the CA certificates file too. + // It should be even less likely to change during the build than resolv.conf. + createDirs(chrootRootDir / "etc/ssl/certs"); + copyFile( + fileTransferSettings.caFile.get().value(), + chrootRootDir / "etc/ssl/certs/ca-certificates.crt", + false, + true); + } + } +} + +void ChrootFreeBSDDerivationBuilder::startChild() +{ + int jid; + + RunChildArgs args{ +#if NIX_WITH_AWS_AUTH + .awsCredentials = preResolveAwsCredentials(), +#endif + }; + + if (derivationType.isSandboxed()) { + jid = jail_setv( + JAIL_CREATE, + "persist", + "true", + "path", + chrootRootDir.c_str(), + "host.hostname", + "localhost", + // TODO: Make our own ruleset + "vnet", + "new", + nullptr); + if (jid < 0) { + throw SysError("failed to create jail (isolated network): %1%", jail_errmsg); + } + autoDelJail->jid = jid; + + // Everything from here to the end of the block is setting up the network + // code adapted from freebsd/sbin/ifconfig/af_inet.c, in_exec_nl + Pid helper = startProcess([&]() { + unix::closeExtraFDs(); + enterChroot(); + + struct snl_state ss = {}; + if (!snl_init(&ss, NETLINK_ROUTE)) { + throw SysError("Failed to init netlink connection"); + } + + struct snl_writer nw = {}; + snl_init_writer(&ss, &nw); + struct nlmsghdr * hdr = snl_create_msg_request(&nw, NL_RTM_NEWADDR); + struct ifaddrmsg * ifahdr = snl_reserve_msg_object(&nw, struct ifaddrmsg); + + ifahdr->ifa_family = AF_INET; + ifahdr->ifa_prefixlen = 8; + ifahdr->ifa_index = if_nametoindex("lo0"); + snl_add_msg_attr_ip4(&nw, IFA_LOCAL, (const struct in_addr *) "\x7f\x00\x00\x01"); + + int off = snl_add_msg_attr_nested(&nw, IFA_FREEBSD); + snl_add_msg_attr_u32(&nw, IFAF_FLAGS, IFF_LOOPBACK | IFF_UP); + snl_end_attr_nested(&nw, off); + + if (!(hdr = snl_finalize_msg(&nw)) || !snl_send_message(&ss, hdr)) { + snl_free(&ss); + throw SysError("failed to sendoff netlink message"); + } + + struct snl_errmsg_data e = {}; + snl_read_reply_code(&ss, hdr->nlmsg_seq, &e); + if (e.error_str != nullptr) { + snl_free(&ss); + throw SysError("failed to configure loopback interface: %1%", e.error_str); + } + snl_free(&ss); + _exit(0); + }); + + /* TODO: Capture the error from the helper? */ + if (auto status = helper.wait(); !statusOk(status)) { + throw Error("failed to configure loopback address: %s", statusToString(status)); + } + } else { + jid = jail_setv( + JAIL_CREATE, + "persist", + "true", + // 4 is the most restrictive devfs ruleset that meets our needs + // which is found in the default installation. Trying to add + // another one is a huge pain... + "devfs_ruleset", + "4", + "path", + chrootRootDir.c_str(), + "host.hostname", + "localhost", + "ip4", + "inherit", + "ip6", + "inherit", + "allow.raw_sockets", + "true", + nullptr); + if (jid < 0) { + throw SysError("failed to create jail (networked): %1%", jail_errmsg); + } + autoDelJail->jid = jid; + } + + pid = startProcess([&]() { + openSlave(); + runChild(args); + }); +} + +void ChrootFreeBSDDerivationBuilder::enterChroot() +{ + /* Close all other file descriptors. This must happen before + jail_attach for FreeBSD. */ + unix::closeExtraFDs(); + + if (jail_attach(autoDelJail->jid) < 0) { + throw SysError("failed to attach to jail"); + } +} + +void ChrootFreeBSDDerivationBuilder::addDependencyImpl(const StorePath & path) +{ + throw UnimplementedError( + "adding store path '%s' to the sandbox is not implemented (recursive-nix)", store.printStorePath(path)); +} + +} // namespace nix diff --git a/src/libstore/freebsd/build/freebsd-derivation-builder.hh b/src/libstore/freebsd/build/freebsd-derivation-builder.hh new file mode 100644 index 000000000000..ce607db3982c --- /dev/null +++ b/src/libstore/freebsd/build/freebsd-derivation-builder.hh @@ -0,0 +1,19 @@ +#pragma once + +#include "derivation-builder-impl.hh" + +namespace nix { + +struct FreeBSDDerivationBuilder : virtual DerivationBuilderImpl +{ + using DerivationBuilderImpl::DerivationBuilderImpl; + + FreeBSDDerivationBuilder(FreeBSDDerivationBuilder &&) = delete; + FreeBSDDerivationBuilder(const FreeBSDDerivationBuilder &) = delete; + FreeBSDDerivationBuilder & operator=(FreeBSDDerivationBuilder &&) = delete; + FreeBSDDerivationBuilder & operator=(const FreeBSDDerivationBuilder &) = delete; + /* To appease Wweak-vtables. */ + ~FreeBSDDerivationBuilder() override; +}; + +} // namespace nix diff --git a/src/libstore/freebsd/meson.build b/src/libstore/freebsd/meson.build new file mode 100644 index 000000000000..bac3949b965f --- /dev/null +++ b/src/libstore/freebsd/meson.build @@ -0,0 +1,3 @@ +include_dirs += [ include_directories('build') ] + +sources += files('build/freebsd-derivation-builder.cc') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index a4814779f34d..73adbac343a1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,9 +1,10 @@ -#include "nix/store/derivations.hh" -#include "nix/store/globals.hh" +#include "nix/store/gc-store.hh" #include "nix/store/local-gc.hh" +#include "nix/store/local-settings.hh" #include "nix/store/local-store.hh" #include "nix/store/path.hh" #include "nix/util/configuration.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/finally.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/signals.hh" @@ -22,6 +23,7 @@ #include #include #include +#include #if HAVE_STATVFS # include #endif @@ -35,6 +37,10 @@ namespace nix { +void LocalSettings::anchor() {} + +void GCSettings::anchor() {} + static std::string gcSocketPath = "gc-socket/socket"; static std::string gcRootsDir = "gcroots"; @@ -182,8 +188,6 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) } auto path = i.path(); - pid_t pid = std::stoi(name); - debug("reading temporary root file %1%", PathFmt(path)); AutoCloseFD fd(toDescriptor(open( path.string().c_str(), @@ -218,7 +222,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) while ((end = contents.find((char) 0, pos)) != std::string::npos) { auto root = std::string_view(contents).substr(pos, end - pos); debug("got temporary root '%s'", root); - tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{nix-process:%d}", pid)); + tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{nix-process:%s}", name)); pos = end + 1; } } @@ -369,12 +373,18 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) const auto & gcSettings = config->getLocalSettings().getGCSettings(); bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; - bool keepOutputs = gcSettings.keepOutputs; - bool keepDerivations = gcSettings.keepDerivations; Roots roots; boost::unordered_flat_set> dead, alive; + /* Return early if nothing to delete */ + if (std::visit( + overloaded{ + [](const GCOptions::SpecificPaths & pathsToDelete) { return pathsToDelete.paths.empty(); }, + [](const GCOptions::WholeStore & _) { return false; }}, + options.pathsToDelete)) + return; + struct Shared { // The temp roots only store the hash part to make it easier to @@ -390,15 +400,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) std::condition_variable wakeup; - /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `keep-outputs' or `keep-derivations' are true - (the garbage collector will recurse into deleting the outputs - or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - keepOutputs = false; - keepDerivations = false; - } - if (shouldDelete) deletePath(reservedPath); @@ -560,7 +561,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) by another process. We need to be sure that we can acquire an exclusive lock before deleting them. */ if (baseName.find("tmp-", 0) == 0) { - auto tmpDirFd = openDirectory(realPath); + /* TODO Reconsider whether Follow is the right choice, here */ + auto tmpDirFd = openDirectory(realPath, FinalSymlink::Follow); if (!tmpDirFd || !lockFile(tmpDirFd.get(), ltWrite, false)) { debug("skipping locked tempdir %s", PathFmt(realPath)); return; @@ -596,7 +598,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) via the referrers edges and optionally derivers and derivation output edges. If none of those paths are roots, then all visited paths are garbage and are deleted. */ - auto deleteReferrersClosure = [&](const StorePath & start) { + auto maybeDeleteReferrersClosure = [&](const StorePath & start) { StorePathSet visited; std::queue todo; @@ -613,6 +615,33 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) todo.push(path); }; + auto markAlive = [&](const StorePath & p) { + alive.insert(p); + try { + StorePathSet closure; + bool includeOutputs = false; + bool includeDerivers = false; + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + includeOutputs = gcSettings.keepOutputs; + includeDerivers = gcSettings.keepDerivations; + }, + [](const GCOptions::SpecificPaths &) {}, + }, + options.pathsToDelete); + computeFSClosure( + p, + closure, + /* flipDirection */ false, + includeOutputs, + includeDerivers); + for (auto & c : closure) + alive.insert(c); + } catch (InvalidPath &) { + } + }; + enqueue(start); while (auto path = pop(todo)) { @@ -620,40 +649,31 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Bail out if we've previously discovered that this path is alive. */ - if (alive.count(*path)) { + if (alive.contains(*path)) { + debug("cannot delete '%s' because '%s' is alive", printStorePath(start), printStorePath(*path)); alive.insert(start); return; } /* If we've previously deleted this path, we don't have to handle it again. */ - if (dead.count(*path)) + if (dead.contains(*path)) continue; - auto markAlive = [&]() { - alive.insert(*path); - alive.insert(start); - try { - StorePathSet closure; - computeFSClosure( - *path, - closure, - /* flipDirection */ false, - keepOutputs, - keepDerivations); - for (auto & p : closure) - alive.insert(p); - } catch (InvalidPath &) { + if (auto pathsToDelete = std::get_if(&options.pathsToDelete)) { + if (!pathsToDelete->deleteReferrers && !pathsToDelete->paths.contains(*path)) { + if (options.action != GCOptions::gcDeleteDead) + throw Error( + "Cannot delete path '%s' because it's referenced by path '%s'.", + printStorePath(start), + printStorePath(*path)); + debug( + "cannot delete '%s' because '%s' is not in the specified paths to delete", + printStorePath(start), + printStorePath(*path)); + return; } - }; - - if (options.action == GCOptions::gcDeleteSpecific && !options.pathsToDelete.count(*path)) { - throw Error( - "Cannot delete path '%s' because it's referenced by path '%s'.", - printStorePath(start), - printStorePath(*path)); } - /* If this is a root, bail out. */ if (auto i = roots.find(*path); i != roots.end()) { if (options.action == GCOptions::gcDeleteSpecific) @@ -662,7 +682,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printStorePath(start), *i->second.begin()); debug("cannot delete '%s' because it's a root", printStorePath(*path)); - return markAlive(); + alive.insert(start); + return markAlive(*path); } static bool inTest = getEnv("_NIX_IN_TEST").has_value(); @@ -673,7 +694,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (options.action == GCOptions::gcDeleteSpecific) throw Error( "Cannot delete path '%s' because it's in use by '%s'.", printStorePath(start), i->second); - return markAlive(); + alive.insert(start); + return markAlive(*path); } shared->pending = hashPart; } @@ -691,27 +713,60 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) for (auto & p : i->second) enqueue(p); - /* If keep-derivations is set and this is a - derivation, then visit the derivation outputs. */ - if (keepDerivations && path->isDerivation()) { - for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) - if (maybeOutPath && isValidPath(*maybeOutPath) - && queryPathInfo(*maybeOutPath)->deriver == *path) - enqueue(*maybeOutPath); - } - - /* If keep-outputs is set, then visit the derivers. */ - if (keepOutputs) { - auto derivers = queryValidDerivers(*path); - for (auto & i : derivers) - enqueue(i); - } + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + /* If keep-derivations is set and this is a derivation, then we only want to delete this + * derivation if we can also delete all its outputs, so visit the derivation outputs. */ + if (gcSettings.keepDerivations && path->isDerivation()) + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) + if (maybeOutPath && isValidPath(*maybeOutPath) + && queryPathInfo(*maybeOutPath)->deriver == path) + enqueue(*maybeOutPath); + + /* If keep-outputs is set, we only want to delete this path if we + * can also delete its derivers, so visit the derivers. */ + if (gcSettings.keepOutputs) { + auto derivers = queryValidDerivers(*path); + for (auto & i : derivers) + enqueue(i); + } + }, + [](const GCOptions::SpecificPaths &) {}, + }, + options.pathsToDelete); } } for (auto & path : topoSortPaths(visited)) { if (!dead.insert(path).second) continue; if (shouldDelete) { + /* Re-check tempRoots before deleting and set pending + to synchronise with addTempRoot. Between the BFS + and this deletion loop, new temproots may have been + added via the GC socket by a concurrent process + (e.g. an evaluator calling addTempRoot). The BFS + only checks tempRoots when it first visits a path, + but the "pending" mechanism only blocks the socket + handler for the single path currently being visited, + not for paths already queued for deletion. */ + { + auto hashPart = std::string(path.hashPart()); + auto shared(_shared.lock()); + if (shared->tempRoots.contains(hashPart)) { + debug( + "not deleting '%s' because it became a temporary root after initial scan", + printStorePath(path)); + markAlive(path); + continue; + } + shared->pending = hashPart; + } + Finally resetPending([&]() { + auto shared(_shared.lock()); + shared->pending.reset(); + wakeup.notify_all(); + }); try { invalidatePathChecked(path); deleteFromStore(path.to_string(), true); @@ -725,45 +780,67 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - /* Either delete all garbage paths, or just the specified - paths (for gcDeleteSpecific). */ - if (options.action == GCOptions::gcDeleteSpecific) { - - for (auto & i : options.pathsToDelete) { - deleteReferrersClosure(i); - assert(dead.count(i)); - } - - } else if (options.maxFreed > 0) { + try { + /* Either delete all garbage paths, or just the specified paths. */ + std::visit( + overloaded{ + [&](const GCOptions::SpecificPaths & pathsToDelete) { + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage within specified paths..."); + break; + case GCOptions::gcDeleteSpecific: + printInfo("deleting specified paths..."); + break; + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } - if (shouldDelete) - printInfo("deleting garbage..."); - else - printInfo("determining live/dead paths..."); + for (auto & i : pathsToDelete.paths) { + maybeDeleteReferrersClosure(i); + assert(options.action == GCOptions::gcDeleteDead || dead.count(i)); + } + }, + [&](const GCOptions::WholeStore & _) { + if (options.maxFreed == 0) + return; - try { - AutoCloseDir dir(opendir(config->realStoreDir.get().string().c_str())); - if (!dir) - throw SysError("opening directory %1%", PathFmt(config->realStoreDir.get())); - - /* Read the store and delete all paths that are invalid or - unreachable. We don't use readDirectory() here so that - GCing can start faster. */ - auto linksName = linksDir.filename(); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == ".." || name == linksName) - continue; + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage..."); + break; + case GCOptions::gcDeleteSpecific: + throw Error("Cannot delete the entire store"); + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } - if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) - deleteReferrersClosure(*storePath); - else - deleteFromStore(name, false); - } - } catch (GCLimitReached & e) { - } + AutoCloseDir dir(opendir(config->realStoreDir.get().string().c_str())); + if (!dir) + throw SysError("opening directory %1%", PathFmt(config->realStoreDir.get())); + + /* Read the store and delete all paths that are invalid or + unreachable. We don't use readDirectory() here so that + GCing can start faster. */ + auto linksName = linksDir.filename(); + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == ".." || name == linksName) + continue; + + if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) + maybeDeleteReferrersClosure(*storePath); + else + deleteFromStore(name, false); + } + }, + }, + options.pathsToDelete); + } catch (GCLimitReached & e) { } if (options.action == GCOptions::gcReturnLive) { @@ -878,12 +955,17 @@ void LocalStore::autoGC(bool sync) if (avail > state->availAfterGC * 0.97) return; + /* Note: since gcRunning is false here, any previous GC thread has exited / is exiting so the join() should be + * almost instantenous. */ + if (state->gcThread.joinable()) + state->gcThread.join(); + state->gcRunning = true; std::promise promise; future = state->gcFuture = promise.get_future().share(); - std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { + state->gcThread = std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { try { /* Wake up any threads waiting for the auto-GC to finish. */ @@ -910,7 +992,7 @@ void LocalStore::autoGC(bool sync) // future, but we don't really care. (what??) ignoreExceptionInDestructor(); } - }).detach(); + }); } sync: diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index cc793db89ef0..57c17680f783 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -4,12 +4,10 @@ #include "nix/util/config-global.hh" #include "nix/util/current-process.hh" #include "nix/util/executable-path.hh" -#include "nix/util/archive.hh" #include "nix/util/args.hh" #include "nix/util/abstract-setting-to-json.hh" #include "nix/util/compute-levels.hh" #include "nix/util/executable-path.hh" -#include "nix/util/signals.hh" #include "nix/store/filetransfer.hh" #include @@ -36,8 +34,6 @@ # include "nix/util/processes.hh" #endif -#include "nix/util/config-impl.hh" - #ifdef __APPLE__ # include #endif @@ -50,46 +46,33 @@ namespace nix { -/* The default location of the daemon socket, relative to nixStateDir. - The socket is in a directory to allow you to control access to the - Nix daemon by setting the mode/ownership of the directory - appropriately. (This wouldn't work on the socket itself since it - must be deleted and recreated on startup.) */ -#define DEFAULT_SOCKET_PATH "daemon-socket/socket" +void Settings::anchor() {} -/** - * Helper to resolve the NIX_CONF_DIR at runtime on Windows. - * On Windows, NIX_CONF_DIR is not defined at compile time, so we determine - * the path at runtime using the Windows known folders API (FOLDERID_ProgramData). - * This allows Nix to work correctly regardless of which drive Windows is installed on. - */ -static std::filesystem::path resolveNixConfDir() -{ -#ifdef _WIN32 -# ifdef NIX_CONF_DIR - // On Windows, NIX_CONF_DIR should not be defined at compile time -# error "NIX_CONF_DIR should not be defined on Windows" -# endif - return windows::known_folders::getProgramData() / "nix"; -#else - return NIX_CONF_DIR; -#endif -} +void NarInfoDiskCacheSettings::anchor() {} -LogFileSettings::LogFileSettings() - : nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) -{ -} +void LogFileSettings::anchor() {} + +void AutoAllocateUidSettings::anchor() {} Settings settings; static GlobalConfig::Register rSettings(&settings); Settings::Settings() - : nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) - , nixDaemonSocketFile(canonPath(getEnvOsNonEmpty(OS_STR("NIX_DAEMON_SOCKET_PATH")) - .transform([](auto && s) { return std::filesystem::path(s); }) - .value_or(nixStateDir / DEFAULT_SOCKET_PATH))) + : nixStateDir(getEnvOsNonEmpty(OS_STR("NIX_STATE_DIR")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { +#ifdef _WIN32 +# ifdef NIX_STATE_DIR +# error "NIX_STATE_DIR should not be defined on Windows" +# endif + return windows::known_folders::getProgramData() / "nix" / "state"; +#else + return NIX_STATE_DIR; +#endif + }) + .transform([](auto && s) { return canonPath(s); }) + .value()) { #ifndef _WIN32 buildUsersGroup = isRootUser() ? "nixbld" : ""; @@ -153,12 +136,28 @@ void loadConfFile(AbstractConfig & config) } } +/** + * On Windows, NIX_CONF_DIR (and other directories like NIX_STATE_DIR, NIX_LOG_DIR) + * are not defined at compile time, so we determine paths at runtime using the + * Windows known folders API (FOLDERID_ProgramData). This allows Nix to work + * correctly regardless of which drive Windows is installed on. + */ const std::filesystem::path & nixConfDir() { - static const std::filesystem::path dir = - canonPath(getEnvOsNonEmpty(OS_STR("NIX_CONF_DIR")) - .transform([](auto && s) { return std::filesystem::path(s); }) - .value_or(resolveNixConfDir())); + static const std::filesystem::path dir = getEnvOsNonEmpty(OS_STR("NIX_CONF_DIR")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { +#ifdef _WIN32 +# ifdef NIX_CONF_DIR +# error "NIX_CONF_DIR should not be defined on Windows" +# endif + return windows::known_folders::getProgramData() / "nix" / "conf"; +#else + return NIX_CONF_DIR; +#endif + }) + .transform([](auto && s) { return canonPath(s); }) + .value(); return dir; } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 7a2f8b8b6d56..eb4613ecc24d 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -7,12 +7,15 @@ #include "nix/util/closure.hh" #include "nix/store/store-registration.hh" #include "nix/store/globals.hh" +#include "nix/util/compression.hh" #include "nix/util/topo-sort.hh" namespace nix { MakeError(UploadToHTTP, Error); +void UploadToHTTP::anchor() {} + StringSet HttpBinaryCacheStoreConfig::uriSchemes() { static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; @@ -22,8 +25,12 @@ StringSet HttpBinaryCacheStoreConfig::uriSchemes() return ret; } +void HttpBinaryCacheStoreConfig::anchor() {} + +void HttpBinaryCacheStore::anchor() {} + HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig(ParsedURL _cacheUri, const Params & params) - : StoreConfig(params) + : StoreConfig(params, FilePathType::Unix) , BinaryCacheStoreConfig(params) , cacheUri(std::move(_cacheUri)) { @@ -203,18 +210,18 @@ void HttpBinaryCacheStore::upsertFile( { try { if (auto compressionMethod = getCompressionMethod(path)) { - CompressedSource compressed(source, *compressionMethod); + StringSource compressed(compress(*compressionMethod, source)); /* TODO: Validate that this is a valid content encoding. We probably shouldn't set non-standard values here. */ Headers headers = {{"Content-Encoding", showCompressionAlgo(*compressionMethod)}}; - upload(path, compressed, compressed.size(), mimeType, std::move(headers)); + upload(path, compressed, compressed.s.size(), mimeType, std::move(headers)); } else { upload(path, source, sizeHint, mimeType, std::nullopt); } } catch (FileTransferError & e) { UploadToHTTP err(e.message()); err.addTrace({}, "while uploading to HTTP binary cache at '%s'", config->cacheUri.to_string()); - throw err; + throw std::move(err); } } @@ -255,6 +262,18 @@ FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path) } } + // Propagate per-substituter retry overrides to the transfer request. + // Only set when the user actually specified the URL parameter; otherwise + // the transfer falls back to the global FileTransferSettings. + auto propagate = [](auto & setting, auto & dest) { + if (setting.isOverridden()) + dest = setting.get(); + }; + propagate(config->retryDelayMs, request.retryDelayMs); + propagate(config->retryDelayRateLimitedMs, request.retryDelayRateLimitedMs); + propagate(config->retryMaxDelayMs, request.retryMaxDelayMs); + propagate(config->retryAttempts, request.retryAttempts); + return request; } diff --git a/src/libstore/http-binary-cache-store.md b/src/libstore/http-binary-cache-store.md index 20c26d0c2caf..03dd350ec518 100644 --- a/src/libstore/http-binary-cache-store.md +++ b/src/libstore/http-binary-cache-store.md @@ -2,7 +2,7 @@ R"( **Store URL format**: `http://...`, `https://...` -This store allows a binary cache to be accessed via the HTTP +This store allows a [binary cache](@docroot@/protocols/binary-cache/index.md) to be accessed via the HTTP protocol. )" diff --git a/src/libstore/include/nix/store/active-builds.hh b/src/libstore/include/nix/store/active-builds.hh index 2dc914f35606..a56b33f5b5ff 100644 --- a/src/libstore/include/nix/store/active-builds.hh +++ b/src/libstore/include/nix/store/active-builds.hh @@ -93,6 +93,8 @@ struct TrackActiveBuildsStore virtual BuildHandle buildStarted(const ActiveBuild & build) = 0; virtual void buildFinished(const BuildHandle & handle) = 0; + + virtual void anchor(); }; struct QueryActiveBuildsStore @@ -102,6 +104,8 @@ struct QueryActiveBuildsStore virtual ~QueryActiveBuildsStore() = default; virtual std::vector queryActiveBuilds() = 0; + + virtual void anchor(); }; } // namespace nix diff --git a/src/libstore/include/nix/store/async-path-writer.hh b/src/libstore/include/nix/store/async-path-writer.hh index 695321ccb1f3..49d1ecbc69fd 100644 --- a/src/libstore/include/nix/store/async-path-writer.hh +++ b/src/libstore/include/nix/store/async-path-writer.hh @@ -6,6 +6,8 @@ namespace nix { struct AsyncPathWriter { + virtual void anchor(); + virtual ~AsyncPathWriter() = default; virtual StorePath addPath( diff --git a/src/libstore/include/nix/store/aws-creds.hh b/src/libstore/include/nix/store/aws-creds.hh index 0751757cb015..3fa747069935 100644 --- a/src/libstore/include/nix/store/aws-creds.hh +++ b/src/libstore/include/nix/store/aws-creds.hh @@ -36,6 +36,10 @@ struct AwsCredentials class AwsAuthError final : public CloneableError { +private: + void anchor() override; + +public: std::optional errorCode; public: @@ -70,7 +74,7 @@ public: } } - virtual ~AwsCredentialProvider() {} + virtual ~AwsCredentialProvider(); }; /** diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 1f600d8fbc8f..1544c4bcec82 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -16,7 +16,14 @@ class RemoteFSAccessor; struct BinaryCacheStoreConfig : virtual StoreConfig { - using StoreConfig::StoreConfig; +private: + void anchor() override; + +public: + BinaryCacheStoreConfig(const Params & params) + : StoreConfig(params, FilePathType::Unix) + { + } Setting compression{ this, @@ -39,8 +46,8 @@ struct BinaryCacheStoreConfig : virtual StoreConfig fetch debug info on demand )"}; - Setting secretKeyFile{ - this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; + Setting> secretKeyFile{ + this, std::nullopt, "secret-key", "Path to the secret key used to sign the binary cache."}; Setting secretKeyFiles{ this, "", "secret-keys", "List of comma-separated paths to the secret keys used to sign the binary cache."}; @@ -55,7 +62,11 @@ struct BinaryCacheStoreConfig : virtual StoreConfig this, false, "parallel-compression", - "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + R"( + Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`. + + If not set explicitly, defaults to `true` when `compression` is `zstd` and `false` otherwise. + )"}; Setting compressionLevel{ this, @@ -85,14 +96,26 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ Config & config; private: + void anchor() override; + std::vector> signers; protected: /** * The prefix under which realisation infos will be stored + * + * @note The previous (still experimental, though) hash-keyed + * realisations were under "realisations". "build trace" is a better + * name anyways (issue #11895). This is call "v2" accordingly. + * + * While we're experimenting, we'll freely increase this version + * number. Old build traces will just be "abandoned" at the old URL. + * When we are done experimenting, we'll try lean more on versioning + * the build trace entries themselves than the entire directory, for + * a smoother migration path. */ - constexpr const static std::string realisationsPrefix = "realisations"; + constexpr const static std::string realisationsPrefix = "build-trace-v2"; constexpr const static std::string cacheInfoFile = "nix-cache-info"; @@ -111,7 +134,7 @@ protected: /** * Compute the path to the given realisation * - * It's `${realisationsPrefix}/${drvOutput}.doi`. + * It's `${realisationsPrefix}/${drvPath}/${outputName}`. */ std::string makeRealisationPath(const DrvOutput & id); diff --git a/src/libstore/include/nix/store/build-result.hh b/src/libstore/include/nix/store/build-result.hh index ef2d5ecc817c..c3bdb2b09973 100644 --- a/src/libstore/include/nix/store/build-result.hh +++ b/src/libstore/include/nix/store/build-result.hh @@ -61,8 +61,11 @@ enum struct BuildResultFailureStatus : uint8_t { * This is both an exception type (inherits from Error) and serves as * the failure variant in BuildResult::inner. */ -struct BuildError : public CloneableError +class BuildError : public CloneableError { + void anchor() override; + +public: using Status = BuildResultFailureStatus; using enum Status; diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index 3b5b8504966a..2562e29da743 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -16,12 +16,27 @@ namespace nix { +/** + * Rethrow the current exception as a subclass of `Error`. + */ +void rethrowExceptionAsError(); + +/** + * Send the current exception to the parent in the format expected by + * `DerivationBuilderImpl::processSandboxSetupMessages()`. + */ +void handleChildException(bool sendException); + /** * Denotes a build failure that stemmed from the builder exiting with a * failing exist status. */ struct BuilderFailureError final : CloneableError { +private: + void anchor() override; + +public: int builderStatus; std::string extraMsgAfter; @@ -48,6 +63,9 @@ struct ChrootPath bool optional = false; }; +void to_json(nlohmann::json & j, const ChrootPath & cp); +void from_json(const nlohmann::json & j, ChrootPath & cp); + typedef std::map PathsInChroot; // maps target path to source path /** @@ -116,7 +134,7 @@ struct DerivationBuilderParams */ struct DerivationBuilderCallbacks { - virtual ~DerivationBuilderCallbacks() = default; + virtual ~DerivationBuilderCallbacks(); /** * Open a log file and a pipe to it. @@ -147,6 +165,10 @@ struct DerivationBuilderCallbacks */ struct DerivationBuilder : RestrictionContext { +private: + void anchor() override; + +public: DerivationBuilder() = default; virtual ~DerivationBuilder() = default; @@ -222,7 +244,7 @@ using DerivationBuilderUnique = std::unique_ptr miscMethods, DerivationBuilderParams params); + LocalStore & store, std::shared_ptr miscMethods, DerivationBuilderParams params); /** * @param handler Must be chosen such that it supports the given @@ -230,7 +252,7 @@ DerivationBuilderUnique makeDerivationBuilder( */ DerivationBuilderUnique makeExternalDerivationBuilder( LocalStore & store, - std::unique_ptr miscMethods, + std::shared_ptr miscMethods, DerivationBuilderParams params, const ExternalBuilder & handler); #endif diff --git a/src/libstore/include/nix/store/build/derivation-building-goal.hh b/src/libstore/include/nix/store/build/derivation-building-goal.hh index 615b5f04ea38..4d366f648c08 100644 --- a/src/libstore/include/nix/store/build/derivation-building-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-building-goal.hh @@ -31,6 +31,8 @@ typedef enum { rpAccept, rpDecline, rpPostpone } HookReply; */ struct DerivationBuildingGoal : public Goal { + friend class Worker; + /** * @param storeDerivation Whether to store the derivation in * `worker.store`. This is useful for newly-resolved derivations. In this @@ -41,7 +43,11 @@ struct DerivationBuildingGoal : public Goal * faithfully reconstruct the build history. */ DerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode, bool storeDerivation); + const StorePath & drvPath, + ref drv, + Worker & worker, + BuildMode buildMode, + bool storeDerivation); ~DerivationBuildingGoal(); private: @@ -52,7 +58,7 @@ private: /** * The derivation stored at drvPath. */ - const std::unique_ptr drv; + const ref drv; /** * The remainder is state held during the build. diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index 2b68fa1782a4..8d6892839c76 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -45,7 +45,6 @@ struct InitialOutputStatus struct InitialOutput { - Hash outputHash; std::optional known; }; diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index aaded75511f0..94aba5c3160d 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -45,7 +45,7 @@ struct DerivationGoal : public Goal */ DerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, Worker & worker, BuildMode buildMode, @@ -64,9 +64,7 @@ private: /** * The derivation stored at drvPath. */ - std::unique_ptr drv; - - const Hash outputHash; + ref drv; const BuildMode buildMode; diff --git a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh index b79e6bbb79e4..972558e9706f 100644 --- a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh @@ -35,7 +35,10 @@ struct BuilderFailureError; */ struct DerivationResolutionGoal : public Goal { - DerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode); + friend class Worker; + + DerivationResolutionGoal( + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode); /** * If the derivation needed to be resolved, this is resulting @@ -53,7 +56,7 @@ private: /** * The derivation stored at drvPath. */ - std::unique_ptr drv; + ref drv; /** * The remainder is state held during the build. diff --git a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh index 5f36bbf06d8b..6652da6a20c8 100644 --- a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh +++ b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh @@ -20,11 +20,12 @@ class Worker; * If the output store object itself should also be substituted, that is * the responsibility of the caller to do so. * - * @todo rename this `BuidlTraceEntryGoal`, which will make sense + * @todo rename this `BuildTraceEntryGoal`, which will make sense * especially once `Realisation` is renamed to `BuildTraceEntry`. */ class DrvOutputSubstitutionGoal : public Goal { + friend class Worker; /** * The drv output we're trying to substitute diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 7706ad20b4d3..121d2b058d63 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -10,8 +10,11 @@ namespace nix { -struct TimedOut final : CloneableError +class TimedOut final : public CloneableError { + void anchor() override; + +public: time_t maxDuration; TimedOut(time_t maxDuration); @@ -75,11 +78,59 @@ enum struct JobCategory { struct Goal : public std::enable_shared_from_this { private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); +public: + /** + * Event types for child process communication, delivered via coroutines. + */ + struct ChildOutput + { + Descriptor fd; + std::string data; + }; + + struct ChildEOF + { + Descriptor fd; + }; + + using ChildEvent = std::variant>; + +private: + class ChildEvents + { + /** + * Structured queue of child events: + * - outputs: stream of data from child + * - eof: optional end-of-stream marker + * - timeout: optional timeout that flushes/overrides other events + */ + std::queue childOutputs; + std::optional childEOF; + std::unique_ptr childTimeout; + + public: + void pushChildEvent(ChildOutput event); + void pushChildEvent(ChildEOF event); + void pushChildEvent(TimedOut event); + bool hasChildEvent() const; + ChildEvent popChildEvent(); + }; + /** * Goals that this goal is waiting for. */ Goals waitees; + /** + * Memoised result of key(). + */ + std::optional cachedKey; + + ChildEvents childEvents; + public: typedef enum { ecBusy, ecSuccess, ecFailed, ecNoSubstituters } ExitCode; @@ -147,22 +198,6 @@ public: friend Goal; }; - /** - * Event types for child process communication, delivered via coroutines. - */ - struct ChildOutput - { - Descriptor fd; - std::string data; - }; - - struct ChildEOF - { - Descriptor fd; - }; - - using ChildEvent = std::variant; - /** * Tag type for `co_await`-ing child events. * Returns a `ChildEvent` when resumed. @@ -228,8 +263,10 @@ public: explicit Co(handle_type handle) : handle(handle) {}; - void operator=(Co &&); - Co(Co && rhs); + Co & operator=(Co &&) noexcept; + Co(Co && rhs) noexcept; + Co & operator=(const Co &) = delete; + Co(const Co & rhs) = delete; ~Co(); bool await_ready() @@ -253,6 +290,12 @@ public: void await_resume() {}; }; + template + struct AsyncCallback + { + fun)> fn; + }; + /** * Used on initial suspend, does the same as `std::suspend_always`, * but asserts that everything has been set correctly. @@ -308,28 +351,6 @@ public: */ bool alive = true; - class - { - /** - * Structured queue of child events: - * - outputs: stream of data from child - * - eof: optional end-of-stream marker - * - timeout: optional timeout that flushes/overrides other events - */ - std::queue childOutputs; - std::optional childEOF; - std::optional childTimeout; - - public: - - void pushChildEvent(ChildOutput event); - void pushChildEvent(ChildEOF event); - void pushChildEvent(TimedOut event); - bool hasChildEvent() const; - ChildEvent popChildEvent(); - - } childEvents; - /** * The awaiter used by @ref final_suspend. */ @@ -423,6 +444,9 @@ public: return static_cast(co); } + template + auto await_transform(AsyncCallback && acb); + /** * Awaiter for @ref Suspend. Always suspends, but asserts * there are no pending child events (those should be @@ -434,7 +458,7 @@ public: bool await_ready() { - assert(!promise.childEvents.hasChildEvent()); + assert(!promise.goal->childEvents.hasChildEvent()); return false; } @@ -462,7 +486,7 @@ public: bool await_ready() { - return handle && handle.promise().childEvents.hasChildEvent(); + return handle && handle.promise().goal->childEvents.hasChildEvent(); } void await_suspend(handle_type h) @@ -473,7 +497,7 @@ public: ChildEvent await_resume() { assert(handle); - return handle.promise().childEvents.popChildEvent(); + return handle.promise().goal->childEvents.popChildEvent(); } }; @@ -591,6 +615,17 @@ public: */ virtual std::string key() = 0; + /** + * Memoising variant of key(). We really don't want to pay the overhead of + * allocating strings just to compare Goals. + */ + std::string_view keyCached() & + { + if (cachedKey) + return *cachedKey; + return *(cachedKey = key()); + } + /** * @brief Hint for the scheduler, which concurrency limit applies. * @see JobCategory @@ -600,7 +635,19 @@ public: protected: Co await(Goals waitees); + /** + * Awaiting on the resulting coroutine yields the goal for several seconds. + * Used for retrying goals blocked on acquiring lockfiles. + */ Co waitForAWhile(); + + /** + * Awaiting on the resulting coroutine yields the goal until it is + * explicitly woken up via Worker::wakeUp. Wakeup can be queued from another + * thread via Worker::Waker. + */ + Co waitUntilWoken(); + Co waitForBuildSlot(); Co yield(); }; diff --git a/src/libstore/include/nix/store/build/substitution-goal.hh b/src/libstore/include/nix/store/build/substitution-goal.hh index 22213c33dc60..ab09ef4a7e95 100644 --- a/src/libstore/include/nix/store/build/substitution-goal.hh +++ b/src/libstore/include/nix/store/build/substitution-goal.hh @@ -4,7 +4,6 @@ #include "nix/store/build/worker.hh" #include "nix/store/store-api.hh" #include "nix/store/build/goal.hh" -#include "nix/util/muxable-pipe.hh" #include #include #include @@ -28,11 +27,6 @@ struct PathSubstitutionGoal : public Goal */ RepairFlag repair; - /** - * Pipe for the substituter's standard output. - */ - MuxablePipe outPipe; - /** * The substituter thread. */ diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 4b00965e95a7..cd84551d001c 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -12,6 +12,7 @@ #include #include #include +#include namespace nix { @@ -91,6 +92,11 @@ private: */ WeakGoals wantingToBuild; + /** + * Goals waiting for a substitution slot. + */ + WeakGoals wantingToSubstitute; + /** * Child processes currently running. */ @@ -121,14 +127,14 @@ private: std::map> drvOutputSubstitutionGoals; /** - * Goals waiting for busy paths to be unlocked. + * Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAnyGoal; + WeakGoals waitingForAWhile; /** - * Goals sleeping for a few seconds (polling a lock). + * Goals awaiting completion callbacks. */ - WeakGoals waitingForAWhile; + WeakGoals waitingForCompletion; /** * Last time the goals in `waitingForAWhile` were woken up. @@ -140,6 +146,42 @@ private: */ std::map pathContentsGoodCache; + class Waker + { +#ifndef _WIN32 + /** + * Wakeup pipe polled alongside all other goal FDs. Gets written to by + * enqueue(). Not needed on Windows. + */ + unix::SelfPipe wakeupPipe; +#else + Descriptor ioport; +#endif + /** + * Queue of goals that need to be woken up. + */ + Sync> wakeupQueue_; + + friend class Worker; + + void wakeAll(Worker & worker); + + Waker() + { +#ifndef _WIN32 + wakeupPipe.create(); +#endif + } + + public: + void enqueue(WeakGoalPtr goal); + }; + + /** + * This is behind a ref, so that other threads can take a weak_ptr to it. + */ + ref wakerState; + public: const Activity act; @@ -214,7 +256,7 @@ public: std::shared_ptr makeDerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, BuildMode buildMode, bool storeDerivation); @@ -223,13 +265,13 @@ public: * @ref DerivationResolutionGoal "derivation resolution goal" */ std::shared_ptr - makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode); + makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode); /** * @ref DerivationBuildingGoal "derivation building goal" */ std::shared_ptr makeDerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, BuildMode buildMode, bool storeDerivation); + const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation); /** * @ref PathSubstitutionGoal "substitution goal" @@ -259,6 +301,12 @@ public: */ void wakeUp(GoalPtr goal); + /** + * Get a weak reference to the goal waker. It can be used to safely enqueue Goals + * for wakeup from other threads. + */ + std::weak_ptr getCrossThreadWaker(); + /** * Return the number of local build processes currently running (but not * remote builds via the build hook). @@ -281,15 +329,13 @@ public: bool respectTimeouts); /** - * Unregisters a running child process. `wakeSleepers` should be - * false if there is no sense in waking up goals that are sleeping - * because they can't run yet (e.g., there is no free build slot, - * or the hook would still say `postpone`). + * Unregisters a running child process. Wakes at most a single goal that is + * awaiting on the corresponding build slot type (building or substitution). * * This overload requires `goal` to point to a fully constructed, * valid goal object, as it calls `goal->jobCategory()`. */ - void childTerminated(Goal * goal, bool wakeSleepers = true); + void childTerminated(Goal * goal); /** * Unregisters a running child process, like the other overload. @@ -298,7 +344,7 @@ public: * weak goal references, so it is safe to call from destructors * where the goal object may be partially destroyed. */ - void childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers = true); + void childTerminated(Goal * goal, JobCategory jobCategory); /** * Put `goal` to sleep until a build slot becomes available (which @@ -306,12 +352,6 @@ public: */ void waitForBuildSlot(GoalPtr goal); - /** - * Wait for any goal to finish. Pretty indiscriminate way to - * wait for some resource that some other goal is holding. - */ - void waitForAnyGoal(GoalPtr goal); - /** * Wait for a few seconds and then retry this goal. Used when * waiting for a lock held by another process. This kind of @@ -320,6 +360,11 @@ public: */ void waitForAWhile(GoalPtr goal); + /** + * Wait until explicitly resumed by Waker::enqueue. + */ + void waitForCompletion(GoalPtr goal); + /** * Loop until the specified top-level goals have finished. */ diff --git a/src/libstore/include/nix/store/builtins/buildenv.hh b/src/libstore/include/nix/store/builtins/buildenv.hh index a0f0b3f24b99..b528871b2801 100644 --- a/src/libstore/include/nix/store/builtins/buildenv.hh +++ b/src/libstore/include/nix/store/builtins/buildenv.hh @@ -25,6 +25,9 @@ struct Package class BuildEnvFileConflictError final : public CloneableError { +private: + void anchor() override; + public: const std::filesystem::path fileA; const std::filesystem::path fileB; diff --git a/src/libstore/include/nix/store/common-protocol-impl.hh b/src/libstore/include/nix/store/common-protocol-impl.hh index cb1020a3c83b..d0eebe0bc23b 100644 --- a/src/libstore/include/nix/store/common-protocol-impl.hh +++ b/src/libstore/include/nix/store/common-protocol-impl.hh @@ -26,12 +26,13 @@ namespace nix { LengthPrefixedProtoHelper::write(store, conn, t); \ } -#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) -COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::map) +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, std::map) #undef COMMA_ /* protocol-specific templates */ diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index 877e3754a3fc..375e1d0338d5 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -90,8 +90,9 @@ DECLARE_COMMON_SERIALISER(std::set); template DECLARE_COMMON_SERIALISER(std::tuple); -template -DECLARE_COMMON_SERIALISER(std::map); +template +DECLARE_COMMON_SERIALISER(std::map); +#undef COMMA_ /** * These use the empty string for the null case, relying on the fact diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index 42b3415b777a..b622e82e26bf 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -10,12 +10,19 @@ class SSHMaster; struct CommonSSHStoreConfig : virtual StoreConfig { - using StoreConfig::StoreConfig; +private: + void anchor() override; + +public: + CommonSSHStoreConfig(const Params & params) + : StoreConfig(params, FilePathType::Unix) + { + } CommonSSHStoreConfig(const ParsedURL::Authority & authority, const Params & params); - Setting sshKey{ - this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; + Setting> sshKey{ + this, std::nullopt, "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; Setting sshPublicHostKey{ this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index 4e6ec22f9ccd..e29f660c4848 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -2,7 +2,6 @@ ///@file #include -#include #include #include @@ -85,12 +84,12 @@ struct DerivationOptions * Either one set of checks for all outputs, or separate checks * per-output. */ - std::variant> outputChecks = OutputChecks{}; + std::variant>> outputChecks = OutputChecks{}; /** * Whether to avoid scanning for references for a given output. */ - std::map unsafeDiscardReferences; + std::map> unsafeDiscardReferences; /** * In non-structured mode, all bindings specified in the derivation @@ -123,7 +122,7 @@ struct DerivationOptions * attributes give to the builder. The set of paths in the original JSON * is replaced with a list of `PathInfo` in JSON format. */ - std::map> exportReferencesGraph; + std::map, std::less<>> exportReferencesGraph; /** * env: __sandboxProfile diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 88bfb6e49be6..56912981b4f7 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -293,7 +293,7 @@ struct BasicDerivation BasicDerivation(const BasicDerivation &) = default; BasicDerivation & operator=(BasicDerivation &&) = default; BasicDerivation & operator=(const BasicDerivation &) = default; - virtual ~BasicDerivation() {}; + virtual ~BasicDerivation(); bool isBuiltin() const; @@ -428,6 +428,11 @@ struct Derivation : BasicDerivation void fillInOutputPaths(Store & store); Derivation() = default; + Derivation(Derivation &&) = default; + Derivation(const Derivation &) = default; + Derivation & operator=(Derivation &&) = default; + Derivation & operator=(const Derivation &) = default; + ~Derivation() override; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) @@ -508,34 +513,39 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName); * derivations (fixed-output or not) will have a different hash for each * output. */ -struct DrvHash +struct DrvHashModulo { /** - * Map from output names to hashes + * Single hash for the derivation + * + * This is for an input-addressed derivation that doesn't + * transitively depend on any floating-CA derivations. */ - std::map hashes; + using DrvHash = Hash; - enum struct Kind : bool { - /** - * Statically determined derivations. - * This hash will be directly used to compute the output paths - */ - Regular, - - /** - * Floating-output derivations (and their reverse dependencies). - */ - Deferred, - }; + /** + * Known CA drv's output hashes, for fixed-output derivations whose + * output hashes are always known since they are fixed up-front. + */ + using CaOutputHashes = std::map; /** - * The kind of derivation this is, simplified for just "derivation hash - * modulo" purposes. + * This derivation doesn't yet have known output hashes. + * + * Either because itself is floating CA, or it (transtively) depends + * on a floating CA derivation. */ - Kind kind; -}; + using DeferredDrv = std::monostate; + + using Raw = std::variant; + + Raw raw; -void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; + bool operator==(const DrvHashModulo &) const = default; + // auto operator <=> (const DrvHashModulo &) const = default; + + MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo); +}; /** * Returns hashes with the details of fixed-output subderivations @@ -561,15 +571,17 @@ void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; * ATerm, after subderivations have been likewise expunged from that * derivation. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /** - * Return a map associating each output to a hash that uniquely identifies its - * derivation (modulo the self-references). + * If a derivation is input addressed and doesn't yet have its input + * addressed (is deferred) try using `hashDerivationModulo`. * - * \todo What is the Hash in this map? + * Does nothing if not deferred input-addressed, or + * `hashDerivationModulo` indicates it is missing inputs' output paths + * and is not yet ready (and must stay deferred). */ -std::map staticOutputHashes(Store & store, const Derivation & drv); +void resolveInputAddressed(Store & store, Derivation & drv); struct DrvHashFct { @@ -584,7 +596,7 @@ struct DrvHashFct /** * Memoisation of hashDerivationModulo(). */ -typedef boost::concurrent_flat_map DrvHashes; +typedef boost::concurrent_flat_map DrvHashes; // FIXME: global, though at least thread-safe. extern DrvHashes drvHashes; @@ -614,5 +626,5 @@ constexpr unsigned expectedJsonVersionDerivation = 4; } // namespace nix JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput) -JSON_IMPL_WITH_XP_FEATURES(nix::Derivation) JSON_IMPL_WITH_XP_FEATURES(nix::BasicDerivation) +JSON_IMPL_WITH_XP_FEATURES(nix::Derivation) diff --git a/src/libstore/include/nix/store/derived-path-map.hh b/src/libstore/include/nix/store/derived-path-map.hh index c10af84ca26c..65be5df3c4fb 100644 --- a/src/libstore/include/nix/store/derived-path-map.hh +++ b/src/libstore/include/nix/store/derived-path-map.hh @@ -93,6 +93,19 @@ struct DerivedPathMap * `ChildNode::value`. */ ChildNode * findSlot(const SingleDerivedPath & k); + + /** + * Find the node for `k` and invoke @ref callback on it, pruning empty + * ancestors afterwards. + * + * @param callback Invoked on the found node. Should return true if + * the node's value is still non-empty (i.e. the node should be kept). + * If it returns false and the node has no children, the node is erased + * and empty ancestors are pruned recursively. + * + * No-op if the node does not exist. + */ + void removeSlot(const SingleDerivedPath & k, fun callback); }; template<> diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index ac7ab9c680fd..bec77a6bee68 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -15,6 +15,10 @@ struct MemorySourceAccessor; */ struct DummyStore : virtual Store { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; ref config; @@ -49,7 +53,7 @@ struct DummyStore : virtual Store * outer map for the derivation, and inner maps for the outputs of a * given derivation. */ - boost::concurrent_flat_map> buildTrace; + boost::concurrent_flat_map> buildTrace; DummyStore(ref config) : Store{*config} diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index 59d1b1fddb22..c8a212c75603 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -12,8 +12,12 @@ struct DummyStore; struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig { +private: + void anchor() override; + +public: DummyStoreConfig(const Params & params) - : StoreConfig(params) + : StoreConfig(params, FilePathType::Unix) { // Disable caching since this a temporary in-memory store. pathInfoCacheSize = 0; diff --git a/src/libstore/include/nix/store/filetransfer-impl.hh b/src/libstore/include/nix/store/filetransfer-impl.hh new file mode 100644 index 000000000000..bac68baf55bd --- /dev/null +++ b/src/libstore/include/nix/store/filetransfer-impl.hh @@ -0,0 +1,73 @@ +#pragma once +/** + * @file + * + * Implementation details of filetransfer.cc exposed for unit testing. + * Not part of the public libstore API. + */ + +#include +#include +#include +#include +#include +#include + +namespace nix { + +/** + * Clamped exponential growth: base * 2^(attempt-1), capped at ceil. + * Shift is clamped at 31 and the intermediate is widened to uint64_t + * so the shift cannot overflow uint32_t. + */ +constexpr uint32_t clampedExponential(uint32_t base, uint32_t attempt, uint32_t ceil) +{ + auto shift = std::min(attempt == 0 ? 0u : attempt - 1, 31u); + uint64_t unclamped = static_cast(base) << shift; + return static_cast(std::min(unclamped, ceil)); +} + +/** + * Saturating conversion: chrono duration → uint32_t milliseconds. + * Negative inputs clamp to 0; values > UINT32_MAX clamp to UINT32_MAX. + */ +constexpr uint32_t saturateMs(std::chrono::milliseconds d) noexcept +{ + auto c = d.count(); + if (c <= 0) + return 0; + return static_cast(std::min(c, std::numeric_limits::max())); +} + +/** + * Parameters for computeRetryDelayMs. + */ +struct RetryDelayParams +{ + /** 1-based retry attempt number (1 = first retry). */ + uint32_t attempt; + /** Base delay in ms for this error class. */ + uint32_t baseMs; + /** Per-attempt delay ceiling (does not cap retryAfterMs). */ + uint32_t ceilMs; + /** Server-provided minimum delay (from Retry-After header). */ + std::optional retryAfterMs = {}; + /** Apply full jitter (false = deterministic). */ + bool jitter = true; +}; + +/** + * Compute the delay before the next retry attempt. + * + * Uses exponential backoff with optional full jitter. When a server-provided + * Retry-After is present, jitter spreads *above* it so that concurrent + * clients don't all retry at the same instant: + * sleep = random(floor, floor + backoff) + * where floor = retryAfter (or 0) and backoff = min(ceilMs, base * 2^(attempt-1)). + * ceilMs caps the backoff growth, not the server-provided floor. + * + * @param rng random number generator (unused if p.jitter is false) + */ +std::chrono::milliseconds computeRetryDelayMs(const RetryDelayParams & p, std::mt19937 & rng); + +} // namespace nix diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 76309dff72fa..606c075f3e0d 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include +#include #include #include @@ -21,16 +23,29 @@ namespace nix { const std::filesystem::path & nixConfDir(); -struct FileTransferSettings : Config +class FileTransferSettings : public Config { -private: - static std::filesystem::path getDefaultSSLCertFile(); + static std::optional getDefaultSSLCertFile(); + + void anchor() override; public: FileTransferSettings(); Setting enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; + Setting enableHttp3{ + this, + false, + "http3", + R"( + Whether to try enabling HTTP/3 (QUIC). + When enabled, Nix requests HTTP/3 and transparently falls back + to HTTP/2 or HTTP/1.1 for servers that do not support it. + This option has no effect unless the `nix` binary is linked + against a libcurl built with HTTP/3 (QUIC) support. + )"}; + Setting userAgentSuffix{ this, "", "user-agent-suffix", "String appended to the user agent in HTTP requests."}; @@ -73,8 +88,63 @@ public: timeout's duration. )"}; - Setting tries{ - this, 5, "download-attempts", "The number of times Nix attempts to download a file before giving up."}; + Setting tries{ + this, + 5, + "filetransfer-retry-attempts", + R"( + The number of times Nix attempts a file transfer (download + or upload) before giving up. Retries apply to transient + failures: connection-level errors, HTTP 408, 429, and most + 5xx responses. Authentication failures (401/403/407), + 404/410, and other 4xx responses are not retried. + )", + {"download-attempts"}}; + + Setting retryDelayMs{ + this, + 100, + "filetransfer-retry-delay", + R"( + Initial delay in milliseconds before retrying a failed file transfer + (download or upload). The delay doubles with each subsequent attempt + (exponential backoff) and is subject to random jitter (see + `filetransfer-retry-jitter`). + )"}; + + Setting retryDelayRateLimitedMs{ + this, + 5000, + "filetransfer-retry-delay-rate-limited", + R"( + Initial delay in milliseconds before retrying a file transfer that + failed with a rate-limit response (HTTP 429 or 503). The delay doubles + with each subsequent attempt. + + Servers may send a `Retry-After` header specifying a longer delay; + when present, Nix respects the larger of the two values. + )"}; + + Setting retryMaxDelayMs{ + this, + 60000, + "filetransfer-retry-max-delay", + R"( + Ceiling on the exponential backoff delay in milliseconds. This does not + cap server-provided `Retry-After` values, which are honored as-is. + )"}; + + Setting retryJitter{ + this, + true, + "filetransfer-retry-jitter", + R"( + Whether to apply random jitter to retry delays. When enabled, each + retry waits for a random duration between 0 and the computed delay + ("full jitter"), which spreads out retry storms from many clients. + + Disable for deterministic retry timing (primarily useful for tests). + )"}; Setting downloadBufferSize{ this, @@ -147,8 +217,6 @@ public: extern FileTransferSettings fileTransferSettings; -extern const unsigned int RETRY_TIME_MS_DEFAULT; - /** * HTTP methods supported by FileTransfer. */ @@ -182,10 +250,19 @@ struct FileTransferRequest Headers headers; std::string expectedETag; HttpMethod method = HttpMethod::Get; - unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT; ActivityId parentAct; bool decompress = true; + /** + * Per-request retry overrides. When set, these take precedence over the + * global `FileTransferSettings`. Typically populated from a store's URL + * parameters (e.g. `s3://bucket?retry-attempts=8`). + */ + std::optional retryDelayMs; + std::optional retryDelayRateLimitedMs; + std::optional retryMaxDelayMs; + std::optional retryAttempts; + /** * Optional path to the client certificate in "PEM" format. Only used for TLS-based protocols. */ @@ -243,6 +320,14 @@ struct FileTransferRequest { } + /** + * `uri` with any userinfo (`user:password@`) stripped, for use in + * progress, warning and error messages so credentials embedded in + * the URL don't leak into logs. Returns `uri` verbatim if it can't + * be parsed. + */ + std::string displayUri() const; + /** * Returns the method description for logging purposes. */ @@ -346,7 +431,7 @@ public: } }; - virtual ~FileTransfer() {} + virtual ~FileTransfer(); /** * Enqueue a data transfer request, returning a future to the result of @@ -385,7 +470,7 @@ public: void download(FileTransferRequest && request, Sink & sink, std::function resultCallback = {}); - enum Error { NotFound, Unauthorized, Forbidden, Misc, Transient, Interrupted }; + enum Error { NotFound, Unauthorized, Forbidden, Misc, Transient }; }; /** @@ -407,6 +492,9 @@ std::shared_ptr resetFileTransfer(); class FileTransferError final : public CloneableError { +private: + void anchor() override; + public: FileTransfer::Error error; /// intentionally optional diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 661d0f28b03a..1e4e620eedca 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -43,6 +43,19 @@ struct GCOptions using GCAction = nix::GCAction; using enum GCAction; + struct WholeStore + {}; + + struct SpecificPaths + { + StorePathSet paths; + + /** + * Allow dead referrers of candidate paths to also be deleted. + */ + bool deleteReferrers = false; + }; + GCAction action{gcDeleteDead}; /** @@ -54,9 +67,10 @@ struct GCOptions bool ignoreLiveness{false}; /** - * For `gcDeleteSpecific`, the paths to delete. + * The paths from which to delete. */ - StorePathSet pathsToDelete; + using GCPaths = std::variant; + GCPaths pathsToDelete; /** * Stop after at least `maxFreed` bytes have been freed. @@ -112,6 +126,10 @@ struct GCResults */ struct GcStore : public virtual Store { +private: + void anchor() override; + +public: inline static std::string operationName = "Garbage collection"; /** diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index c4e1f6eb24c7..7841bc5949f9 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -29,13 +29,8 @@ struct ProfileDirsOptions; struct LogFileSettings : public virtual Config { - /** - * The directory where we log various operations. - */ - std::filesystem::path nixLogDir; - -protected: - LogFileSettings(); +private: + void anchor() override; public: Setting keepLog{ @@ -64,6 +59,10 @@ public: struct NarInfoDiskCacheSettings : public virtual Config { +private: + void anchor() override; + +public: Setting ttlNegative{ this, 3600, @@ -115,6 +114,9 @@ class Settings : public virtual Config, private WorkerSettings, private NarInfoDiskCacheSettings { +private: + void anchor() override; +public: StringSet getDefaultSystemFeatures(); StringSet getDefaultExtraPlatforms(); @@ -184,11 +186,6 @@ public: */ std::filesystem::path nixStateDir; - /** - * File name of the socket the daemon listens to. - */ - std::filesystem::path nixDaemonSocketFile; - Setting storeUri{ this, StoreReference::parse(getEnv("NIX_REMOTE").value_or("auto")), @@ -196,9 +193,14 @@ public: R"( The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for most operations. + See the [Store Types](@docroot@/store/types/index.md) section of the manual for supported store types and settings. + + Can be overridden by the [`NIX_REMOTE`](@docroot@/command-ref/env-common.md#env-NIX_REMOTE) environment variable. + + The default value is [`auto`](@docroot@/store/types/index.md#auto). )"}; Setting useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", "Whether SQLite should use WAL mode."}; diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index 475dc95104ab..12465261caef 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -14,7 +14,15 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this> tlsKey{ this, std::nullopt, "tls-private-key", "Path to an optional TLS client certificate private key in PEM format."}; + Setting retryDelayMs{ + this, + 0, + "retry-delay", + "Override [`filetransfer-retry-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay) for requests to this store (milliseconds)."}; + + Setting retryDelayRateLimitedMs{ + this, + 0, + "retry-delay-rate-limited", + "Override [`filetransfer-retry-delay-rate-limited`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay-rate-limited) for requests to this store (milliseconds)."}; + + Setting retryMaxDelayMs{ + this, + 0, + "retry-max-delay", + "Override [`filetransfer-retry-max-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-max-delay) for requests to this store (milliseconds)."}; + + Setting retryAttempts{ + this, + 0, + "retry-attempts", + "Override [`filetransfer-retry-attempts`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-attempts) for requests to this store."}; + static const std::string name() { return "HTTP Binary Cache Store"; @@ -60,6 +92,8 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, virtual CommonSSHStoreConfig { - using CommonSSHStoreConfig::CommonSSHStoreConfig; +private: + void anchor() override; + +public: + LegacySSHStoreConfig(const Params & params) + : StoreConfig(params, FilePathType::Unix) + , CommonSSHStoreConfig(params) + { + } LegacySSHStoreConfig(const ParsedURL::Authority & authority, const Params & params); @@ -59,6 +67,10 @@ struct LegacySSHStoreConfig : std::enable_shared_from_this struct LegacySSHStore : public virtual Store { +private: + void anchor() override; + +public: using Config = LegacySSHStoreConfig; ref config; diff --git a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh index 035019340f50..e1a80e8dc58e 100644 --- a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh +++ b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh @@ -56,14 +56,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); #define COMMA_ , template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); -#undef COMMA_ template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); -template -#define LENGTH_PREFIXED_PROTO_HELPER_X std::map +template +#define LENGTH_PREFIXED_PROTO_HELPER_X std::map LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X); +#undef COMMA_ template std::vector @@ -109,11 +109,11 @@ void LengthPrefixedProtoHelper>::write( } } -template -std::map -LengthPrefixedProtoHelper>::read(const StoreDirConfig & store, typename Inner::ReadConn conn) +template +std::map LengthPrefixedProtoHelper>::read( + const StoreDirConfig & store, typename Inner::ReadConn conn) { - std::map resMap; + std::map resMap; auto size = readNum(conn.from); while (size--) { auto k = S::read(store, conn); @@ -123,9 +123,9 @@ LengthPrefixedProtoHelper>::read(const StoreDirConfig & st return resMap; } -template -void LengthPrefixedProtoHelper>::write( - const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) +template +void LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index 8c4e68d26398..181b33e4bdf8 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -1,3 +1,6 @@ +#pragma once +/// @file + #include "nix/store/binary-cache-store.hh" namespace nix { @@ -6,7 +9,15 @@ struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this> makeRootDirSetting(LocalFSStoreConfig & self, std::optional defaultValue) { @@ -22,7 +24,10 @@ private: } public: - using StoreConfig::StoreConfig; + LocalFSStoreConfig(const Params & params) + : StoreConfig(params, FilePathType::Native) + { + } /** * Used to override the `root` settings. Can't be done via modifying @@ -35,42 +40,48 @@ public: Setting> rootDir = makeRootDirSetting(*this, std::nullopt); -private: - - /** - * An indirection so that we don't need to refer to global settings - * in headers. - */ - static std::filesystem::path getDefaultStateDir(); - - /** - * An indirection so that we don't need to refer to global settings - * in headers. - */ - static std::filesystem::path getDefaultLogDir(); - -public: - Setting stateDir{ this, - rootDir.get() ? *rootDir.get() / "nix" / "var" / "nix" : getDefaultStateDir(), + rootDir.get() ? *rootDir.get() / "nix" / "var" / "nix" : StoreConfig::getStateDir(), "state", - "Directory where Nix stores state.", + R"( + Directory where Nix stores state. + + Defaults to [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) when [`root`](#@store-slug@-root) is not set. + )", }; Setting logDir{ this, - rootDir.get() ? *rootDir.get() / "nix" / "var" / "log" / "nix" : getDefaultLogDir(), + rootDir.get() ? *rootDir.get() / "nix" / "var" / "log" / "nix" : StoreConfig::getLogDir(), "log", - "directory where Nix stores log files.", + R"( + Directory where Nix stores log files. + + Defaults to [`NIX_LOG_DIR`](@docroot@/command-ref/env-common.md#env-NIX_LOG_DIR) when [`root`](#@store-slug@-root) is not set. + )", }; Setting realStoreDir{ this, rootDir.get() ? *rootDir.get() / "nix" / "store" : std::filesystem::path{storeDir}, "real", - "Physical path of the Nix store.", + R"( + Physical path of the Nix store. + + Defaults to [`store`](#@store-slug@-store) when [`root`](#@store-slug@-root) is not set. + )", }; + + const std::filesystem::path & getStateDir() const override + { + return stateDir.get(); + } + + const std::filesystem::path & getLogDir() const override + { + return logDir.get(); + } }; struct alignas(8) /* Work around ASAN failures on i686-linux. */ @@ -78,6 +89,10 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ virtual GcStore, virtual LogStore { +private: + void anchor() override; + +public: using Config = LocalFSStoreConfig; const Config & config; diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index 5d8ec1b4571a..10b04937c3aa 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -7,13 +7,17 @@ namespace nix { */ struct LocalOverlayStoreConfig : virtual LocalStoreConfig { +private: + void anchor() override; + +public: LocalOverlayStoreConfig(const StringMap & params) : LocalOverlayStoreConfig("", params) { } LocalOverlayStoreConfig(const std::filesystem::path & path, const Params & params) - : StoreConfig(params) + : StoreConfig(params, FilePathType::Native) , LocalFSStoreConfig(path, params) , LocalStoreConfig(path, params) { @@ -33,7 +37,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig const Setting upperLayer{ (StoreConfig *) this, - "", + "/upper-layer-must-be-set", "upper-layer", R"( Directory containing the OverlayFS upper layer for this store's store dir. @@ -53,9 +57,9 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig default, but can be disabled if needed. )"}; - const Setting remountHook{ + const Setting> remountHook{ (StoreConfig *) this, - "", + std::nullopt, "remount-hook", R"( Script or other executable to run when overlay filesystem needs remounting. @@ -119,6 +123,8 @@ struct LocalOverlayStore : virtual LocalStore LocalOverlayStore(ref); private: + void anchor() override; + /** * The store beneath us. * diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index f702293260bd..88198802b66d 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -47,6 +47,10 @@ std::string BaseSetting>::to_string() const; struct GCSettings : public virtual Config { +private: + void anchor() override; + +public: Setting reservedSize{ this, 8 * 1024 * 1024, @@ -69,6 +73,9 @@ struct GCSettings : public virtual Config collector still deletes store paths that are used only at build time (e.g., the C compiler, or source tarballs downloaded from the network). To prevent it from doing so, set this option to `true`. + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-outputs"}, }; @@ -88,6 +95,9 @@ struct GCSettings : public virtual Config store path was built), so by default this option is on. Turn it off to save a bit of disk space (or a lot if `keep-outputs` is also turned on). + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-derivations"}, }; @@ -136,6 +146,10 @@ const uint32_t maxIdsPerBuild = struct AutoAllocateUidSettings : public virtual Config { +private: + void anchor() override; + +public: Setting startId{ this, #ifdef __linux__ @@ -171,6 +185,10 @@ struct AutoAllocateUidSettings : public virtual Config */ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllocateUidSettings { +private: + void anchor() override; + +public: /** * Get the GC settings. */ @@ -198,7 +216,7 @@ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllo 0, "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation. + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#env-vars) of a derivation. The `builder` executable can use this variable to control its own maximum amount of parallelism.