From 87baf29d6aebcda00eb2c46275afca7d3ce958b5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Nov 2025 15:30:38 +0100 Subject: [PATCH 001/555] Git fetcher: Don't compute revCount if it's already specified We don't care if the user (or more likely the lock file) specifies an incorrect value for revCount, since it doesn't matter for security (unlikely content hashes like narHash). --- src/libfetchers/fetchers.cc | 5 ----- src/libfetchers/git.cc | 9 +++++++-- tests/nixos/tarball-flakes.nix | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 7e091ef1071e..0d8bd4514fe4 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -282,11 +282,6 @@ void Input::checkLocks(Input specified, Input & result) if (result.getRev() != prevRev) throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev()); } - - if (auto prevRevCount = specified.getRevCount()) { - if (result.getRevCount() != prevRevCount) - throw Error("'revCount' attribute mismatch in input '%s', expected %d", result.to_string(), *prevRevCount); - } } std::pair, Input> Input::getAccessor(const Settings & settings, Store & store) const diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 75e3f1214814..f4d492308d76 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -891,8 +891,13 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev)); - if (!getShallowAttr(input)) - input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); + if (!getShallowAttr(input)) { + /* Skip revCount computation if it's already supplied by the caller. + We don't care if they specify an incorrect value; it doesn't + matter for security, unlike narHash. */ + if (!input.attrs.contains("revCount")) + input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); + } printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix index 26c20cb1aef4..b0402412d856 100644 --- a/tests/nixos/tarball-flakes.nix +++ b/tests/nixos/tarball-flakes.nix @@ -99,7 +99,6 @@ in # Check that fetching fails if we provide incorrect attributes. machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?rev=493300eb13ae6fb387fbd47bf54a85915acc31c0") - machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?revCount=789") machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?narHash=sha256-tbudgBSg+bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw=") ''; From 8f32f28ebdaf3272d386756724d345883eec760c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Nov 2025 15:39:19 +0100 Subject: [PATCH 002/555] Git fetcher: Don't compute lastModified if it's already specified Same as revCount. --- src/libfetchers/fetchers.cc | 9 --------- src/libfetchers/git.cc | 10 ++++++---- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 0d8bd4514fe4..48a23ad36934 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -269,15 +269,6 @@ void Input::checkLocks(Input specified, Input & result) } } - if (auto prevLastModified = specified.getLastModified()) { - if (result.getLastModified() != prevLastModified) - throw Error( - "'lastModified' attribute mismatch in input '%s', expected %d, got %d", - result.to_string(), - *prevLastModified, - result.getLastModified().value_or(-1)); - } - if (auto prevRev = specified.getRev()) { if (result.getRev() != prevRev) throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev()); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f4d492308d76..96008d45bd28 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -889,12 +889,14 @@ struct GitInputScheme : InputScheme auto rev = *input.getRev(); - input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev)); + /* Skip lastModified computation if it's already supplied by the caller. + We don't care if they specify an incorrect value; it doesn't + matter for security, unlike narHash. */ + if (!input.attrs.contains("lastModified")) + input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev)); if (!getShallowAttr(input)) { - /* Skip revCount computation if it's already supplied by the caller. - We don't care if they specify an incorrect value; it doesn't - matter for security, unlike narHash. */ + /* Like lastModified, skip revCount if supplied by the caller. */ if (!input.attrs.contains("revCount")) input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); } From 4ecc09c43f5f42808134ec8144ac2afdcce0fed2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Nov 2025 19:58:08 +0100 Subject: [PATCH 003/555] Make content-encoding test more reliable --- tests/nixos/content-encoding.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/nixos/content-encoding.nix b/tests/nixos/content-encoding.nix index debee377bdf2..1e188cb060b7 100644 --- a/tests/nixos/content-encoding.nix +++ b/tests/nixos/content-encoding.nix @@ -131,6 +131,7 @@ in start_all() machine.wait_for_unit("nginx.service") + machine.wait_for_open_port(80) # Original test: zstd archive with gzip content-encoding # Make sure that the file is properly compressed as the test would be meaningless otherwise From 0255d696c6658f1cc1bd0aad57e9df495f177dda Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 28 Jan 2026 03:31:31 -0500 Subject: [PATCH 004/555] Kill build hook with SIGTERM instead of SIGKILL This gives custom build hooks the chance to perform any needed cleanup, which is impossible when killed with SIGKILL. After 500ms, if the hook still hasn't exited, it will be forcibly killed with SIGKILL. Resolves #14760 Signed-off-by: Lisanna Dettwyler --- src/libstore/unix/build/hook-instance.cc | 4 ++++ src/libutil/include/nix/util/processes.hh | 4 ++++ src/libutil/unix/processes.cc | 29 +++++++++++++++++++++-- src/nix/build-remote/build-remote.cc | 17 +++++++++++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index af9249487070..65f528aa18dd 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -72,6 +72,10 @@ HookInstance::HookInstance() throw SysError("executing %s", PathFmt(buildHook)); }); + /* Give custom build hooks the chance to cleanup. */ + pid.setKillSignal(SIGTERM); + pid.setKillTimeout(500ms); + pid.setSeparatePG(true); fromHook.writeSide = -1; toHook.readSide = -1; diff --git a/src/libutil/include/nix/util/processes.hh b/src/libutil/include/nix/util/processes.hh index 81e6a2b83cd5..cc33caebfa1e 100644 --- a/src/libutil/include/nix/util/processes.hh +++ b/src/libutil/include/nix/util/processes.hh @@ -18,6 +18,7 @@ #include #include #include +#include namespace nix { @@ -30,6 +31,8 @@ class Pid pid_t pid = -1; bool separatePG = false; int killSignal = SIGKILL; + std::chrono::milliseconds killTimeout; + std::thread killThread; #else AutoCloseFD pid = INVALID_DESCRIPTOR; #endif @@ -55,6 +58,7 @@ public: #ifndef _WIN32 void setSeparatePG(bool separatePG); void setKillSignal(int signal); + void setKillTimeout(std::chrono::milliseconds duration); pid_t release(); #endif diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 2393da55d4a9..930020c23e11 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -12,7 +12,8 @@ #include #include #include -#include +#include +using namespace std::chrono_literals; #include #include @@ -75,6 +76,20 @@ int Pid::kill(bool allowInterrupts) debug("killing process %1%", pid); + std::atomic killed = false; + + if (killTimeout > 0ms && killSignal != SIGKILL) + killThread = std::thread([&]() { + auto elapsed = 0ms; + while (elapsed < killTimeout) { + std::this_thread::sleep_for(25ms); + elapsed += 25ms; + if (killed) + return; + } + ::kill(separatePG ? -pid : pid, SIGKILL); + }); + /* Send the requested signal to the child. If it has its own process group, send the signal to every process in the child process group (which hopefully includes *all* its children). */ @@ -88,7 +103,12 @@ int Pid::kill(bool allowInterrupts) logError(SysError("killing process %d", pid).info()); } - return wait(allowInterrupts); + int ret = wait(allowInterrupts); + if (killThread.joinable()) { + killed = true; + killThread.join(); + } + return ret; } int Pid::wait(bool allowInterrupts) @@ -118,6 +138,11 @@ void Pid::setKillSignal(int signal) this->killSignal = signal; } +void Pid::setKillTimeout(std::chrono::milliseconds duration) +{ + this->killTimeout = duration; +} + pid_t Pid::release() { pid_t p = pid; diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index eec0a0f28410..3333678026cb 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -53,6 +53,23 @@ static bool allSupportedLocally(Store & store, const StringSet & requiredFeature static int main_build_remote(int argc, char ** argv) { { + /* Upon exiting, Nix will attempt to terminate this process with + SIGTERM. initNix will block or handle SIGTERM, so we need to unblock + and unhandle it here. + */ + struct sigaction act; + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = SIG_DFL; + if (sigaction(SIGTERM, &act, 0)) + throw SysError("resetting SIGTERM"); + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGTERM); + if (pthread_sigmask(SIG_UNBLOCK, &set, nullptr)) + throw SysError("unblocking SIGTERM"); + logger = makeJSONLogger(getStandardError()); /* Ensure we don't get any SSH passphrase or host key popups. */ From f25ec8344bddbeeea4a59e45e3522bd57e1bcd22 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Fri, 30 Jan 2026 11:52:41 -0500 Subject: [PATCH 005/555] Display shutdown message on interrupt Signed-off-by: Lisanna Dettwyler --- src/libmain/progress-bar.cc | 7 +++++++ src/libstore/unix/build/hook-instance.cc | 2 ++ src/libutil/unix/signals.cc | 17 +++++++++++++---- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index a973102f9509..71c7789de254 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,6 +1,7 @@ #include "nix/main/progress-bar.hh" #include "nix/util/terminal.hh" #include "nix/util/sync.hh" +#include "nix/util/signals.hh" #include "nix/store/store-api.hh" #include "nix/store/names.hh" @@ -94,10 +95,16 @@ class ProgressBar : public Logger bool printBuildLogs = false; bool isTTY; + std::unique_ptr interruptCallback; + public: ProgressBar(bool isTTY) : isTTY(isTTY) + , interruptCallback(createInterruptCallback([&]() { + pause(); + redraw("\rshutting down\e[K"); + })) { state_.lock()->active = isTTY; updateThread = std::thread([&]() { diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 65f528aa18dd..364c1c4ff36c 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -6,6 +6,8 @@ #include "nix/util/strings.hh" #include "nix/util/executable-path.hh" +using namespace std::chrono_literals; + namespace nix { HookInstance::HookInstance() diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index deff40276453..d073c9e46c44 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -42,7 +42,16 @@ struct InterruptCallbacks std::map> callbacks; }; -static Sync _interruptCallbacks; +/* Required to avoid static initialization order fiasco. This allows global + objects to safely register callbacks. */ +static Sync & getInterruptCallbacks() +{ + /* Intentionally leak, according to the Construct On First Use Idiom. + An alternative is to use the Nifty Counter Idiom, but + InterruptCallbacks' destructor is not very important. */ + static Sync * _interruptCallbacks = new Sync(); + return *_interruptCallbacks; +} static void signalHandlerThread(sigset_t set) { @@ -68,7 +77,7 @@ void unix::triggerInterrupt() while (true) { std::function callback; { - auto interruptCallbacks(_interruptCallbacks.lock()); + auto interruptCallbacks(getInterruptCallbacks().lock()); auto lb = interruptCallbacks->callbacks.lower_bound(i); if (lb == interruptCallbacks->callbacks.end()) break; @@ -153,14 +162,14 @@ struct InterruptCallbackImpl : InterruptCallback ~InterruptCallbackImpl() override { - auto interruptCallbacks(_interruptCallbacks.lock()); + auto interruptCallbacks(getInterruptCallbacks().lock()); interruptCallbacks->callbacks.erase(token); } }; std::unique_ptr createInterruptCallback(std::function callback) { - auto interruptCallbacks(_interruptCallbacks.lock()); + auto interruptCallbacks(getInterruptCallbacks().lock()); auto token = interruptCallbacks->nextToken++; interruptCallbacks->callbacks.emplace(token, callback); return std::make_unique(token); From 9868310d6f2d6a2225eaf9d48c94b88b67cc9ac5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 6 Feb 2026 14:58:27 +0100 Subject: [PATCH 006/555] Add test for builtins.getFlake --- tests/functional/flakes/common.sh | 2 ++ tests/functional/flakes/get-flake.sh | 20 ++++++++++++++++++++ tests/functional/flakes/meson.build | 1 + 3 files changed, 23 insertions(+) create mode 100644 tests/functional/flakes/get-flake.sh diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 2dcf2e0fd541..6fef78925592 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -32,6 +32,8 @@ writeSimpleFlake() { baseName = builtins.baseNameOf ./.; root = ./.; + + number = 123; }; } EOF diff --git a/tests/functional/flakes/get-flake.sh b/tests/functional/flakes/get-flake.sh new file mode 100644 index 000000000000..85ef8c23d1a4 --- /dev/null +++ b/tests/functional/flakes/get-flake.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +source ./common.sh + +createFlake1 + +mkdir -p "$flake1Dir/subflake" +cat > "$flake1Dir/subflake/flake.nix" < Date: Fri, 6 Feb 2026 15:48:47 +0100 Subject: [PATCH 007/555] builtins.getFlake: Support path values This allows doing `builtins.getFlake ./subflake` instead of ugly hacks. --- src/libflake/flake-primops.cc | 50 +++++++++++++------------ src/libflake/flake.cc | 24 +++++++++--- src/libflake/include/nix/flake/flake.hh | 7 ++++ tests/functional/flakes/get-flake.sh | 6 +++ 4 files changed, 58 insertions(+), 29 deletions(-) diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index a43777574f79..6ff66a964e91 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -35,35 +35,39 @@ namespace nix::flake::primops { PrimOp getFlake(const Settings & settings) { auto prim_getFlake = [&settings](EvalState & state, const PosIdx pos, Value ** args, Value & v) { - std::string flakeRefS( - state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); - auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); - if (state.settings.pureEval && !flakeRef.input.isLocked(state.fetchSettings)) - throw Error( - "cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", - flakeRefS, - state.positions[pos]); - - callFlake( - state, - lockFlake( - settings, - state, - flakeRef, - LockFlags{ - .updateLockFile = false, - .writeLockFile = false, - .useRegistries = !state.settings.pureEval && settings.useRegistries, - .allowUnlocked = !state.settings.pureEval, - }), - v); + state.forceValue(*args[0], pos); + + LockFlags lockFlags{ + .updateLockFile = false, + .writeLockFile = false, + .useRegistries = !state.settings.pureEval && settings.useRegistries, + .allowUnlocked = !state.settings.pureEval, + }; + + if (args[0]->type() == nPath) { + auto path = state.realisePath(pos, *args[0]); + callFlake(state, lockFlake(settings, state, path, lockFlags), v); + } else { + NixStringContext context; + std::string flakeRefS( + state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); + + auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); + if (state.settings.pureEval && !flakeRef.input.isLocked(state.fetchSettings)) + throw Error( + "cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", + flakeRefS, + state.positions[pos]); + + callFlake(state, lockFlake(settings, state, flakeRef, lockFlags), v); + } }; return PrimOp{ .name = "__getFlake", .args = {"args"}, .doc = R"( - Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: + Fetch a flake from a flake reference or a path, and return its output attributes and some metadata. For example: ```nix (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 9dab20c5c10e..13999a5c30f6 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -416,10 +416,8 @@ static LockFile readLockFile(const fetchers::Settings & fetchSettings, const Sou : LockFile(); } -/* Compute an in-memory lock file for the specified top-level flake, - and optionally write it to file, if the flake is writable. */ -LockedFlake -lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) +LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake) { experimentalFeatureSettings.require(Xp::Flakes); @@ -427,8 +425,6 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; - auto flake = getFlake(state, topRef, useRegistriesTop, {}); - if (lockFlags.applyNixConfig) { flake.config.apply(settings); state.store->setOptions(); @@ -908,6 +904,22 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, } } +LockedFlake +lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) +{ + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; + return lockFlake(settings, state, topRef, lockFlags, getFlake(state, topRef, useRegistriesTop, {})); +} + +LockedFlake +lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags) +{ + /* We need a fake flakeref to put in the `Flake` struct, but it's not used for anything. */ + auto fakeRef = parseFlakeRef(state.fetchSettings, "flake:get-flake"); + return lockFlake(settings, state, fakeRef, lockFlags, readFlake(state, fakeRef, fakeRef, fakeRef, flakeDir, {})); +} + static ref makeInternalFS() { auto internalFS = make_ref(MemorySourceAccessor{}); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 3bee6556f643..fd52dbebac5d 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -214,9 +214,16 @@ struct LockFlags std::set inputUpdates; }; +/* + * Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is + * writable. + */ LockedFlake lockFlake(const Settings & settings, EvalState & state, const FlakeRef & flakeRef, const LockFlags & lockFlags); +LockedFlake +lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags); + void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & v); /** diff --git a/tests/functional/flakes/get-flake.sh b/tests/functional/flakes/get-flake.sh index 85ef8c23d1a4..e462607db8f7 100644 --- a/tests/functional/flakes/get-flake.sh +++ b/tests/functional/flakes/get-flake.sh @@ -9,12 +9,18 @@ cat > "$flake1Dir/subflake/flake.nix" < Date: Wed, 18 Feb 2026 22:50:37 +0100 Subject: [PATCH 008/555] Add release note --- doc/manual/rl-next/getflake-path.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/manual/rl-next/getflake-path.md diff --git a/doc/manual/rl-next/getflake-path.md b/doc/manual/rl-next/getflake-path.md new file mode 100644 index 000000000000..2360fe7693e0 --- /dev/null +++ b/doc/manual/rl-next/getflake-path.md @@ -0,0 +1,6 @@ +--- +synopsis: "`builtins.getFlake` now supports path values" +prs: [15290] +--- + +`builtins.getFlake` now accepts path values in addition to flakerefs, allowing you to write `builtins.getFlake ./subflake` instead of having to use ugly workarounds to construct a pure flakeref. From 6cae299bd972892549ddaee722dc210514d481bc Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Tue, 24 Feb 2026 16:45:55 -0500 Subject: [PATCH 009/555] Error on invalid URL param in github fetcher Resolves #15304 Signed-off-by: Lisanna Dettwyler --- .../github-fetcher-param-validation.md | 7 ++++ src/libfetchers-tests/input.cc | 18 ++++++++++ src/libfetchers/github.cc | 36 +++++++------------ 3 files changed, 38 insertions(+), 23 deletions(-) create mode 100644 doc/manual/rl-next/github-fetcher-param-validation.md diff --git a/doc/manual/rl-next/github-fetcher-param-validation.md b/doc/manual/rl-next/github-fetcher-param-validation.md new file mode 100644 index 000000000000..2fb430ae4b24 --- /dev/null +++ b/doc/manual/rl-next/github-fetcher-param-validation.md @@ -0,0 +1,7 @@ +--- +synopsis: GitHub fetcher now validates URL parameters +prs: [15331] +issues: [15304] +--- + +The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. diff --git a/src/libfetchers-tests/input.cc b/src/libfetchers-tests/input.cc index faff55f2c2d0..170ff4485c7b 100644 --- a/src/libfetchers-tests/input.cc +++ b/src/libfetchers-tests/input.cc @@ -1,8 +1,12 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/attrs.hh" #include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/util/tests/gmock-matchers.hh" +#include "nix/util/url.hh" #include +#include #include @@ -58,4 +62,18 @@ INSTANTIATE_TEST_SUITE_P( }), [](const ::testing::TestParamInfo & info) { return info.param.description; }); +namespace fetchers { + +class GitHubInputTest : public ::testing::Test +{}; + +TEST_F(GitHubInputTest, throwOnInvalidURLParam) +{ + EXPECT_THAT( + []() { Input::fromURL(fetchers::Settings{}, "github:a/b?tag=foo"); }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("tag"))); +} + +} // namespace fetchers + } // namespace nix diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index b86fa926a668..ac8c02418dab 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -41,16 +41,14 @@ struct GitArchiveInputScheme : InputScheme /* This ignores empty path segments for back-compat. Older versions used a tokenizeString here. */ auto path = url.pathSegments(/*skipEmpty=*/true) | std::ranges::to>(); - std::optional rev; - std::optional ref; - std::optional host_url; + Attrs attrs; auto size = path.size(); if (size == 3) { if (std::regex_match(path[2], revRegex)) - rev = path[2]; + attrs.insert_or_assign("rev", path[2]); else - ref = path[2]; + attrs.insert_or_assign("ref", path[2]); } else if (size > 3) { std::string rs; for (auto i = std::next(path.begin(), 2); i != path.end(); i++) { @@ -59,38 +57,30 @@ struct GitArchiveInputScheme : InputScheme rs += "/"; } } - ref = rs; + attrs.insert_or_assign("ref", rs); } else if (size < 2) throw BadURL("URL '%s' is invalid", url); for (auto & [name, value] : url.query) { if (name == "rev") { - if (rev) + if (attrs.contains(name)) throw BadURL("URL '%s' contains multiple commit hashes", url); - rev = value; + attrs.insert_or_assign("rev", value); } else if (name == "ref") { - if (ref) + if (attrs.contains(name)) throw BadURL("URL '%s' contains multiple branch/tag names", url); - ref = value; + attrs.insert_or_assign("ref", value); } else if (name == "host") - host_url = value; - // FIXME: barf on unsupported attributes + attrs.insert_or_assign("host", value); + else if (name == "narHash") + attrs.insert_or_assign("narHash", value); + else + throw BadURL("URL '%s' contains unknown parameter '%s'", url, name); } - Attrs attrs; attrs.insert_or_assign("type", std::string{schemeName()}); attrs.insert_or_assign("owner", path[0]); attrs.insert_or_assign("repo", path[1]); - if (rev) - attrs.insert_or_assign("rev", *rev); - if (ref) - attrs.insert_or_assign("ref", *ref); - if (host_url) - attrs.insert_or_assign("host", *host_url); - - auto narHash = url.query.find("narHash"); - if (narHash != url.query.end()) - attrs.insert_or_assign("narHash", narHash->second); return inputFromAttrs(settings, attrs); } From 26c1c8fb4a34e2669027c02cb9e480b2c0f53c80 Mon Sep 17 00:00:00 2001 From: eveeifyeve <88671402+Eveeifyeve@users.noreply.github.com> Date: Sun, 1 Mar 2026 05:52:48 +1100 Subject: [PATCH 010/555] feat(nix): add shortname to bundle bundler arg --- src/nix/bundle.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 5bca8e0cb31f..a9266a11bde4 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -18,6 +18,7 @@ struct CmdBundle : InstallableValueCommand { addFlag({ .longName = "bundler", + .shortName = 'B', .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, From 30b6bba0feb1aa5b37ec0937c647a794a271538a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 26 Feb 2026 14:37:27 -0500 Subject: [PATCH 011/555] Convert `parseURLRelative` tests to parameters test suite This will allow extending it further in the next commit --- src/libutil-tests/url.cc | 490 +++++++++++++++++++++++---------------- 1 file changed, 291 insertions(+), 199 deletions(-) diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index efc322feb85a..1197adaa8c87 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -168,7 +168,11 @@ INSTANTIATE_TEST_SUITE_P( .expected = ParsedURL{ .scheme = "http", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {}, .fragment = "", @@ -179,7 +183,11 @@ INSTANTIATE_TEST_SUITE_P( .expected = ParsedURL{ .scheme = "https", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {}, .fragment = "", @@ -190,7 +198,11 @@ INSTANTIATE_TEST_SUITE_P( .expected = ParsedURL{ .scheme = "https", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {{"download", "fast"}, {"when", "now"}}, .fragment = "hello", @@ -201,7 +213,11 @@ INSTANTIATE_TEST_SUITE_P( .expected = ParsedURL{ .scheme = "file+https", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "video.mp4"}, .query = (StringMap) {}, .fragment = "", @@ -212,7 +228,12 @@ INSTANTIATE_TEST_SUITE_P( .expected = ParsedURL{ .scheme = "http", - .authority = Authority{.hostType = HostType::IPv4, .host = "127.0.0.1", .port = 8080}, + .authority = + Authority{ + .hostType = HostType::IPv4, + .host = "127.0.0.1", + .port = 8080, + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {{"download", "fast"}, {"when", "now"}}, .fragment = "hello", @@ -225,7 +246,10 @@ INSTANTIATE_TEST_SUITE_P( .scheme = "http", .authority = Authority{ - .hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080}, + .hostType = HostType::IPv6, + .host = "fe80::818c:da4d:8975:415c\%enp0s25", + .port = 8080, + }, .path = {""}, .query = (StringMap) {}, .fragment = "", @@ -277,7 +301,11 @@ TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment) ParsedURL expected{ .scheme = "http", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {{"field", "value"}}, .fragment = "?foo=bar#", @@ -484,7 +512,11 @@ TEST(parseURL, parsesHttpUrlWithEmptyPort) ParsedURL expected{ .scheme = "http", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {{"foo", "bar"}}, .fragment = "", @@ -498,203 +530,263 @@ TEST(parseURL, parsesHttpUrlWithEmptyPort) * parseURLRelative * --------------------------------------------------------------------------*/ -TEST(parseURLRelative, resolvesRelativePath) +struct ParseURLRelativeParam { - ParsedURL base = parseURL("http://example.org/dir/page.html"); - auto parsed = parseURLRelative("subdir/file.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"}, - .path = {"", "dir", "subdir", "file.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, baseUrlIpv6AddressWithoutZoneId) -{ - ParsedURL base = parseURL("http://[fe80::818c:da4d:8975:415c]/dir/page.html"); - auto parsed = parseURLRelative("subdir/file.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c"}, - .path = {"", "dir", "subdir", "file.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, resolvesRelativePathIpv6AddressWithZoneId) -{ - ParsedURL base = parseURL("http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080/dir/page.html"); - auto parsed = parseURLRelative("subdir/file2.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080}, - .path = {"", "dir", "subdir", "file2.txt"}, - .query = {}, - .fragment = "", - }; - - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, resolvesRelativePathWithDot) -{ - ParsedURL base = parseURL("http://example.org/dir/page.html"); - auto parsed = parseURLRelative("./subdir/file.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"}, - .path = {"", "dir", "subdir", "file.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, resolvesParentDirectory) -{ - ParsedURL base = parseURL("http://example.org:234/dir/page.html"); - auto parsed = parseURLRelative("../up.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org", .port = 234}, - .path = {"", "up.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, resolvesParentDirectoryNotTrickedByEscapedSlash) -{ - ParsedURL base = parseURL("http://example.org:234/dir\%2Ffirst-trick/another-dir\%2Fsecond-trick/page.html"); - auto parsed = parseURLRelative("../up.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org", .port = 234}, - .path = {"", "dir/first-trick", "up.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, replacesPathWithAbsoluteRelative) -{ - ParsedURL base = parseURL("http://example.org/dir/page.html"); - auto parsed = parseURLRelative("/rooted.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"}, - .path = {"", "rooted.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, keepsQueryAndFragmentFromRelative) -{ - // But discard query params on base URL - ParsedURL base = parseURL("https://www.example.org/path/index.html?z=3"); - auto parsed = parseURLRelative("other.html?x=1&y=2#frag", base); - ParsedURL expected{ - .scheme = "https", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = {"", "path", "other.html"}, - .query = {{"x", "1"}, {"y", "2"}}, - .fragment = "frag", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, absOverride) -{ - ParsedURL base = parseURL("http://example.org/path/page.html"); - std::string_view abs = "https://127.0.0.1.org/secure"; - auto parsed = parseURLRelative(abs, base); - auto parsedAbs = parseURL(abs); - ASSERT_EQ(parsed, parsedAbs); -} - -TEST(parseURLRelative, absOverrideWithZoneId) -{ - ParsedURL base = parseURL("http://example.org/path/page.html"); - std::string_view abs = "https://[fe80::818c:da4d:8975:415c\%25enp0s25]/secure?foo=bar"; - auto parsed = parseURLRelative(abs, base); - auto parsedAbs = parseURL(abs); - ASSERT_EQ(parsed, parsedAbs); -} - -TEST(parseURLRelative, bothWithoutAuthority) -{ - ParsedURL base = parseURL("mailto:mail-base@bar.baz?bcc=alice@asdf.com"); - std::string_view over = "mailto:mail-override@foo.bar?subject=url-testing"; - auto parsed = parseURLRelative(over, base); - auto parsedOverride = parseURL(over); - ASSERT_EQ(parsed, parsedOverride); -} - -TEST(parseURLRelative, emptyRelative) -{ - ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); - auto parsed = parseURLRelative("", base); - ParsedURL expected{ - .scheme = "https", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = {"", "path", "index.html"}, - .query = {{"a b", "5 6"}, {"x y", "34"}}, - .fragment = "", - }; - EXPECT_EQ(base.fragment, "frag"); - EXPECT_EQ(parsed, expected); -} + std::string_view base; + std::string_view relative; + ParsedURL expected; + std::string description; +}; -TEST(parseURLRelative, fragmentRelative) -{ - ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); - auto parsed = parseURLRelative("#frag2", base); - ParsedURL expected{ - .scheme = "https", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = {"", "path", "index.html"}, - .query = {{"a b", "5 6"}, {"x y", "34"}}, - .fragment = "frag2", - }; - EXPECT_EQ(parsed, expected); -} +class ParseURLRelativeTestSuite : public ::testing::TestWithParam +{}; -TEST(parseURLRelative, queryRelative) +TEST_P(ParseURLRelativeTestSuite, resolve) { - ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); - auto parsed = parseURLRelative("?asdf\%20qwer=1\%202\%203", base); - ParsedURL expected{ - .scheme = "https", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = {"", "path", "index.html"}, - .query = {{"asdf qwer", "1 2 3"}}, - .fragment = "", - }; - EXPECT_EQ(parsed, expected); + auto & p = GetParam(); + auto base = parseURL(p.base); + auto parsed = parseURLRelative(p.relative, base); + EXPECT_EQ(parsed, p.expected); } -TEST(parseURLRelative, queryFragmentRelative) -{ - ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); - auto parsed = parseURLRelative("?asdf\%20qwer=1\%202\%203#frag2", base); - ParsedURL expected{ - .scheme = "https", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = {"", "path", "index.html"}, - .query = {{"asdf qwer", "1 2 3"}}, - .fragment = "frag2", - }; - EXPECT_EQ(parsed, expected); -} +INSTANTIATE_TEST_SUITE_P( + parseURLRelative, + ParseURLRelativeTestSuite, + ::testing::Values( + ParseURLRelativeParam{ + .base = "http://example.org/dir/page.html", + .relative = "subdir/file.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "example.org", + }, + .path = {"", "dir", "subdir", "file.txt"}, + }, + .description = "resolvesRelativePath", + }, + ParseURLRelativeParam{ + .base = "http://[fe80::818c:da4d:8975:415c]/dir/page.html", + .relative = "subdir/file.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::IPv6, + .host = "fe80::818c:da4d:8975:415c", + }, + .path = {"", "dir", "subdir", "file.txt"}, + }, + .description = "baseUrlIpv6AddressWithoutZoneId", + }, + ParseURLRelativeParam{ + .base = "http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080/dir/page.html", + .relative = "subdir/file2.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::IPv6, + .host = "fe80::818c:da4d:8975:415c%enp0s25", + .port = 8080, + }, + .path = {"", "dir", "subdir", "file2.txt"}, + }, + .description = "resolvesRelativePathIpv6AddressWithZoneId", + }, + ParseURLRelativeParam{ + .base = "http://example.org/dir/page.html", + .relative = "./subdir/file.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "example.org", + }, + .path = {"", "dir", "subdir", "file.txt"}, + }, + .description = "resolvesRelativePathWithDot", + }, + ParseURLRelativeParam{ + .base = "http://example.org:234/dir/page.html", + .relative = "../up.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "example.org", + .port = 234, + }, + .path = {"", "up.txt"}, + }, + .description = "resolvesParentDirectory", + }, + ParseURLRelativeParam{ + .base = "http://example.org:234/dir\%2Ffirst-trick/another-dir\%2Fsecond-trick/page.html", + .relative = "../up.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "example.org", + .port = 234, + }, + .path = {"", "dir/first-trick", "up.txt"}, + }, + .description = "resolvesParentDirectoryNotTrickedByEscapedSlash", + }, + ParseURLRelativeParam{ + .base = "http://example.org/dir/page.html", + .relative = "/rooted.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "example.org", + }, + .path = {"", "rooted.txt"}, + }, + .description = "replacesPathWithAbsoluteRelative", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?z=3", + .relative = "other.html?x=1&y=2#frag", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "other.html"}, + .query = {{"x", "1"}, {"y", "2"}}, + .fragment = "frag", + }, + .description = "keepsQueryAndFragmentFromRelative", + }, + ParseURLRelativeParam{ + .base = "http://example.org/path/page.html", + .relative = "https://127.0.0.1.org/secure", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "127.0.0.1.org", + }, + .path = {"", "secure"}, + }, + .description = "absOverride", + }, + ParseURLRelativeParam{ + .base = "http://example.org/path/page.html", + .relative = "https://[fe80::818c:da4d:8975:415c\%25enp0s25]/secure?foo=bar", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::IPv6, + .host = "fe80::818c:da4d:8975:415c%enp0s25", + }, + .path = {"", "secure"}, + .query = {{"foo", "bar"}}, + }, + .description = "absOverrideWithZoneId", + }, + ParseURLRelativeParam{ + .base = "mailto:mail-base@bar.baz?bcc=alice@asdf.com", + .relative = "mailto:mail-override@foo.bar?subject=url-testing", + .expected = + ParsedURL{ + .scheme = "mailto", + .path = {"mail-override@foo.bar"}, + .query = {{"subject", "url-testing"}}, + }, + .description = "bothWithoutAuthority", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {{"a b", "5 6"}, {"x y", "34"}}, + }, + .description = "emptyRelative", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "#frag2", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {{"a b", "5 6"}, {"x y", "34"}}, + .fragment = "frag2", + }, + .description = "fragmentRelative", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "?asdf\%20qwer=1\%202\%203", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {{"asdf qwer", "1 2 3"}}, + }, + .description = "queryRelative", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "?asdf\%20qwer=1\%202\%203#frag2", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {{"asdf qwer", "1 2 3"}}, + .fragment = "frag2", + }, + .description = "queryFragmentRelative", + }), + [](const auto & info) { return info.param.description; }); /* ---------------------------------------------------------------------------- * decodeQuery From e19c0a5a14d8eecd5ca2529fe1028406a9b4838d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 26 Feb 2026 14:45:18 -0500 Subject: [PATCH 012/555] Add test cases for relative `?` and `#` empty explicit params vs none at all Unlike with absolute (complete) URLs, with relative URLs, this does make a difference -- explicit empty overrides, implicit empty does not. --- src/libutil-tests/url.cc | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index 1197adaa8c87..a6d464507c9a 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -753,6 +753,39 @@ INSTANTIATE_TEST_SUITE_P( }, .description = "fragmentRelative", }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "#", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {{"a b", "5 6"}, {"x y", "34"}}, + .fragment = "", + }, + .description = "emptyFragmentRelative", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "?", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {}, + }, + .description = "emptyQueryRelative", + }, ParseURLRelativeParam{ .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", .relative = "?asdf\%20qwer=1\%202\%203", From 62d275d7c0709817c7d7f21dfffbe528f9828bef Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 27 Feb 2026 13:38:45 -0500 Subject: [PATCH 013/555] tests: use pathToUrlPath for file:// URL construction in git test --- src/libfetchers-tests/git.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libfetchers-tests/git.cc b/src/libfetchers-tests/git.cc index abc3dd74c5c6..9fb9a2ce1cd6 100644 --- a/src/libfetchers-tests/git.cc +++ b/src/libfetchers-tests/git.cc @@ -4,6 +4,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/git-utils.hh" +#include "nix/util/url.hh" #include #include @@ -190,7 +191,7 @@ TEST_F(GitTest, submodulePeriodSupport) auto input = fetchers::Input::fromAttrs( settings, { - {"url", "file://" + repoPath.string()}, + {"url", "file://" + encodeUrlPath(pathToUrlPath(repoPath))}, {"submodules", Explicit{true}}, {"type", "git"}, {"ref", "main"}, From b26f2ca3e68bfc670ab3ffddc7e81ab41dd1ea62 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 27 Feb 2026 14:50:00 -0500 Subject: [PATCH 014/555] tests: clear NIX_STORE env in fetchers, expr, and flake test environments --- src/libexpr-tests/meson.build | 2 ++ src/libfetchers-tests/meson.build | 2 ++ src/libflake-tests/meson.build | 1 + 3 files changed, 5 insertions(+) diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index c5b72851da53..0b0a01c20654 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -84,6 +84,8 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index 6bccdb05c9a1..ba9774e956b9 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -66,6 +66,8 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 59094abe8661..3512be10bce9 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -62,6 +62,7 @@ test( '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', 'NIX_CONFIG' : 'extra-experimental-features = flakes', 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) From f3792cdad5c6d8e415b5d125c456acb64c026562 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 27 Feb 2026 14:50:00 -0500 Subject: [PATCH 015/555] flakeref: use portable root detection in directory walk loops --- src/libflake/flakeref.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index c35040adb5d8..2960a158e37b 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -146,7 +146,7 @@ std::pair parsePathFlakeRefWithFragment( // Save device to detect filesystem boundary dev_t device = lstat(path).st_dev; bool found = false; - while (path != "/") { + while (path.parent_path() != path) { if (pathExists(path / "flake.nix")) { found = true; break; @@ -171,7 +171,7 @@ std::pair parsePathFlakeRefWithFragment( auto flakeRoot = path; std::string subdir; - while (flakeRoot != "/") { + while (flakeRoot.parent_path() != flakeRoot) { if (pathExists(flakeRoot / ".git")) { auto parsedURL = ParsedURL{ .scheme = "git+file", From 9c59f628905ae4c3c6c3d919f1aa15cd66a6e4d2 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 27 Feb 2026 14:50:00 -0500 Subject: [PATCH 016/555] url: handle Windows drive letters in fixGitURL --- src/libutil/url.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 184eba263f08..abf0acf00883 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -408,15 +408,22 @@ ParsedURL fixGitURL(std::string url) url = std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); if (!hasPrefix(url, "file:") && !hasPrefix(url, "git+file:") && url.find("://") == std::string::npos) { auto path = splitString>(url, "/"); +#ifdef _WIN32 + // Windows drive letters (e.g., "C:") look like SCP hosts but are local paths + bool hasDriveLetter = !path.empty() && path[0].size() == 2 + && std::isalpha(static_cast(path[0][0])) && path[0][1] == ':'; +#else + constexpr bool hasDriveLetter = false; +#endif // Reject SCP-like URLs without user (e.g., "github.com:path") - colon in first component - if (!path.empty() && path[0].find(':') != std::string::npos) + if (!path.empty() && path[0].find(':') != std::string::npos && !hasDriveLetter) throw BadURL("SCP-like URL '%s' is not supported; use SSH URL syntax instead (ssh://...)", url); // Absolute paths get an empty authority (file:///path), relative paths get none (file:path) - if (hasPrefix(url, "/")) + if (hasPrefix(url, "/") || hasDriveLetter) return ParsedURL{ .scheme = "file", .authority = ParsedURL::Authority{}, - .path = path, + .path = hasDriveLetter ? pathToUrlPath(std::filesystem::path(url)) : path, }; else return ParsedURL{ From 0d8ca7a8886c3ff0f8ea2f4eff56e33e2f947d3a Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 27 Feb 2026 14:50:00 -0500 Subject: [PATCH 017/555] fetchers: use pathToUrlPath in PathInputScheme::toURL --- src/libfetchers/path.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index cf7f6aa920dd..9d846c675367 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -95,7 +95,7 @@ struct PathInputScheme : InputScheme query.erase("__final"); return ParsedURL{ .scheme = "path", - .path = splitString>(getStrAttr(input.attrs, "path"), "/"), + .path = pathToUrlPath(std::filesystem::path{getStrAttr(input.attrs, "path")}), .query = query, }; } From 450c1850c9cf3ab509fdc71608cdea7e1f8d6014 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 2 Mar 2026 11:13:03 -0500 Subject: [PATCH 018/555] libutil: Add `finalSymlink` parameter to openDirectory Add a type-safe boolean enum `FinalSymlink` to control whether `openDirectory` follows symlinks on the final path component. - `FinalSymlink::Follow` (default): follow symlinks (current behavior) - `FinalSymlink::DontFollow`: fail if the path is a symlink On Unix, this uses `O_NOFOLLOW`. On Windows, this uses `FILE_FLAG_OPEN_REPARSE_POINT`. Update all call sites with explicit `FinalSymlink` values. --- src/libstore/gc.cc | 3 ++- src/libstore/local-store.cc | 2 +- src/libstore/unix/build/derivation-builder.cc | 2 +- src/libutil-tests/file-system-at.cc | 10 +++++----- src/libutil-tests/source-accessor.cc | 2 +- src/libutil-tests/unix/file-system-at.cc | 8 ++++---- src/libutil/file-system.cc | 4 ++-- src/libutil/include/nix/util/file-system.hh | 9 ++++++++- src/libutil/unix/file-system.cc | 5 +++-- src/libutil/windows/file-system.cc | 4 ++-- 10 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 16b81abf2821..a88fe6a64acb 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -548,7 +548,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) by another process. We need to be sure that we can acquire an exclusive lock before deleting them. */ if (baseName.find("tmp-", 0) == 0) { - auto tmpDirFd = openDirectory(realPath); + /* TODO Reconsider whether Follow is the right choice, here */ + auto tmpDirFd = openDirectory(realPath, FinalSymlink::Follow); if (!tmpDirFd || !lockFile(tmpDirFd.get(), ltWrite, false)) { debug("skipping locked tempdir %s", PathFmt(realPath)); return; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8631df218418..209ee2fc18b8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1285,7 +1285,7 @@ std::pair LocalStore::createTempDirInStore() the GC between createTempDir() and when we acquire a lock on it. We'll repeat until 'tmpDir' exists and we've locked it. */ tmpDirFn = createTempDir(std::filesystem::path{config->realStoreDir.get()}, "tmp"); - tmpDirFd = openDirectory(tmpDirFn); + tmpDirFd = openDirectory(tmpDirFn, FinalSymlink::DontFollow); if (!tmpDirFd) { continue; } diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index da225dc0d1bd..4996affbebd6 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -764,7 +764,7 @@ std::optional DerivationBuilderImpl::startBuild() /* The TOCTOU between the previous mkdir call and this open call is unavoidable due to POSIX semantics.*/ - tmpDirFd = AutoCloseFD{open(tmpDir.c_str(), O_RDONLY | O_NOFOLLOW | O_DIRECTORY)}; + tmpDirFd = openDirectory(tmpDir, FinalSymlink::DontFollow); if (!tmpDirFd) throw SysError("failed to open the build temporary directory descriptor %1%", PathFmt(tmpDir)); diff --git a/src/libutil-tests/file-system-at.cc b/src/libutil-tests/file-system-at.cc index 5c6f93c35429..8c75ec14e39a 100644 --- a/src/libutil-tests/file-system-at.cc +++ b/src/libutil-tests/file-system-at.cc @@ -32,7 +32,7 @@ TEST(readLinkAt, works) { RestoreSink sink(/*startFsync=*/false); sink.dstPath = tmpDir; - sink.dirFd = openDirectory(tmpDir); + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); sink.createSymlink(CanonPath("link"), "target"); sink.createSymlink(CanonPath("relative"), "../relative/path"); sink.createSymlink(CanonPath("absolute"), "/absolute/path"); @@ -45,7 +45,7 @@ TEST(readLinkAt, works) sink.createDirectory(CanonPath("dir")); } - auto dirFd = openDirectory(tmpDir); + auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("link")), OS_STR("target")); EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("relative")), OS_STR("../relative/path")); @@ -54,7 +54,7 @@ TEST(readLinkAt, works) EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("long")), string_to_os_string(longTarget)); EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("a/b/link")), OS_STR("nested_target")); - auto subDirFd = openDirectory(tmpDir / "a"); + auto subDirFd = openDirectory(tmpDir / "a", FinalSymlink::Follow); EXPECT_EQ(readLinkAt(subDirFd.get(), CanonPath("b/link")), OS_STR("nested_target")); // Test error cases - expect SystemError on both platforms @@ -78,7 +78,7 @@ TEST(openFileEnsureBeneathNoSymlinks, works) { RestoreSink sink(/*startFsync=*/false); sink.dstPath = tmpDir; - sink.dirFd = openDirectory(tmpDir); + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); sink.createDirectory(CanonPath("a")); sink.createDirectory(CanonPath("c")); sink.createDirectory(CanonPath("c/d")); @@ -107,7 +107,7 @@ TEST(openFileEnsureBeneathNoSymlinks, works) SymlinkNotAllowed); } - auto dirFd = openDirectory(tmpDir); + auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); // Helper to open files with platform-specific arguments auto openRead = [&](std::string_view path) -> AutoCloseFD { diff --git a/src/libutil-tests/source-accessor.cc b/src/libutil-tests/source-accessor.cc index b9da508659bb..fd4e2429e11a 100644 --- a/src/libutil-tests/source-accessor.cc +++ b/src/libutil-tests/source-accessor.cc @@ -95,7 +95,7 @@ TEST_F(FSSourceAccessorTest, works) RestoreSink sink(false); sink.dstPath = tmpDir; #ifndef _WIN32 - sink.dirFd = openDirectory(tmpDir); + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); #endif sink.createDirectory(CanonPath("subdir")); sink.createRegularFile(CanonPath("file1"), [](CreateRegularFileSink & crf) { crf("content1"); }); diff --git a/src/libutil-tests/unix/file-system-at.cc b/src/libutil-tests/unix/file-system-at.cc index 609c4e2aa1d1..6ae2dcce2cab 100644 --- a/src/libutil-tests/unix/file-system-at.cc +++ b/src/libutil-tests/unix/file-system-at.cc @@ -29,7 +29,7 @@ TEST(fchmodatTryNoFollow, works) { RestoreSink sink(/*startFsync=*/false); sink.dstPath = tmpDir; - sink.dirFd = openDirectory(tmpDir); + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink & crf) {}); sink.createDirectory(CanonPath("dir")); sink.createSymlink(CanonPath("filelink"), "file"); @@ -39,7 +39,7 @@ TEST(fchmodatTryNoFollow, works) ASSERT_NO_THROW(chmod(tmpDir / "file", 0644)); ASSERT_NO_THROW(chmod(tmpDir / "dir", 0755)); - auto dirFd = openDirectory(tmpDir); + auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); ASSERT_TRUE(dirFd); struct ::stat st; @@ -86,7 +86,7 @@ TEST(fchmodatTryNoFollow, fallbackWithoutProc) { RestoreSink sink(/*startFsync=*/false); sink.dstPath = tmpDir; - sink.dirFd = openDirectory(tmpDir); + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink & crf) {}); sink.createSymlink(CanonPath("link"), "file"); } @@ -104,7 +104,7 @@ TEST(fchmodatTryNoFollow, fallbackWithoutProc) if (mount("tmpfs", "/proc", "tmpfs", 0, 0) == -1) _exit(1); - auto dirFd = openDirectory(tmpDir); + auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); if (!dirFd) exit(1); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index e1efca0ddf7c..5dec2decb39c 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -363,7 +363,7 @@ void writeFile(const std::filesystem::path & path, Source & source, mode_t mode, void syncParent(const std::filesystem::path & path) { assert(path.has_parent_path()); - AutoCloseFD fd = openDirectory(path.parent_path()); + AutoCloseFD fd = openDirectory(path.parent_path(), FinalSymlink::Follow); if (!fd) throw NativeSysError("opening file %s", PathFmt(path)); /* TODO: Fix on windows, FlushFileBuffers requires GENERIC_WRITE. */ @@ -408,7 +408,7 @@ void recursiveSync(const std::filesystem::path & path) /* Fsync all the directories. */ for (auto dir = dirsToFsync.rbegin(); dir != dirsToFsync.rend(); ++dir) { - AutoCloseFD fd = openDirectory(*dir); /* TODO: O_NOFOLLOW? */ + AutoCloseFD fd = openDirectory(*dir, FinalSymlink::DontFollow); if (!fd) throw NativeSysError("opening directory %1%", PathFmt(*dir)); fd.fsync(); diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 067240812f9a..58aecade75a2 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -166,10 +166,17 @@ std::filesystem::path readLink(const std::filesystem::path & path); */ std::filesystem::path descriptorToPath(Descriptor fd); +/** + * Type-safe boolean for whether to follow the final symlink component. + */ +enum struct FinalSymlink : bool { DontFollow = false, Follow = true }; + /** * Open a `Descriptor` with read-only access to the given directory. + * + * @param finalSymlink If `DontFollow`, fail if the path is a symlink. */ -AutoCloseFD openDirectory(const std::filesystem::path & path); +AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink finalSymlink = FinalSymlink::Follow); /** * Open a `Descriptor` with read-only access to the given file. diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index 55fd7584fbf0..9423b76a1e9c 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -23,9 +23,10 @@ namespace nix { -AutoCloseFD openDirectory(const std::filesystem::path & path) +AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink finalSymlink) { - return AutoCloseFD{open(path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC)}; + return AutoCloseFD{open( + path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC | (finalSymlink == FinalSymlink::Follow ? 0 : O_NOFOLLOW))}; } AutoCloseFD openFileReadonly(const std::filesystem::path & path) diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index a707343ecedf..c739ed041384 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -23,7 +23,7 @@ void setWriteTime( warn("Changing file times is not yet implemented on Windows, path is %s", PathFmt(path)); } -AutoCloseFD openDirectory(const std::filesystem::path & path) +AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink finalSymlink) { return AutoCloseFD{CreateFileW( path.c_str(), @@ -31,7 +31,7 @@ AutoCloseFD openDirectory(const std::filesystem::path & path) FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_BACKUP_SEMANTICS | (finalSymlink == FinalSymlink::Follow ? 0 : FILE_FLAG_OPEN_REPARSE_POINT), /*hTemplateFile=*/nullptr)}; } From faca7db6335f64f7b11c5ee5441805cc84f1f6d2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 19 Feb 2026 12:09:03 -0500 Subject: [PATCH 019/555] Revert "Reapply "Use the hash modulo in the derivation outputs"" This reverts commit 100e7cc3376c0f921493e035b7dc9233cd13151b. Unlike the last version, this fixes `--print-out-paths` for old clients When the client lacks `featureRealisationWithPath` (worker) or version < 2.8 (serve), the daemon was sending an empty `StringMap` for `builtOutputs`, causing `--print-out-paths` to print nothing. This commit constructs the old wire format instead, using a dummy `sha256:` hash since the derivation hash no longer exists. Old clients only need the output name and path from the JSON. --- .../json/schema/build-result-v1.yaml | 2 +- .../json/schema/build-trace-entry-v2.yaml | 57 ++-- src/json-schema-checks/meson.build | 3 - src/libcmd/built-path.cc | 18 +- src/libstore-tests/build-result.cc | 16 +- src/libstore-tests/common-protocol.cc | 63 ---- .../data/build-result/success.json | 4 - .../data/dummy-store/one-realisation.json | 3 +- .../data/realisation/simple.json | 12 +- .../with-dependent-realisations.json | 8 - .../data/realisation/with-signature.json | 16 +- .../build-result-2.7-compat.bin | Bin 0 -> 664 bytes .../build-result-2.7-compat.json | 37 +++ .../data/serve-protocol/build-result-2.8.bin | Bin 0 -> 360 bytes .../data/serve-protocol/build-result-2.8.json | 37 +++ .../data/serve-protocol/drv-output-2.8.bin | Bin 0 -> 160 bytes .../data/serve-protocol/drv-output-2.8.json | 10 + .../data/serve-protocol/realisation-2.8.bin | Bin 0 -> 352 bytes .../data/serve-protocol/realisation-2.8.json | 13 + .../unkeyed-realisation-2.8.bin | Bin 0 -> 272 bytes .../unkeyed-realisation-2.8.json | 7 + .../build-result-1.29-compat.bin | Bin 0 -> 664 bytes .../build-result-1.29-compat.json | 37 +++ ...-result-realisation-with-path-not-hash.bin | Bin 0 -> 424 bytes ...result-realisation-with-path-not-hash.json | 39 +++ ...-output-realisation-with-path-not-hash.bin | Bin 0 -> 160 bytes ...output-realisation-with-path-not-hash.json | 10 + ...isation-realisation-with-path-not-hash.bin | Bin 0 -> 352 bytes ...sation-realisation-with-path-not-hash.json | 13 + ...isation-realisation-with-path-not-hash.bin | Bin 0 -> 272 bytes ...sation-realisation-with-path-not-hash.json | 7 + .../ca-drv/store-after.json | 3 +- .../ca-drv/substituter.json | 3 +- .../issue-11928/store-after.json | 3 +- .../issue-11928/substituter.json | 6 +- src/libstore-tests/dummy-store.cc | 16 +- src/libstore-tests/realisation.cc | 17 +- src/libstore-tests/serve-protocol.cc | 226 +++++++++---- src/libstore-tests/worker-protocol.cc | 310 +++++++++++------- src/libstore-tests/worker-substitution.cc | 28 +- src/libstore/binary-cache-store.cc | 12 +- .../build/derivation-building-goal.cc | 7 +- src/libstore/build/derivation-goal.cc | 58 +--- .../build/drv-output-substitution-goal.cc | 7 +- src/libstore/ca-specific-schema.sql | 40 +-- src/libstore/common-protocol.cc | 28 -- src/libstore/daemon.cc | 28 +- src/libstore/derivations.cc | 256 ++++++++------- src/libstore/dummy-store.cc | 12 +- .../include/nix/store/binary-cache-store.hh | 14 +- .../store/build/derivation-building-misc.hh | 1 - .../nix/store/build/derivation-goal.hh | 2 - .../include/nix/store/common-protocol-impl.hh | 5 +- .../include/nix/store/common-protocol.hh | 5 +- src/libstore/include/nix/store/derivations.hh | 59 ++-- .../include/nix/store/dummy-store-impl.hh | 2 +- .../store/length-prefixed-protocol-helper.hh | 20 +- src/libstore/include/nix/store/realisation.hh | 74 +++-- .../include/nix/store/serve-protocol-impl.hh | 6 +- .../include/nix/store/serve-protocol.hh | 18 +- .../include/nix/store/worker-protocol-impl.hh | 6 +- .../include/nix/store/worker-protocol.hh | 27 +- src/libstore/local-store.cc | 22 +- src/libstore/misc.cc | 9 +- src/libstore/nar-info-disk-cache.cc | 53 +-- src/libstore/realisation.cc | 74 +++-- src/libstore/remote-store.cc | 55 +--- src/libstore/restricted-store.cc | 15 +- src/libstore/serve-protocol.cc | 121 ++++++- src/libstore/store-api.cc | 7 +- src/libstore/unix/build/derivation-builder.cc | 5 +- src/libstore/worker-protocol.cc | 153 ++++++++- src/nix/build-remote/build-remote.cc | 6 +- src/perl/lib/Nix/Store.xs | 7 +- tests/functional/ca/common.sh | 3 + tests/functional/ca/substitute.sh | 6 +- 76 files changed, 1368 insertions(+), 879 deletions(-) delete mode 100644 src/libstore-tests/data/realisation/with-dependent-realisations.json create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.8.json create mode 100644 src/libstore-tests/data/serve-protocol/drv-output-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/drv-output-2.8.json create mode 100644 src/libstore-tests/data/serve-protocol/realisation-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/realisation-2.8.json create mode 100644 src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json create mode 100644 src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin create mode 100644 src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json create mode 100644 src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin create mode 100644 src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json create mode 100644 src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin create mode 100644 src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.json create mode 100644 src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.bin create mode 100644 src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json create mode 100644 src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin create mode 100644 src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml index 55e55d3e5cbe..7730f3d93999 100644 --- a/doc/manual/source/protocols/json/schema/build-result-v1.yaml +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -83,7 +83,7 @@ properties: description: | A mapping from output names to their build trace entries. additionalProperties: - "$ref": "build-trace-entry-v2.yaml" + "$ref": "build-trace-entry-v2.yaml#/$defs/value" failure: type: object diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml index 4340f82388c2..2e8cd07da2a8 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml @@ -16,24 +16,21 @@ description: | - Version 1: Original format - - Version 2: Remove `dependentRealisations` + - Version 2: + - Use `drvPath` not `drvHash` to refer to derivation in a more conventional way. + - Remove `dependentRealisations` + - Separate into `key` and `value` type: object required: - - id - - outPath - - signatures -allOf: - - "$ref": "#/$defs/key" - - "$ref": "#/$defs/value" + - key + - value properties: - id: {} - outPath: {} - signatures: {} -additionalProperties: - dependentRealisations: - description: deprecated field - type: object + key: + "$ref": "#/$defs/key" + value: + "$ref": "#/$defs/value" +additionalProperties: false "$defs": key: @@ -43,23 +40,20 @@ additionalProperties: This is the "key" part, refering to a derivation and output. type: object required: - - id + - drvPath + - outputName properties: - id: + drvPath: + "$ref": "store-path-v1.yaml" + title: Derivation Path + description: | + The store path of the derivation that was built. + outputName: type: string - title: Derivation Output ID - pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$" + title: Output Name description: | - Unique identifier for the derivation output that was built. - - Format: `{hash-quotient-drv}!{output-name}` - - - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv). - Begins with `sha256:`. - - - **output-name**: Name of the specific output (e.g., "out", "dev", "doc") - - Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"` + Name of the specific output (e.g., "out", "dev", "doc") + additionalProperties: false value: title: Build Trace Value @@ -77,13 +71,6 @@ additionalProperties: description: | The path to the store object that resulted from building this derivation for the given output name. - patternProperties: - "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$": - "$ref": "store-path-v1.yaml" - title: Dependent Store Path - description: Store path that this dependency resolved to during the build - additionalProperties: false - signatures: type: array title: Build Signatures diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index a1b525bc51e4..a4d946160969 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -65,9 +65,6 @@ schemas = [ 'schema' : schema_dir / 'build-trace-entry-v2.yaml', 'files' : [ 'simple.json', - # The field is no longer supported, but we want to show that we - # ignore it during parsing. - 'with-dependent-realisations.json', 'with-signature.json', ], }, diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index fc7f1849384c..f60e4499c3e0 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -108,20 +108,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const overloaded{ [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { - auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto & [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath->outPath()), - outputName); - DrvOutput key{*drvOutput, outputName}; + DrvOutput key{ + .drvPath = p.drvPath->outPath(), + .outputName = outputName, + }; auto thisRealisation = store.queryRealisation(key); - assert(thisRealisation); // We’ve built it, so we must - // have the realisation - res.insert(Realisation{*thisRealisation, std::move(key)}); + // We’ve built it, so we must have the realisation. + assert(thisRealisation); + res.insert(Realisation{*thisRealisation, key}); } else { res.insert(outputPath); } diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc index b7e8f83f9e7c..a1d8ddee6412 100644 --- a/src/libstore-tests/build-result.cc +++ b/src/libstore-tests/build-result.cc @@ -75,25 +75,13 @@ INSTANTIATE_TEST_SUITE_P( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index d63cbeeeed2c..9146af219c20 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -108,69 +108,6 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( - drvOutput, - "drv-output", - (std::tuple{ - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - })) - -CHARACTERIZATION_TEST( - realisation, - "realisation", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - -READ_CHARACTERIZATION_TEST( - realisation_with_deps, - "realisation-with-deps", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - CHARACTERIZATION_TEST( vector, "vector", diff --git a/src/libstore-tests/data/build-result/success.json b/src/libstore-tests/data/build-result/success.json index 4baadb547758..ec3479d5564b 100644 --- a/src/libstore-tests/data/build-result/success.json +++ b/src/libstore-tests/data/build-result/success.json @@ -1,14 +1,10 @@ { "builtOutputs": { "bar": { - "dependentRealisations": {}, - "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "signatures": [] }, "foo": { - "dependentRealisations": {}, - "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [] } diff --git a/src/libstore-tests/data/dummy-store/one-realisation.json b/src/libstore-tests/data/dummy-store/one-realisation.json index b5c8b8c5621d..576de0838177 100644 --- a/src/libstore-tests/data/dummy-store/one-realisation.json +++ b/src/libstore-tests/data/dummy-store/one-realisation.json @@ -1,8 +1,7 @@ { "buildTrace": { - "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv": { "out": { - "dependentRealisations": {}, "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [] } diff --git a/src/libstore-tests/data/realisation/simple.json b/src/libstore-tests/data/realisation/simple.json index 2ccb1e721198..1e4760b56827 100644 --- a/src/libstore-tests/data/realisation/simple.json +++ b/src/libstore-tests/data/realisation/simple.json @@ -1,6 +1,10 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } } diff --git a/src/libstore-tests/data/realisation/with-dependent-realisations.json b/src/libstore-tests/data/realisation/with-dependent-realisations.json deleted file mode 100644 index a58e0d7fe1c2..000000000000 --- a/src/libstore-tests/data/realisation/with-dependent-realisations.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "dependentRealisations": { - "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" - }, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [] -} diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json index 3270f1cdebf4..cca29ef3e45e 100644 --- a/src/libstore-tests/data/realisation/with-signature.json +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -1,8 +1,12 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] + } } diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin new file mode 100644 index 0000000000000000000000000000000000000000..fd745215c0330ffb0fb337617da5b09e71e728f1 GIT binary patch literal 664 zcmZQ&fBEet;BU3Xg0}6p+QeqL<#8#-e)k>KuN>)l#G+s$ZDZjKNAh9F^Xl}Y;xp_vVaUm{R d7w8_PTA*8~?a#FQd|LVwXgq2D1Ze>I696<2ZXf^v literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json new file mode 100644 index 000000000000..9c97678dadad --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.bin b/src/libstore-tests/data/serve-protocol/build-result-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..fa072599578a351e0f289c1c6bbdc5c1ff90107c GIT binary patch literal 360 zcmZQ&fBv3WB>s7x-9ts literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.json b/src/libstore-tests/data/serve-protocol/build-result-2.8.json new file mode 100644 index 000000000000..9c97678dadad --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/build-result-2.8.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin b/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..5be0b15a34567aed217e56dbf2543f2f60a200be GIT binary patch literal 160 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQsQgQeqXDWenw$YaRN>LeDBQsQgQeqXDr4QwkXdVL- eR9`HVPApDIvvQ;fu(bu+0kfyDJhh0H_5c784ONx^ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.json b/src/libstore-tests/data/serve-protocol/realisation-2.8.json new file mode 100644 index 000000000000..1977d8485b99 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/realisation-2.8.json @@ -0,0 +1,13 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] + } +} diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..fce3a25dd8811573e84e81985b1cbf3d972bbffa GIT binary patch literal 272 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7EtQ6I5R;luj&8NwadK39z*V+5xkt LuspShmi7Pu*K$Gw literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json new file mode 100644 index 000000000000..2c2d264f9a17 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json @@ -0,0 +1,7 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] +} diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin new file mode 100644 index 0000000000000000000000000000000000000000..fd745215c0330ffb0fb337617da5b09e71e728f1 GIT binary patch literal 664 zcmZQ&fBEet;BU3Xg0}6p+QeqL<#8#-e)k>KuN>)l#G+s$ZDZjKNAh9F^Xl}Y;xp_vVaUm{R d7w8_PTA*8~?a#FQd|LVwXgq2D1Ze>I696<2ZXf^v literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json new file mode 100644 index 000000000000..9c97678dadad --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin new file mode 100644 index 0000000000000000000000000000000000000000..11bec3e6e3b7a82f7b2d792d687a807644ff9d05 GIT binary patch literal 424 zcmZQ&fB``9-Pv>4xRz8I{I`xM*FNUS#vq N^7F|y52hDn001VXF1-K% literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json new file mode 100644 index 000000000000..1d566b13bb1a --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json @@ -0,0 +1,39 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "cpuSystem": 604000000, + "cpuUser": 500000000, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin new file mode 100644 index 0000000000000000000000000000000000000000..5be0b15a34567aed217e56dbf2543f2f60a200be GIT binary patch literal 160 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQsQgQeqXDWenw$YaRN>LeDBQsQgQeqXDr4QwkXdVL- eR9`HVPApDIvvQ;fu(bu+0kfyDJhh0H_5c784ONx^ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json new file mode 100644 index 000000000000..1977d8485b99 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json @@ -0,0 +1,13 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] + } +} diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin new file mode 100644 index 0000000000000000000000000000000000000000..fce3a25dd8811573e84e81985b1cbf3d972bbffa GIT binary patch literal 272 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7EtQ6I5R;luj&8NwadK39z*V+5xkt LuspShmi7Pu*K$Gw literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json new file mode 100644 index 000000000000..2c2d264f9a17 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json @@ -0,0 +1,7 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] +} diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json index 7f0f62f876c0..3271cb985987 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json @@ -1,8 +1,7 @@ { "buildTrace": { - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", "signatures": [] } diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json index 93f4fb22a620..f10653a3a7f3 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json @@ -1,8 +1,7 @@ { "buildTrace": { - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", "signatures": [] } diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json index 617984a27eb6..c4d3901236c2 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json @@ -1,8 +1,7 @@ { "buildTrace": { - "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", "signatures": [] } diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json index a63d6243fae3..ecd228214691 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json @@ -1,15 +1,13 @@ { "buildTrace": { - "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", "signatures": [] } }, - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", "signatures": [] } diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index e3422fe585f8..cdb92cd1ea42 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -37,20 +37,19 @@ TEST(DummyStore, realisation_read) return cfg->openDummyStore(); }(); - auto drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16); + StorePath drvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}; auto outputName = "foo"; - EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr); + EXPECT_EQ(store->queryRealisation({drvPath, outputName}), nullptr); UnkeyedRealisation value{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }; - store->buildTrace.insert({drvHash, {{outputName, value}}}); + store->buildTrace.insert({drvPath, {{outputName, value}}}); - auto value2 = store->queryRealisation({drvHash, outputName}); + auto value2 = store->queryRealisation({drvPath, outputName}); ASSERT_TRUE(value2); EXPECT_EQ(*value2, value); @@ -131,10 +130,7 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { [&] { auto store = writeCfg->openDummyStore(); store->buildTrace.insert_or_assign( - Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, std::map{ { "out", diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index d2d7df80f59a..9c0ebbe8abf9 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -46,13 +46,10 @@ TEST_P(RealisationJsonTest, to_json) Realisation simple{ { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, .outputName = "foo", }, }; @@ -77,14 +74,4 @@ INSTANTIATE_TEST_SUITE_P( }(), })); -/** - * We no longer have a notion of "dependent realisations", but we still - * want to parse old realisation files. So make this just be a read test - * (no write direction), accordingly. - */ -TEST_F(RealisationTest, dependent_realisations_from_json) -{ - readJsonTest("with-dependent-realisations", simple); -} - } // namespace nix diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 159abeb60827..3eeecd70f4c8 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -78,16 +78,19 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - drvOutput, - "drv-output", - defaultVersion, + drvOutput_2_8, + "drv-output-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) @@ -96,54 +99,37 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, + unkeyedRealisation_2_8, + "unkeyed-realisation-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, + realisation_2_8, + "realisation-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -209,7 +195,10 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully write + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, buildResult_2_6, "build-result-2.6", @@ -242,27 +231,72 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_8, + "build-result-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -578,4 +612,62 @@ TEST_F(ServeProtoTest, handshake_client_corrupted_throws) }); } +/** + * The old-protocol fallback writes `builtOutputs` as a `StringMap` + * with a dummy hash so that old clients can still extract output + * paths. This round-trips because the read side only uses the + * `outputName` (from the key) and `outPath` (from the JSON value). + */ +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_7_compat, + "build-result-2.7-compat", + (ServeProto::Version{ + .major = 2, + .minor = 7, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + } // namespace nix diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index aac5822d01f8..044fbcc92028 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -18,17 +18,17 @@ namespace nix { TEST(WorkerProtoVersionNumber, ordering) { using Number = WorkerProto::Version::Number; - EXPECT_LT((Number{1, 10}), (Number{1, 20})); - EXPECT_GT((Number{1, 30}), (Number{1, 20})); - EXPECT_EQ((Number{1, 10}), (Number{1, 10})); - EXPECT_LT((Number{0, 255}), (Number{1, 0})); + EXPECT_LT((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 20})); + EXPECT_GT((Number{.major = 1, .minor = 30}), (Number{.major = 1, .minor = 20})); + EXPECT_EQ((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 10})); + EXPECT_LT((Number{.major = 0, .minor = 255}), (Number{.major = 1, .minor = 0})); } TEST(WorkerProtoVersion, partialOrderingSameFeatures) { using V = WorkerProto::Version; - V v1{.number = {1, 20}, .features = {"a", "b"}}; - V v2{.number = {1, 30}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; + V v2{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}}; EXPECT_TRUE(v1 < v2); EXPECT_TRUE(v2 > v1); @@ -40,8 +40,8 @@ TEST(WorkerProtoVersion, partialOrderingSameFeatures) TEST(WorkerProtoVersion, partialOrderingSubsetFeatures) { using V = WorkerProto::Version; - V fewer{.number = {1, 30}, .features = {"a"}}; - V more{.number = {1, 30}, .features = {"a", "b"}}; + V fewer{.number = {.major = 1, .minor = 30}, .features = {"a"}}; + V more{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}}; // fewer <= more: JUST the features are a subset EXPECT_TRUE(fewer < more); @@ -54,8 +54,8 @@ TEST(WorkerProtoVersion, partialOrderingUnordered) { using V = WorkerProto::Version; // Same number but incomparable features - V v1{.number = {1, 20}, .features = {"a", "c"}}; - V v2{.number = {1, 20}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "c"}}; + V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; EXPECT_FALSE(v1 < v2); EXPECT_FALSE(v1 > v2); @@ -69,8 +69,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures) { using V = WorkerProto::Version; // Higher number but fewer features — unordered - V v1{.number = {1, 30}, .features = {"a"}}; - V v2{.number = {1, 20}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 30}, .features = {"a"}}; + V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; EXPECT_FALSE(v1 < v2); EXPECT_FALSE(v1 > v2); @@ -80,8 +80,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures) TEST(WorkerProtoVersion, partialOrderingEmptyFeatures) { using V = WorkerProto::Version; - V empty{.number = {1, 20}, .features = {}}; - V some{.number = {1, 30}, .features = {"a"}}; + V empty{.number = {.major = 1, .minor = 20}, .features = {}}; + V some{.number = {.major = 1, .minor = 30}, .features = {"a"}}; // empty features is a subset of everything EXPECT_TRUE(empty < some); @@ -223,69 +223,67 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, drvOutput, - "drv-output", - defaultVersion, + "drv-output-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 38, + }, + .features = {"realisation-with-path-not-hash"}, + }), (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, + unkeyedRealisation_realisation_with_path, + "unkeyed-realisation-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", + .major = 1, + .minor = 38, }, - }, + .features = {"realisation-with-path-not-hash"}, + }), + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, + realisation_realisation_with_path, + "realisation-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", + .major = 1, + .minor = 38, }, + .features = {"realisation-with-path-not-hash"}, + }), + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -318,7 +316,10 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully write + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_28, "build-result-1.28", @@ -347,25 +348,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -374,7 +363,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_29, "build-result-1.29", @@ -410,27 +400,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -443,7 +419,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_37, "build-result-1.37", @@ -479,27 +456,71 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_realisation_with_path, + "build-result-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 38, + }, + .features = {"realisation-with-path-not-hash"}, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -920,7 +941,11 @@ TEST_F(WorkerProtoTest, handshake_features) out, in, WorkerProto::Version{ - .number = {.major = 1, .minor = 123}, + .number = + { + .major = 1, + .minor = 123, + }, .features = {"bar", "aap", "mies", "xyzzy"}, }); }); @@ -931,7 +956,11 @@ TEST_F(WorkerProtoTest, handshake_features) out, in, WorkerProto::Version{ - .number = {.major = 1, .minor = 200}, + .number = + { + .major = 1, + .minor = 200, + }, .features = {"foo", "bar", "xyzzy"}, }); @@ -1013,4 +1042,65 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) }); } +/** + * The old-protocol fallback writes `builtOutputs` as a `StringMap` + * with a dummy hash so that old clients can still extract output + * paths. This round-trips because the read side only uses the + * `outputName` (from the key) and `outPath` (from the JSON value). + */ +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_29_compat, + "build-result-1.29-compat", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 29, + }, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + } // namespace nix diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index 3a049b7e7216..9cc7b56dcb8b 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -198,12 +198,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Snapshot the destination store before checkpointJson("ca-drv/store-before", dummyStore); - // Compute the hash modulo of the derivation - // For CA floating derivations, the kind is Deferred since outputs aren't known until build - auto hashModulo = hashDerivationModulo(*dummyStore, drv, true); - ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred); - auto drvHash = hashModulo.hashes.at("out"); - // Create the output store object auto outputPath = substituter->addToStore( "test-ca-drv-out", @@ -222,7 +216,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Add the realisation (build trace) to the substituter substituter->buildTrace.insert_or_assign( - drvHash, + drvPath, std::map{ { "out", @@ -236,7 +230,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) checkpointJson("ca-drv/substituter", substituter); // The realisation should not exist in the destination store yet - DrvOutput drvOutput{drvHash, "out"}; + DrvOutput drvOutput{drvPath, "out"}; ASSERT_FALSE(dummyStore->queryRealisation(drvOutput)); // Create a worker with our custom substituter @@ -299,11 +293,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Write the dependency derivation to the destination store auto depDrvPath = dummyStore->writeDerivation(depDrv); - // Compute the hash modulo for the dependency derivation - auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true); - ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred); - auto depDrvHash = depHashModulo.hashes.at("out"); - // Create the output store object for the dependency in the substituter auto depOutputPath = substituter->addToStore( "dep-drv-out", @@ -322,7 +311,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Add the realisation for the dependency to the substituter substituter->buildTrace.insert_or_assign( - depDrvHash, + depDrvPath, std::map{ { "out", @@ -353,11 +342,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Snapshot the destination store before checkpointJson("issue-11928/store-before", dummyStore); - // Compute the hash modulo for the root derivation - auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true); - ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred); - auto rootDrvHash = rootHashModulo.hashes.at("out"); - // Create the output store object for the root derivation // Note: it does NOT reference the dependency's output auto rootOutputPath = substituter->addToStore( @@ -378,12 +362,12 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) HashAlgorithm::SHA256); // The DrvOutputs for both derivations - DrvOutput depDrvOutput{depDrvHash, "out"}; - DrvOutput rootDrvOutput{rootDrvHash, "out"}; + DrvOutput depDrvOutput{depDrvPath, "out"}; + DrvOutput rootDrvOutput{rootDrvPath, "out"}; // Add the realisation for the root derivation to the substituter substituter->buildTrace.insert_or_assign( - rootDrvHash, + rootDrvPath, std::map{ { "out", diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 31fe2c6173da..34a32096201f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -514,7 +514,7 @@ StorePath BinaryCacheStore::addToStore( std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id) { - return realisationsPrefix + "/" + id.to_string() + ".doi"; + return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi"; } void BinaryCacheStore::queryRealisationUncached( @@ -535,7 +535,10 @@ void BinaryCacheStore::queryRealisationUncached( realisation = std::make_shared(nlohmann::json::parse(*data)); } catch (Error & e) { e.addTrace( - {}, "while parsing file '%s' as a realisation for key '%s'", outputInfoFilePath, id.to_string()); + {}, + "while parsing file '%s' as a build trace value for key '%s'", + outputInfoFilePath, + id.to_string()); throw; } return (*callbackPtr)(std::move(realisation)); @@ -551,7 +554,10 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info) { if (diskCache) diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); - upsertFile(makeRealisationPath(info.id), static_cast(info).dump(), "application/json"); + upsertFile( + makeRealisationPath(info.id), + static_cast(static_cast(info)).dump(), + "application/json"); } ref BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 60ca5492c168..0f577a2007e9 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -304,9 +304,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths) given this information by the downstream goal, that cannot happen anymore if the downstream goal only cares about one output, but we care about all outputs. */ - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - for (auto & [outputName, outputHash] : outputHashes) { - InitialOutput v{.outputHash = outputHash}; + for (auto & [outputName, _] : drv->outputs) { + InitialOutput v; /* TODO we might want to also allow randomizing the paths for regular CA derivations, e.g. for sake of checking @@ -1276,7 +1275,7 @@ DerivationBuildingGoal::checkPathValidity(std::map & : PathStatus::Corrupt, }; } - auto drvOutput = DrvOutput{info.outputHash, i.first}; + auto drvOutput = DrvOutput{drvPath, i.first}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 4f99928d7850..4a726e295033 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -37,12 +37,6 @@ DerivationGoal::DerivationGoal( , drvPath(drvPath) , wantedOutput(wantedOutput) , drv{std::make_unique(drv)} - , outputHash{[&] { - auto outputHashes = staticOutputHashes(worker.evalStore, drv); - if (auto * mOutputHash = get(outputHashes, wantedOutput)) - return *mOutputHash; - throw Error("derivation '%s' does not have output '%s'", worker.store.printStorePath(drvPath), wantedOutput); - }()} , buildMode(buildMode) { @@ -102,7 +96,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) them. */ if (worker.settings.useSubstitutes && drvOptions.substitutesAllowed(worker.settings)) { if (!checkResult) { - DrvOutput id{outputHash, wantedOutput}; + DrvOutput id{drvPath, wantedOutput}; auto g = worker.makeDrvOutputSubstitutionGoal(id); waitees.insert(g); co_await await(std::move(waitees)); @@ -186,12 +180,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) // No `std::visit` for coroutines yet if (auto * successP = resolvedResult.tryGetSuccess()) { auto & success = *successP; - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - auto resolvedHashes = staticOutputHashes(worker.store, drvResolved); - - auto outputHash = get(outputHashes, wantedOutput); - auto resolvedHash = get(resolvedHashes, wantedOutput); - if ((!outputHash) || (!resolvedHash)) + if (!drv->outputs.contains(wantedOutput)) throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", worker.store.printStorePath(drvPath), @@ -200,7 +189,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto realisation = [&] { auto take1 = get(success.builtOutputs, wantedOutput); if (take1) - return static_cast(*take1); + return *take1; /* The above `get` should work. But stateful tracking of outputs in resolvedResult, this can get out of sync with the @@ -208,7 +197,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) check the store directly if it fails. */ auto take2 = worker.evalStore.queryRealisation( DrvOutput{ - .drvHash = *resolvedHash, + .drvPath = pathResolved, .outputName = wantedOutput, }); if (take2) @@ -224,7 +213,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) Realisation newRealisation{ realisation, { - .drvHash = *outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }}; newRealisation.signatures.clear(); @@ -270,16 +259,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* In checking mode, the builder will not register any outputs. So we want to make sure the ones that we wanted to check are properly there. */ - success.builtOutputs = {{ - wantedOutput, - { - assertPathValidity(), - { - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}; + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; } else { /* Otherwise the builder will give us info for out output, but also for other outputs. Filter down to just our output so as @@ -298,16 +278,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) if (success.builtOutputs.count(wantedOutput) == 0) { debug( "BUG! wanted output '%s' not in builtOutputs, working around by adding it manually", wantedOutput); - success.builtOutputs = {{ - wantedOutput, - { - assertPathValidity(), - { - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}; + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; } } } @@ -404,7 +375,7 @@ std::optional> DerivationGoal::checkPa if (drv->type().isImpure()) return std::nullopt; - auto drvOutput = DrvOutput{outputHash, wantedOutput}; + auto drvOutput = DrvOutput{drvPath, wantedOutput}; std::optional mRealisation; @@ -444,7 +415,7 @@ std::optional> DerivationGoal::checkPa Realisation{ *mRealisation, { - .drvHash = outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }, }); @@ -475,16 +446,7 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke return Goal::doneSuccess( BuildResult::Success{ .status = status, - .builtOutputs = {{ - wantedOutput, - { - std::move(builtOutput), - DrvOutput{ - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}, + .builtOutputs = {{wantedOutput, std::move(builtOutput)}}, }); } diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 71f0ae1e7db4..5a5fe06d5e89 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -10,7 +10,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput & id, Worke : Goal(worker, init()) , id(id) { - name = fmt("substitution of '%s'", id.to_string()); + name = fmt("substitution of '%s'", id.render(worker.store)); trace("created"); } @@ -93,7 +93,8 @@ Goal::Co DrvOutputSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); + debug( + "derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store)); if (substituterFailed) { worker.failedSubstitutions++; @@ -108,7 +109,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() std::string DrvOutputSubstitutionGoal::key() { - return "a$" + std::string(id.to_string()); + return "a$" + std::string(id.render(worker.store)); } } // namespace nix diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index c5e4e3897992..5daeef8c8539 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -2,7 +2,16 @@ -- Won't be loaded unless the experimental feature `ca-derivations` -- is enabled -create table if not exists Realisations ( +-- Why the `*V` tables +-- +-- We are trying to keep different versions of the experiment to have +-- completely independent extra schemas from one another. This will +-- enable people to switch between versions of the experiment (including +-- newer to older) without migrating between them, but at the cost +-- of having many abandoned tables lying around. Closer to the end of +-- the experiment, we'll provide guidance on how to clean this up. + +create table if not exists BuildTraceV2 ( id integer primary key autoincrement not null, drvPath text not null, outputName text not null, -- symbolic output id, usually "out" @@ -11,31 +20,4 @@ create table if not exists Realisations ( foreign key (outputPath) references ValidPaths(id) on delete cascade ); -create index if not exists IndexRealisations on Realisations(drvPath, outputName); - --- We can end-up in a weird edge-case where a path depends on itself because --- it’s an output of a CA derivation, that happens to be the same as one of its --- dependencies. --- In that case we have a dependency loop (path -> realisation1 -> realisation2 --- -> path) that we need to break by removing the dependencies between the --- realisations -create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths - begin - delete from RealisationsRefs where realisationReference in ( - select id from Realisations where outputPath = old.id - ); - end; - -create table if not exists RealisationsRefs ( - referrer integer not null, - realisationReference integer, - foreign key (referrer) references Realisations(id) on delete cascade, - foreign key (realisationReference) references Realisations(id) on delete restrict -); --- used by deletion trigger -create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); - --- used by QueryRealisationReferences -create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); --- used by cascade deletion when ValidPaths is deleted -create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); +create index if not exists IndexBuildTraceV2 on BuildTraceV2(drvPath, outputName); diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index a0afe948d649..fc88ee889056 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -47,34 +47,6 @@ void CommonProto::Serialise::write( conn.to << renderContentAddress(ca); } -Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - try { - return nlohmann::json::parse(rawInput); - } catch (Error & e) { - e.addTrace({}, "while parsing a realisation object in the remote protocol"); - throw; - } -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << static_cast(realisation).dump(); -} - -DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); -} - std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 3e5eac517008..a7be521a3d7f 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -955,14 +955,8 @@ static void performOp( case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { - auto outputId = WorkerProto::Serialise::read(*store, rconn); - auto outputPath = StorePath(readString(conn.from)); - store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId}); - } else { - auto realisation = WorkerProto::Serialise::read(*store, rconn); - store->registerDrvOutput(realisation); - } + auto realisation = WorkerProto::Serialise::read(*store, rconn); + store->registerDrvOutput(realisation); logger->stopWork(); break; } @@ -970,19 +964,13 @@ static void performOp( case WorkerProto::Op::QueryRealisation: { logger->startWork(); auto outputId = WorkerProto::Serialise::read(*store, rconn); - auto info = store->queryRealisation(outputId); + std::optional info = *store->queryRealisation(outputId); logger->stopWork(); - if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { - std::set outPaths; - if (info) - outPaths.insert(info->outPath); - WorkerProto::write(*store, wconn, outPaths); - } else { - std::set realisations; - if (info) - realisations.insert({*info, outputId}); - WorkerProto::write(*store, wconn, realisations); - } + /* Only return the new format because if we got past + `DrvOutput` serialization, we know that is what we're using. + */ + assert(conn.protoVersion.features.contains(WorkerProto::featureRealisationWithPath)); + WorkerProto::write(*store, wconn, info); break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index ab969a089f57..b20972df7b91 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -861,13 +861,14 @@ DrvHashes drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) +static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) { - std::optional hash; + std::optional hash; if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) { return *hash; } auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false); + // Cache it drvHashes.insert_or_assign(drvPath, h); return h; @@ -890,12 +891,10 @@ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPa don't leak the provenance of fixed outputs, reducing pointless cache misses as the build itself won't know this. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { - auto type = drv.type(); - /* Return a fixed hash for fixed-output derivations. */ - if (type.isFixed()) { + if (drv.type().isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw); @@ -905,54 +904,66 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } - return DrvHash{ - .hashes = outputHashes, - .kind = DrvHash::Kind::Regular, - }; + return outputHashes; } - auto kind = std::visit( - overloaded{ - [](const DerivationType::InputAddressed & ia) { - /* This might be a "pesimistically" deferred output, so we don't - "taint" the kind yet. */ - return DrvHash::Kind::Regular; - }, - [](const DerivationType::ContentAddressed & ca) { - return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; - }, - [](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }}, - drv.type().raw); + if (std::visit( + overloaded{ + [](const DerivationType::InputAddressed & ia) { + /* This might be a "pesimistically" deferred output, so we don't + "taint" the kind yet. */ + return false; + }, + [](const DerivationType::ContentAddressed & ca) { + // Already covered + assert(!ca.fixed); + return true; + }, + [](const DerivationType::Impure &) { return true; }}, + drv.type().raw)) { + return DrvHashModulo::DeferredDrv{}; + } + /* For other derivations, replace the inputs paths with recursive + calls to this function. */ DerivedPathMap::ChildNode::Map inputs2; for (auto & [drvPath, node] : drv.inputDrvs.map) { - const auto & res = pathDerivationModulo(store, drvPath); - if (res.kind == DrvHash::Kind::Deferred) - kind = DrvHash::Kind::Deferred; - for (auto & outputName : node.value) { - const auto h = get(res.hashes, outputName); - if (!h) - throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); + /* Need to build and resolve dynamic derivations first */ + if (!node.childMap.empty()) { + return DrvHashModulo::DeferredDrv{}; } - } - auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); - - std::map outputHashes; - for (const auto & [outputName, _] : drv.outputs) { - outputHashes.insert_or_assign(outputName, hash); + const auto & res = pathDerivationModulo(store, drvPath); + if (std::visit( + overloaded{ + [&](const DrvHashModulo::DeferredDrv &) { return true; }, + // Regular non-CA derivation, replace derivation + [&](const DrvHashModulo::DrvHash & drvHash) { + inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node); + return false; + }, + // CA derivation's output hashes + [&](const DrvHashModulo::CaOutputHashes & outputHashes) { + for (auto & outputName : node.value) { + /* Put each one in with a single "out" output.. */ + const auto h = get(outputHashes, outputName); + if (!h) + throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); + inputs2.insert_or_assign( + h->to_string(HashFormat::Base16, false), + DerivedPathMap::ChildNode{ + .value = {"out"}, + }); + } + return false; + }, + }, + res.raw)) { + return DrvHashModulo::DeferredDrv{}; + } } - return DrvHash{ - .hashes = outputHashes, - .kind = kind, - }; -} - -std::map staticOutputHashes(Store & store, const Derivation & drv) -{ - return hashDerivationModulo(store, drv, true).hashes; + return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); } static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) @@ -1102,26 +1113,6 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites) } } -static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) -{ - drv.applyRewrites(rewrites); - - auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); - for (auto & [outputName, output] : drv.outputs) { - if (std::holds_alternative(output.raw)) { - auto h = get(hashModulo.hashes, outputName); - if (!h) - throw Error( - "derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", drv.name, outputName); - auto outPath = store.makeOutputPath(outputName, *h, drv.name); - drv.env[outputName] = store.printStorePath(outPath); - output = DerivationOutput::InputAddressed{ - .path = std::move(outPath), - }; - } - } -} - bool Derivation::shouldResolve() const { /* No input drvs means nothing to resolve. */ @@ -1233,9 +1224,13 @@ std::optional Derivation::tryResolve( queryResolutionChain)) return std::nullopt; - rewriteDerivation(store, resolved, inputRewrites); + resolved.applyRewrites(inputRewrites); + + Derivation resolved2{std::move(resolved)}; + + resolved2.fillInOutputPaths(store); - return resolved; + return resolved2; } /** @@ -1262,7 +1257,15 @@ std::optional Derivation::tryResolve( template static void processDerivationOutputPaths(Store & store, auto && drv, std::string_view drvName) { - std::optional hashesModulo; + std::optional hashModulo_; + + auto hashModulo = [&]() -> const auto & { + if (!hashModulo_) { + // somewhat expensive so we do lazily + hashModulo_ = hashDerivationModulo(store, drv, true); + } + return *hashModulo_; + }; for (auto & [outputName, output] : drv.outputs) { auto envHasRightPath = [&](const StorePath & actual, bool isDeferred = false) { @@ -1299,65 +1302,64 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string } }; auto hash = [&](const Output & outputVariant) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(store, drv, true); - } - switch (hashesModulo->kind) { - case DrvHash::Kind::Regular: { - auto h = get(hashesModulo->hashes, outputName); - if (!h) - throw Error("derivation produced no hash for output '%s'", outputName); - auto outPath = store.makeOutputPath(outputName, *h, drvName); - - if constexpr (std::is_same_v) { - if (outputVariant.path == outPath) { - return; // Correct case - } - /* Error case, an explicitly wrong path is - always an error. */ - throw Error( - "derivation has incorrect output '%s', should be '%s'", - store.printStorePath(outputVariant.path), - store.printStorePath(outPath)); - } else if constexpr (std::is_same_v) { - if constexpr (fillIn) - /* Fill in output path for Deferred - outputs */ - output = DerivationOutput::InputAddressed{ - .path = outPath, - }; - else - /* Validation mode: deferred outputs - should have been filled in */ - warn( - "derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.", - store.printStorePath(outPath)); - } else { - /* Will never happen, based on where - `hash` is called. */ - static_assert(false); - } - envHasRightPath(outPath); - break; - } - case DrvHash::Kind::Deferred: - if constexpr (std::is_same_v) { - /* Error case, an explicitly wrong path is - always an error. */ - throw Error( - "derivation has incorrect output '%s', should be deferred", - store.printStorePath(outputVariant.path)); - } else if constexpr (std::is_same_v) { - /* Correct: Deferred output with Deferred - hash kind. */ - } else { - /* Will never happen, based on where - `hash` is called. */ - static_assert(false); - } - break; - } + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + auto outPath = store.makeOutputPath(outputName, drvHash, drvName); + + if constexpr (std::is_same_v) { + if (outputVariant.path == outPath) { + envHasRightPath(outPath); + return; // Correct case + } + /* Error case, an explicitly wrong path is + always an error. */ + throw Error( + "derivation has incorrect output '%s', should be '%s'", + store.printStorePath(outputVariant.path), + store.printStorePath(outPath)); + } else if constexpr (std::is_same_v) { + if constexpr (fillIn) { + /* Fill in output path for Deferred outputs */ + output = DerivationOutput::InputAddressed{ + .path = outPath, + }; + envHasRightPath(outPath); + } else { + /* Validation mode: deferred outputs + should have been filled in */ + warn( + "derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.", + store.printStorePath(outPath)); + } + } else { + /* Will never happen, based on where + `hash` is called. */ + static_assert(false); + } + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + input-addressed (or deferred waiting to be). */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + if constexpr (std::is_same_v) { + /* Error case, an explicitly wrong path is + always an error. */ + throw Error( + "derivation has incorrect output '%s', should be deferred", + store.printStorePath(outputVariant.path)); + } else if constexpr (std::is_same_v) { + /* Correct: Deferred output with Deferred hash kind. */ + } else { + /* Will never happen, based on where + `hash` is called. */ + static_assert(false); + } + }, + }, + hashModulo().raw); }; std::visit( overloaded{ diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 943873119ca5..052ec9b1283e 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -330,7 +330,7 @@ struct DummyStoreImpl : DummyStore void registerDrvOutput(const Realisation & output) override { - buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, output}}}, [&](auto & kv) { + buildTrace.insert_or_visit({output.id.drvPath, {{output.id.outputName, output}}}, [&](auto & kv) { kv.second.insert_or_assign(output.id.outputName, output); }); } @@ -339,7 +339,7 @@ struct DummyStoreImpl : DummyStore const DrvOutput & drvOutput, Callback> callback) noexcept override { bool visited = false; - buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) { + buildTrace.cvisit(drvOutput.drvPath, [&](const auto & kv) { if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) { visited = true; callback(std::make_shared(it->second)); @@ -436,11 +436,7 @@ ref adl_serializer>::from_json(const json & json) for (auto & [k1, v2] : getObject(v)) { UnkeyedRealisation realisation = v2; res->buildTrace.insert_or_visit( - { - Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), - {{k1, realisation}}, - }, - [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); }); + {StorePath{k0}, {{k1, realisation}}}, [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); }); } } return res; @@ -473,7 +469,7 @@ void adl_serializer::to_json(json & json, const DummyStore & val) auto obj = json::object(); val.buildTrace.cvisit_all([&](const auto & kv) { auto & [k, v] = kv; - auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); + auto & obj2 = obj[k.to_string()] = json::object(); for (auto & [k2, v2] : kv.second) obj2[k2] = v2; }); diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 2c3fbdc3a0d1..bc499130b087 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -91,8 +91,18 @@ protected: /** * The prefix under which realisation infos will be stored + * + * @note The previous (still experimental, though) hash-keyed + * realisations were under "realisations". "build trace" is a better + * name anyways (issue #11895). This is call "v2" accordingly. + * + * While we're experimenting, we'll freely increase this version + * number. Old build traces will just be "abandoned" at the old URL. + * When we are done experimenting, we'll try lean more on versioning + * the build trace entries themselves than the entire directory, for + * a smoother migration path. */ - constexpr const static std::string realisationsPrefix = "realisations"; + constexpr const static std::string realisationsPrefix = "build-trace-v2"; constexpr const static std::string cacheInfoFile = "nix-cache-info"; @@ -101,7 +111,7 @@ protected: /** * Compute the path to the given realisation * - * It's `${realisationsPrefix}/${drvOutput}.doi`. + * It's `${realisationsPrefix}/${drvPath}/${outputName}`. */ std::string makeRealisationPath(const DrvOutput & id); diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index 2b68fa1782a4..8d6892839c76 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -45,7 +45,6 @@ struct InitialOutputStatus struct InitialOutput { - Hash outputHash; std::optional known; }; diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index aaded75511f0..c6bd412182f3 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -66,8 +66,6 @@ private: */ std::unique_ptr drv; - const Hash outputHash; - const BuildMode buildMode; /** diff --git a/src/libstore/include/nix/store/common-protocol-impl.hh b/src/libstore/include/nix/store/common-protocol-impl.hh index cb1020a3c83b..d0eebe0bc23b 100644 --- a/src/libstore/include/nix/store/common-protocol-impl.hh +++ b/src/libstore/include/nix/store/common-protocol-impl.hh @@ -26,12 +26,13 @@ namespace nix { LengthPrefixedProtoHelper::write(store, conn, t); \ } -#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) -COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::map) +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, std::map) #undef COMMA_ /* protocol-specific templates */ diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index 341d87b41667..2ef0f795b4ea 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -88,8 +88,9 @@ DECLARE_COMMON_SERIALISER(std::set); template DECLARE_COMMON_SERIALISER(std::tuple); -template -DECLARE_COMMON_SERIALISER(std::map); +template +DECLARE_COMMON_SERIALISER(std::map); +#undef COMMA_ /** * These use the empty string for the null case, relying on the fact diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 0d137eaf36a2..b74fb420e69a 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -507,34 +507,39 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName); * derivations (fixed-output or not) will have a different hash for each * output. */ -struct DrvHash +struct DrvHashModulo { /** - * Map from output names to hashes + * Single hash for the derivation + * + * This is for an input-addressed derivation that doesn't + * transitively depend on any floating-CA derivations. */ - std::map hashes; - - enum struct Kind : bool { - /** - * Statically determined derivations. - * This hash will be directly used to compute the output paths - */ - Regular, + using DrvHash = Hash; - /** - * Floating-output derivations (and their reverse dependencies). - */ - Deferred, - }; + /** + * Known CA drv's output hashes, for fixed-output derivations whose + * output hashes are always known since they are fixed up-front. + */ + using CaOutputHashes = std::map; /** - * The kind of derivation this is, simplified for just "derivation hash - * modulo" purposes. + * This derivation doesn't yet have known output hashes. + * + * Either because itself is floating CA, or it (transtively) depends + * on a floating CA derivation. */ - Kind kind; -}; + using DeferredDrv = std::monostate; + + using Raw = std::variant; -void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; + Raw raw; + + bool operator==(const DrvHashModulo &) const = default; + // auto operator <=> (const DrvHashModulo &) const = default; + + MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo); +}; /** * Returns hashes with the details of fixed-output subderivations @@ -560,15 +565,17 @@ void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; * ATerm, after subderivations have been likewise expunged from that * derivation. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /** - * Return a map associating each output to a hash that uniquely identifies its - * derivation (modulo the self-references). + * If a derivation is input addressed and doesn't yet have its input + * addressed (is deferred) try using `hashDerivationModulo`. * - * \todo What is the Hash in this map? + * Does nothing if not deferred input-addressed, or + * `hashDerivationModulo` indicates it is missing inputs' output paths + * and is not yet ready (and must stay deferred). */ -std::map staticOutputHashes(Store & store, const Derivation & drv); +void resolveInputAddressed(Store & store, Derivation & drv); struct DrvHashFct { @@ -583,7 +590,7 @@ struct DrvHashFct /** * Memoisation of hashDerivationModulo(). */ -typedef boost::concurrent_flat_map DrvHashes; +typedef boost::concurrent_flat_map DrvHashes; // FIXME: global, though at least thread-safe. extern DrvHashes drvHashes; diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index ac7ab9c680fd..8fdeeb362515 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -49,7 +49,7 @@ struct DummyStore : virtual Store * outer map for the derivation, and inner maps for the outputs of a * given derivation. */ - boost::concurrent_flat_map> buildTrace; + boost::concurrent_flat_map> buildTrace; DummyStore(ref config) : Store{*config} diff --git a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh index 035019340f50..e1a80e8dc58e 100644 --- a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh +++ b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh @@ -56,14 +56,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); #define COMMA_ , template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); -#undef COMMA_ template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); -template -#define LENGTH_PREFIXED_PROTO_HELPER_X std::map +template +#define LENGTH_PREFIXED_PROTO_HELPER_X std::map LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X); +#undef COMMA_ template std::vector @@ -109,11 +109,11 @@ void LengthPrefixedProtoHelper>::write( } } -template -std::map -LengthPrefixedProtoHelper>::read(const StoreDirConfig & store, typename Inner::ReadConn conn) +template +std::map LengthPrefixedProtoHelper>::read( + const StoreDirConfig & store, typename Inner::ReadConn conn) { - std::map resMap; + std::map resMap; auto size = readNum(conn.from); while (size--) { auto k = S::read(store, conn); @@ -123,9 +123,9 @@ LengthPrefixedProtoHelper>::read(const StoreDirConfig & st return resMap; } -template -void LengthPrefixedProtoHelper>::write( - const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) +template +void LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index 670f8ce499ec..9866b1245b7f 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -18,33 +18,40 @@ struct OutputsSpec; /** * A general `Realisation` key. * - * This is similar to a `DerivedPath::Opaque`, but the derivation is - * identified by its "hash modulo" instead of by its store path. + * This is similar to a `DerivedPath::Built`, except it is only a single + * step: `drvPath` is a `StorePath` rather than a `DerivedPath`. */ struct DrvOutput { /** - * The hash modulo of the derivation. - * - * Computed from the derivation itself for most types of - * derivations, but computed from the (fixed) content address of the - * output for fixed-output derivations. + * The store path to the derivation */ - Hash drvHash; + StorePath drvPath; /** * The name of the output. */ OutputName outputName; + /** + * Skips the store dir on the `drvPath` + */ std::string to_string() const; - std::string strHash() const - { - return drvHash.to_string(HashFormat::Base16, true); - } + /** + * Skips the store dir on the `drvPath` + */ + static DrvOutput from_string(std::string_view); - static DrvOutput parse(const std::string &); + /** + * Includes the store dir on `drvPath` + */ + std::string render(const StoreDirConfig & store) const; + + /** + * Includes the store dir on `drvPath` + */ + static DrvOutput parse(const StoreDirConfig & store, std::string_view); bool operator==(const DrvOutput &) const = default; auto operator<=>(const DrvOutput &) const = default; @@ -64,6 +71,16 @@ struct UnkeyedRealisation size_t checkSignatures(const DrvOutput & key, const PublicKeys & publicKeys) const; + /** + * Just check the `outPath`. Signatures don't matter for this. + * Callers must ensure that the corresponding key is the same for + * most use-cases. + */ + bool isCompatibleWith(const UnkeyedRealisation & other) const + { + return outPath == other.outPath; + } + const StorePath & getPath() const { return outPath; @@ -77,8 +94,6 @@ struct Realisation : UnkeyedRealisation { DrvOutput id; - bool isCompatibleWith(const UnkeyedRealisation & other) const; - bool operator==(const Realisation &) const = default; auto operator<=>(const Realisation &) const = default; }; @@ -89,16 +104,7 @@ struct Realisation : UnkeyedRealisation * Since these are the outputs of a single derivation, we know the * output names are unique so we can use them as the map key. */ -typedef std::map SingleDrvOutputs; - -/** - * Collection type for multiple derivations' outputs' `Realisation`s. - * - * `DrvOutput` is used because in general the derivations are not all - * the same, so we need to identify firstly which derivation, and - * secondly which output of that derivation. - */ -typedef std::map DrvOutputs; +typedef std::map SingleDrvOutputs; struct OpaquePath { @@ -149,19 +155,17 @@ struct RealisedPath class MissingRealisation final : public CloneableError { public: - MissingRealisation(DrvOutput & outputId) - : MissingRealisation(outputId.outputName, outputId.strHash()) + MissingRealisation(const StoreDirConfig & store, DrvOutput & outputId) + : MissingRealisation(store, outputId.drvPath, outputId.outputName) { } - MissingRealisation(std::string_view drv, OutputName outputName) - : CloneableError( - "cannot operate on output '%s' of the " - "unbuilt derivation '%s'", - outputName, - drv) - { - } + MissingRealisation(const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName); + MissingRealisation( + const StoreDirConfig & store, + const SingleDerivedPath & drvPath, + const StorePath & drvPathResolved, + const OutputName & outputName); }; } // namespace nix diff --git a/src/libstore/include/nix/store/serve-protocol-impl.hh b/src/libstore/include/nix/store/serve-protocol-impl.hh index a9617165a72e..24fc3c9abd8d 100644 --- a/src/libstore/include/nix/store/serve-protocol-impl.hh +++ b/src/libstore/include/nix/store/serve-protocol-impl.hh @@ -34,8 +34,10 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA , SERVE_USE_LENGTH_PREFIX_SERIALISER( - template, - std::map) + template + , + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/serve-protocol.hh b/src/libstore/include/nix/store/serve-protocol.hh index dba05a34548f..94a250b2e3c0 100644 --- a/src/libstore/include/nix/store/serve-protocol.hh +++ b/src/libstore/include/nix/store/serve-protocol.hh @@ -8,12 +8,18 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb +#define SERVE_PROTOCOL_VERSION (2 << 8 | 8) +#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) +#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) struct StoreDirConfig; struct Source; // items being serialised struct BuildResult; struct UnkeyedValidPathInfo; +struct DrvOutput; +struct UnkeyedRealisation; +struct Realisation; /** * The "serve protocol", used by ssh:// stores. @@ -63,7 +69,7 @@ struct ServeProto static constexpr Version latest = { .major = 2, - .minor = 7, + .minor = 8, }; /** @@ -205,6 +211,12 @@ inline std::ostream & operator<<(std::ostream & s, ServeProto::Command op) template<> DECLARE_SERVE_SERIALISER(BuildResult); template<> +DECLARE_SERVE_SERIALISER(DrvOutput); +template<> +DECLARE_SERVE_SERIALISER(UnkeyedRealisation); +template<> +DECLARE_SERVE_SERIALISER(Realisation); +template<> DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); template<> DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); @@ -217,8 +229,8 @@ DECLARE_SERVE_SERIALISER(std::set); template DECLARE_SERVE_SERIALISER(std::tuple); -template -DECLARE_SERVE_SERIALISER(std::map); +template +DECLARE_SERVE_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/include/nix/store/worker-protocol-impl.hh b/src/libstore/include/nix/store/worker-protocol-impl.hh index 26f6b9d44e46..605663d2eeb4 100644 --- a/src/libstore/include/nix/store/worker-protocol-impl.hh +++ b/src/libstore/include/nix/store/worker-protocol-impl.hh @@ -34,8 +34,10 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA , WORKER_USE_LENGTH_PREFIX_SERIALISER( - template, - std::map) + template + , + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 4098e8fd9123..8e20d3c992cf 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -12,6 +12,12 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f +/* Note: you generally shouldn't change the protocol version. Define a + new `WorkerProto::Feature` instead. */ +#define PROTOCOL_VERSION (1 << 8 | 39) +#define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18) +#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) +#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define STDERR_NEXT 0x6f6c6d67 #define STDERR_READ 0x64617461 // data needed from source #define STDERR_WRITE 0x64617416 // data for sink @@ -30,6 +36,9 @@ struct BuildResult; struct KeyedBuildResult; struct ValidPathInfo; struct UnkeyedValidPathInfo; +struct DrvOutput; +struct UnkeyedRealisation; +struct Realisation; enum BuildMode : uint8_t; enum TrustedFlag : bool; enum class GCAction; @@ -109,6 +118,12 @@ struct WorkerProto static const Version minimum; + /** + * Feature for transmitting `UnkeyedRealisation` and `DrvOutput` + * using drvPath (store path) instead of the old hash-based JSON format. + */ + static constexpr std::string_view featureRealisationWithPath = "realisation-with-path-not-hash"; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. @@ -307,6 +322,14 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo); template<> DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template<> +DECLARE_WORKER_SERIALISER(DrvOutput); +template<> +DECLARE_WORKER_SERIALISER(UnkeyedRealisation); +template<> +DECLARE_WORKER_SERIALISER(Realisation); +template<> +DECLARE_WORKER_SERIALISER(std::optional); +template<> DECLARE_WORKER_SERIALISER(BuildMode); template<> DECLARE_WORKER_SERIALISER(GCAction); @@ -325,8 +348,8 @@ DECLARE_WORKER_SERIALISER(std::set); template DECLARE_WORKER_SERIALISER(std::tuple); -template -DECLARE_WORKER_SERIALISER(std::map); +template +DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8631df218418..b8409b056830 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -357,14 +357,14 @@ LocalStore::LocalStore(ref config) state->stmts->RegisterRealisedOutput.create( state->db, R"( - insert into Realisations (drvPath, outputName, outputPath, signatures) + insert into BuildTraceV2 (drvPath, outputName, outputPath, signatures) values (?, ?, (select id from ValidPaths where path = ?), ?) ; )"); state->stmts->UpdateRealisedOutput.create( state->db, R"( - update Realisations + update BuildTraceV2 set signatures = ? where drvPath = ? and @@ -374,16 +374,16 @@ LocalStore::LocalStore(ref config) state->stmts->QueryRealisedOutput.create( state->db, R"( - select Realisations.id, Output.path, Realisations.signatures from Realisations - inner join ValidPaths as Output on Output.id = Realisations.outputPath + select BuildTraceV2.id, Output.path, BuildTraceV2.signatures from BuildTraceV2 + inner join ValidPaths as Output on Output.id = BuildTraceV2.outputPath where drvPath = ? and outputName = ? ; )"); state->stmts->QueryAllRealisedOutputs.create( state->db, R"( - select outputName, Output.path from Realisations - inner join ValidPaths as Output on Output.id = Realisations.outputPath + select outputName, Output.path from BuildTraceV2 + inner join ValidPaths as Output on Output.id = BuildTraceV2.outputPath where drvPath = ? ; )"); @@ -603,7 +603,7 @@ void LocalStore::upgradeDBSchema(State & state) if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) doUpgrade( - "20220326-ca-derivations", + "20251016-ca-derivations", #include "ca-specific-schema.sql.gen.hh" ); } @@ -647,8 +647,8 @@ void LocalStore::registerDrvOutput(const Realisation & info) auto combinedSignatures = oldR->signatures; combinedSignatures.insert(info.signatures.begin(), info.signatures.end()); state->stmts->UpdateRealisedOutput - .use()(concatStringsSep(" ", Signature::toStrings(combinedSignatures)))(info.id.strHash())( - info.id.outputName) + .use()(concatStringsSep(" ", Signature::toStrings(combinedSignatures)))( + info.id.drvPath.to_string())(info.id.outputName) .exec(); } else { throw Error( @@ -662,7 +662,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) } } else { state->stmts->RegisterRealisedOutput - .use()(info.id.strHash())(info.id.outputName)(printStorePath(info.outPath))( + .use()(info.id.drvPath.to_string())(info.id.outputName)(printStorePath(info.outPath))( concatStringsSep(" ", Signature::toStrings(info.signatures))) .exec(); } @@ -1540,7 +1540,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const std::set> LocalStore::queryRealisationCore_(LocalStore::State & state, const DrvOutput & id) { - auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.strHash())(id.outputName)); + auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.drvPath.to_string())(id.outputName)); if (!useQueryRealisedOutput.next()) return std::nullopt; auto realisationDbId = useQueryRealisedOutput.getInt(0); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e55dc424f0c5..44c9a72e5f2c 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -243,16 +243,15 @@ MissingPaths Store::queryMissing(const std::vector & targets) // If there are unknown output paths, attempt to find if the // paths are known to substituters through a realisation. - auto outputHashes = staticOutputHashes(*this, *drv); knownOutputPaths = true; - for (auto [outputName, hash] : outputHashes) { + for (auto & [outputName, _] : drv->outputs) { if (!bfd.outputs.contains(outputName)) continue; bool found = false; for (auto & sub : getDefaultSubstituters()) { - auto realisation = sub->queryRealisation({hash, outputName}); + auto realisation = sub->queryRealisation({drvPath, outputName}); if (!realisation) continue; found = true; @@ -369,7 +368,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, OutputPathMap outputs; for (auto & [outputName, outputPathOpt] : outputsOpt) { if (!outputPathOpt) - throw MissingRealisation(bfd.drvPath->to_string(store), outputName); + throw MissingRealisation(store, *bfd.drvPath, drvPath, outputName); auto & outputPath = *outputPathOpt; outputs.insert_or_assign(outputName, outputPath); } @@ -393,7 +392,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store bfd.output); auto & optPath = outputPaths.at(bfd.output); if (!optPath) - throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); + throw MissingRealisation(store, *bfd.drvPath, drvPath, bfd.output); return *optPath; }, }, diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 3f330753d949..632b31ce0934 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -42,12 +42,18 @@ create table if not exists NARs ( foreign key (cache) references BinaryCaches(id) on delete cascade ); -create table if not exists Realisations ( +create table if not exists BuildTrace ( cache integer not null, - outputId text not null, - content blob, -- Json serialisation of the realisation, or null if the realisation is absent + + drvPath text not null, + outputName text not null, + + -- The following are null if the realisation is absent + outputPath text, + sigs text, + timestamp integer not null, - primary key (cache, outputId), + primary key (cache, drvPath, outputName), foreign key (cache) references BinaryCaches(id) on delete cascade ); @@ -84,7 +90,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache NarInfoDiskCacheImpl( const Settings & settings, SQLiteSettings sqliteSettings, - std::filesystem::path dbPath = getCacheDir() / "binary-cache-v7.sqlite") + std::filesystem::path dbPath = getCacheDir() / "binary-cache-v8.sqlite") : NarInfoDiskCache{settings} { auto state(_state.lock()); @@ -120,24 +126,24 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache state->insertRealisation.create( state->db, R"( - insert or replace into Realisations(cache, outputId, content, timestamp) - values (?, ?, ?, ?) + insert or replace into BuildTrace(cache, drvPath, outputName, outputPath, sigs, timestamp) + values (?, ?, ?, ?, ?, ?) )"); state->insertMissingRealisation.create( state->db, R"( - insert or replace into Realisations(cache, outputId, timestamp) - values (?, ?, ?) + insert or replace into BuildTrace(cache, drvPath, outputName, timestamp) + values (?, ?, ?, ?) )"); state->queryRealisation.create( state->db, R"( - select content from Realisations - where cache = ? and outputId = ? and - ((content is null and timestamp > ?) or - (content is not null and timestamp > ?)) + select outputPath, sigs from BuildTrace + where cache = ? and drvPath = ? and outputName = ? and + ((outputPath is null and timestamp > ?) or + (outputPath is not null and timestamp > ?)) )"); /* Periodically purge expired entries from the database. */ @@ -298,22 +304,27 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache auto now = time(nullptr); - auto queryRealisation(state->queryRealisation.use()(cache.id)(id.to_string())( + auto queryRealisation(state->queryRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)( now - settings.ttlNegative)(now - settings.ttlPositive)); if (!queryRealisation.next()) - return {oUnknown, 0}; + return {oUnknown, nullptr}; if (queryRealisation.isNull(0)) - return {oInvalid, 0}; + return {oInvalid, nullptr}; try { return { oValid, - std::make_shared(nlohmann::json::parse(queryRealisation.getStr(0))), + std::make_shared( + UnkeyedRealisation{ + .outPath = StorePath{queryRealisation.getStr(0)}, + .signatures = nlohmann::json::parse(queryRealisation.getStr(1)), + }, + id), }; } catch (Error & e) { - e.addTrace({}, "while parsing the local disk cache"); + e.addTrace({}, "reading build trace key-value from the local disk cache"); throw; } }); @@ -359,7 +370,8 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache auto & cache(getCache(*state, uri)); state->insertRealisation - .use()(cache.id)(realisation.id.to_string())(static_cast(realisation).dump())( + .use()(cache.id)(realisation.id.drvPath.to_string())(realisation.id.outputName)( + realisation.outPath.to_string())(static_cast(realisation.signatures).dump())( time(nullptr)) .exec(); }); @@ -371,7 +383,8 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache auto state(_state.lock()); auto & cache(getCache(*state, uri)); - state->insertMissingRealisation.use()(cache.id)(id.to_string())(time(nullptr)).exec(); + state->insertMissingRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(time(nullptr)) + .exec(); }); } }; diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 433d67f26ba2..90a032612d28 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -8,28 +8,34 @@ namespace nix { MakeError(InvalidDerivationOutputId, Error); -DrvOutput DrvOutput::parse(const std::string & strRep) +DrvOutput DrvOutput::parse(const StoreDirConfig & store, std::string_view s) { - size_t n = strRep.find("!"); - if (n == strRep.npos) - throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); - + size_t n = s.rfind('^'); + if (n == s.npos) + throw InvalidDerivationOutputId("Invalid derivation output id '%s': missing '^'", s); return DrvOutput{ - .drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)), - .outputName = strRep.substr(n + 1), + .drvPath = store.parseStorePath(s.substr(0, n)), + .outputName = OutputName{s.substr(n + 1)}, }; } +std::string DrvOutput::render(const StoreDirConfig & store) const +{ + return std::string(store.printStorePath(drvPath)) + "^" + outputName; +} + std::string DrvOutput::to_string() const { - return strHash() + "!" + outputName; + return std::string(drvPath.to_string()) + "^" + outputName; } std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const { - nlohmann::json serialized = Realisation{*this, key}; - serialized.erase("signatures"); - return serialized.dump(); + auto serialised = static_cast(Realisation{*this, key}); + auto value = serialised.find("value"); + assert(value != serialised.end()); + value->erase("signatures"); + return serialised.dump(); } void UnkeyedRealisation::sign(const DrvOutput & key, const Signer & signer) @@ -61,9 +67,21 @@ const StorePath & RealisedPath::path() const & return std::visit([](auto & arg) -> auto & { return arg.getPath(); }, raw); } -bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const +MissingRealisation::MissingRealisation( + const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName) + : CloneableError( + "cannot operate on output '%s' of the unbuilt derivation '%s'", outputName, store.printStorePath(drvPath)) +{ +} + +MissingRealisation::MissingRealisation( + const StoreDirConfig & store, + const SingleDerivedPath & drvPath, + const StorePath & drvPathResolved, + const OutputName & outputName) + : MissingRealisation{store, drvPathResolved, outputName} { - return outPath == other.outPath; + addTrace({}, "looking up realisation for derivation '%s'", drvPath.to_string(store)); } } // namespace nix @@ -74,12 +92,20 @@ using namespace nix; DrvOutput adl_serializer::from_json(const json & json) { - return DrvOutput::parse(getString(json)); + auto obj = getObject(json); + + return { + .drvPath = valueAt(obj, "drvPath"), + .outputName = getString(valueAt(obj, "outputName")), + }; } void adl_serializer::to_json(json & json, const DrvOutput & drvOutput) { - json = drvOutput.to_string(); + json = { + {"drvPath", drvOutput.drvPath}, + {"outputName", drvOutput.outputName}, + }; } UnkeyedRealisation adl_serializer::from_json(const json & json0) @@ -101,25 +127,25 @@ void adl_serializer::to_json(json & json, const UnkeyedReali json = { {"outPath", r.outPath}, {"signatures", r.signatures}, - // back-compat - {"dependentRealisations", json::object()}, }; } -Realisation adl_serializer::from_json(const json & json0) +Realisation adl_serializer::from_json(const json & json) { - auto json = getObject(json0); + auto obj = getObject(json); - return Realisation{ - static_cast(json0), - valueAt(json, "id"), + return { + static_cast(valueAt(obj, "value")), + static_cast(valueAt(obj, "key")), }; } void adl_serializer::to_json(json & json, const Realisation & r) { - json = static_cast(r); - json["id"] = r.id; + json = { + {"key", r.id}, + {"value", static_cast(r)}, + }; } } // namespace nlohmann diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 05dd5cf13a89..c261e3384a61 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -503,12 +503,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << WorkerProto::Op::RegisterDrvOutput; - if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) { - WorkerProto::write(*this, *conn, info.id); - conn->to << std::string(info.outPath.to_string()); - } else { - WorkerProto::write(*this, *conn, info); - } + WorkerProto::write(*this, *conn, info); conn.processStderr(); } @@ -518,8 +513,10 @@ void RemoteStore::queryRealisationUncached( try { auto conn(getConnection()); - if (conn->protoVersion.number < WorkerProto::Version::Number{1, 27}) { - warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4"); + if (!conn->protoVersion.features.contains(WorkerProto::featureRealisationWithPath)) { + warn( + "the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations", + WorkerProto::featureRealisationWithPath); return callback(nullptr); } @@ -527,21 +524,12 @@ void RemoteStore::queryRealisationUncached( WorkerProto::write(*this, *conn, id); conn.processStderr(); - auto real = [&]() -> std::shared_ptr { - if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) { - auto outPaths = WorkerProto::Serialise>::read(*this, *conn); - if (outPaths.empty()) - return nullptr; - return std::make_shared(UnkeyedRealisation{.outPath = *outPaths.begin()}); - } else { - auto realisations = WorkerProto::Serialise>::read(*this, *conn); - if (realisations.empty()) - return nullptr; - return std::make_shared(*realisations.begin()); - } - }(); - - callback(std::shared_ptr(real)); + callback([&]() -> std::shared_ptr { + auto realisation = WorkerProto::Serialise>::read(*this, *conn); + if (!realisation) + return nullptr; + return std::make_shared(*realisation); + }()); } catch (...) { return callback.rethrow(); } @@ -623,30 +611,19 @@ std::vector RemoteStore::buildPathsWithResults( OutputPathMap outputs; auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); - auto drv = evalStore->readDerivation(drvPath); - const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - printStorePath(drvPath), - output); - auto outputId = DrvOutput{*outputHash, output}; + auto outputId = DrvOutput{drvPath, output}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); if (!realisation) - throw MissingRealisation(outputId); - success.builtOutputs.emplace(output, Realisation{*realisation, outputId}); + throw MissingRealisation(*this, outputId); + success.builtOutputs.emplace(output, *realisation); } else { success.builtOutputs.emplace( output, - Realisation{ - UnkeyedRealisation{ - .outPath = outputPath, - }, - outputId, + UnkeyedRealisation{ + .outPath = outputPath, }); } } diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 5248a313c240..8001d43ec8d8 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -280,9 +280,18 @@ std::vector RestrictedStore::buildPathsWithResults( for (auto & result : results) { if (auto * successP = result.tryGetSuccess()) { - for (auto & [outputName, output] : successP->builtOutputs) { - newPaths.insert(output.outPath); - newRealisations.insert(output); + if (auto * pathBuilt = std::get_if(&result.path)) { + // TODO ugly extra IO + auto drvPath = resolveDerivedPath(*next, *pathBuilt->drvPath); + for (auto & [outputName, output] : successP->builtOutputs) { + newPaths.insert(output.outPath); + newRealisations.insert( + {output, + { + .drvPath = drvPath, + .outputName = outputName, + }}); + } } } } diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index ec7b85ed8826..bddd17da8b60 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -7,6 +7,7 @@ #include "nix/store/serve-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" +#include "nix/util/json-utils.hh" #include @@ -28,10 +29,19 @@ BuildResult ServeProto::Serialise::read(const StoreDirConfig & stor if (conn.version >= ServeProto::Version{2, 3}) conn.from >> res.timesBuilt >> isNonDeterministic >> res.startTime >> res.stopTime; - if (conn.version >= ServeProto::Version{2, 6}) { - auto builtOutputs = ServeProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + + if (conn.version >= ServeProto::Version{2, 8}) { + success.builtOutputs = ServeProto::Serialise>::read(store, conn); + } else if (conn.version >= ServeProto::Version{2, 6}) { + for (auto & [output, realisation] : ServeProto::Serialise::read(store, conn)) { + size_t n = output.find("!"); + if (n == output.npos) + throw Error("Invalid derivation output id %s", output); + success.builtOutputs.insert_or_assign( + output.substr(n + 1), + UnkeyedRealisation{ + StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); + } } res.inner = std::visit( @@ -63,15 +73,30 @@ void ServeProto::Serialise::write( default value for the fields that don't exist in that case. */ auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { conn.to << errorMsg; + if (conn.version >= ServeProto::Version{2, 3}) conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; - if (conn.version >= ServeProto::Version{2, 6}) { - DrvOutputs builtOutputsFullKey; - for (auto & [output, realisation] : builtOutputs) - builtOutputsFullKey.insert_or_assign(realisation.id, realisation); - ServeProto::write(store, conn, builtOutputsFullKey); + + if (conn.version >= ServeProto::Version{2, 8}) { + ServeProto::write(store, conn, builtOutputs); + } else if (conn.version >= ServeProto::Version{2, 6}) { + // Old clients read `builtOutputs` as a `StringMap` keyed + // by `sha256:!` with JSON-encoded + // realisations. The derivation hash no longer exists, but + // old clients only extract `outputName` and `outPath`, so + // a dummy hash suffices. + StringMap sm; + for (auto & [outputName, realisation] : builtOutputs) { + auto dummyId = Hash::dummy.to_string(HashFormat::Base16, true) + "!" + outputName; + nlohmann::json j; + j["id"] = dummyId; + j["outPath"] = realisation.outPath.to_string(); + sm[dummyId] = j.dump(); + } + ServeProto::write(store, conn, sm); } }; + std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -158,4 +183,82 @@ void ServeProto::Serialise::write( } } +UnkeyedRealisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (conn.version < ServeProto::Version{2, 8}) { + throw Error( + "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", + conn.version.major, + conn.version.minor); + } + + auto outPath = ServeProto::Serialise::read(store, conn); + auto signatures = ServeProto::Serialise>::read(store, conn); + + return UnkeyedRealisation{ + .outPath = std::move(outPath), + .signatures = std::move(signatures), + }; +} + +void ServeProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) +{ + if (conn.version < ServeProto::Version{2, 8}) { + throw Error( + "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", + conn.version.major, + conn.version.minor); + } + ServeProto::write(store, conn, info.outPath); + ServeProto::write(store, conn, info.signatures); +} + +DrvOutput ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (conn.version < ServeProto::Version{2, 8}) { + throw Error( + "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", + conn.version.major, + conn.version.minor); + } + + auto drvPath = ServeProto::Serialise::read(store, conn); + auto outputName = ServeProto::Serialise::read(store, conn); + + return DrvOutput{ + .drvPath = std::move(drvPath), + .outputName = std::move(outputName), + }; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) +{ + if (conn.version < ServeProto::Version{2, 8}) { + throw Error( + "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", + conn.version.major, + conn.version.minor); + } + ServeProto::write(store, conn, info.drvPath); + ServeProto::write(store, conn, info.outputName); +} + +Realisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto id = ServeProto::Serialise::read(store, conn); + auto unkeyed = ServeProto::Serialise::read(store, conn); + + return Realisation{ + std::move(unkeyed), + std::move(id), + }; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) +{ + ServeProto::write(store, conn, info.id); + ServeProto::write(store, conn, static_cast(info)); +} + } // namespace nix diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 845e7bcb37af..dd38a36bc8cc 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -403,9 +403,8 @@ Store::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore return outputs; auto drv = evalStore.readInvalidDerivation(path); - auto drvHashes = staticOutputHashes(*this, drv); - for (auto & [outputName, hash] : drvHashes) { - auto realisation = queryRealisation(DrvOutput{hash, outputName}); + for (auto & [outputName, _] : drv.outputs) { + auto realisation = queryRealisation(DrvOutput{path, outputName}); if (realisation) { outputs.insert_or_assign(outputName, realisation->outPath); } else { @@ -425,7 +424,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * ev OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw MissingRealisation(printStorePath(path), outName); + throw MissingRealisation(*this, path, outName); result.insert_or_assign(outName, *optOutPath); } return result; diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index da225dc0d1bd..911d4b3f8193 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1968,7 +1968,10 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { .outPath = newInfo.path, }, - DrvOutput{oldinfo->outputHash, outputName}, + DrvOutput{ + .drvPath = drvPath, + .outputName = outputName, + }, }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().isImpure()) { store.signRealisation(thisRealisation); diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 49ab4c36b1ed..3da966a62019 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -8,6 +8,7 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" +#include "nix/util/json-utils.hh" #include #include @@ -20,6 +21,12 @@ const WorkerProto::Version WorkerProto::latest = { .major = 1, .minor = 38, }, + .features = + { + std::string{ + WorkerProto::featureRealisationWithPath, + }, + }, }; const WorkerProto::Version WorkerProto::minimum = { @@ -254,10 +261,19 @@ BuildResult WorkerProto::Serialise::read(const StoreDirConfig & sto res.cpuUser = WorkerProto::Serialise>::read(store, conn); res.cpuSystem = WorkerProto::Serialise>::read(store, conn); } - if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { - auto builtOutputs = WorkerProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + + if (conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + success.builtOutputs = WorkerProto::Serialise>::read(store, conn); + } else if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { + for (auto && [output, realisation] : WorkerProto::Serialise::read(store, conn)) { + size_t n = output.find("!"); + if (n == output.npos) + throw Error("Invalid derivation output id %s", output); + success.builtOutputs.insert_or_assign( + output.substr(n + 1), + UnkeyedRealisation{ + StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); + } } res.inner = std::visit( @@ -296,13 +312,27 @@ void WorkerProto::Serialise::write( WorkerProto::write(store, conn, res.cpuUser); WorkerProto::write(store, conn, res.cpuSystem); } - if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { - DrvOutputs builtOutputsFullKey; - for (auto & [output, realisation] : builtOutputs) - builtOutputsFullKey.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, conn, builtOutputsFullKey); + + if (conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + WorkerProto::write(store, conn, builtOutputs); + } else if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { + // Old clients read `builtOutputs` as a `StringMap` keyed + // by `sha256:!` with JSON-encoded + // realisations. The derivation hash no longer exists, but + // old clients only extract `outputName` and `outPath`, so + // a dummy hash suffices. + StringMap sm; + for (auto & [outputName, realisation] : builtOutputs) { + auto dummyId = Hash::dummy.to_string(HashFormat::Base16, true) + "!" + outputName; + nlohmann::json j; + j["id"] = dummyId; + j["outPath"] = realisation.outPath.to_string(); + sm[dummyId] = j.dump(); + } + WorkerProto::write(store, conn, sm); } }; + std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -395,4 +425,109 @@ void WorkerProto::Serialise::write( } } +UnkeyedRealisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + throw Error( + "the daemon is missing the '%s' protocol feature, needed to understand build trace", + WorkerProto::featureRealisationWithPath); + } + + auto outPath = WorkerProto::Serialise::read(store, conn); + auto signatures = WorkerProto::Serialise>::read(store, conn); + + return UnkeyedRealisation{ + .outPath = std::move(outPath), + .signatures = std::move(signatures), + }; +} + +void WorkerProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) +{ + if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + throw Error( + "the daemon is missing the '%s' protocol feature, needed to understand build trace", + WorkerProto::featureRealisationWithPath); + } + WorkerProto::write(store, conn, info.outPath); + WorkerProto::write(store, conn, info.signatures); +} + +std::optional +WorkerProto::Serialise>::read(const StoreDirConfig & store, ReadConn conn) +{ + if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + // Hack to improve compat + (void) WorkerProto::Serialise::read(store, conn); + return std::nullopt; + } else { + auto temp = readNum(conn.from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return WorkerProto::Serialise::read(store, conn); + default: + throw Error("Invalid optional build trace from remote"); + } + } +} + +void WorkerProto::Serialise>::write( + const StoreDirConfig & store, WriteConn conn, const std::optional & info) +{ + if (!info) { + conn.to << uint8_t{0}; + } else { + conn.to << uint8_t{1}; + WorkerProto::write(store, conn, *info); + } +} + +DrvOutput WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + throw Error( + "the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations", + WorkerProto::featureRealisationWithPath); + } + + auto drvPath = WorkerProto::Serialise::read(store, conn); + auto outputName = WorkerProto::Serialise::read(store, conn); + + return DrvOutput{ + .drvPath = std::move(drvPath), + .outputName = std::move(outputName), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) +{ + if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + throw Error( + "the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations", + WorkerProto::featureRealisationWithPath); + } + WorkerProto::write(store, conn, info.drvPath); + WorkerProto::write(store, conn, info.outputName); +} + +Realisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto id = WorkerProto::Serialise::read(store, conn); + auto unkeyed = WorkerProto::Serialise::read(store, conn); + + return Realisation{ + std::move(unkeyed), + std::move(id), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) +{ + WorkerProto::write(store, conn, info.id); + WorkerProto::write(store, conn, static_cast(info)); +} + } // namespace nix diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index d46f3011a4ed..14a0cad7609f 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -346,13 +346,11 @@ static int main_build_remote(int argc, char ** argv) optResult = std::move(res[0]); } - auto outputHashes = staticOutputHashes(*store, drv); std::set missingRealisations; StorePathSet missingPaths; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { - auto thisOutputHash = outputHashes.at(outputName); - auto thisOutputId = DrvOutput{thisOutputHash, outputName}; + auto thisOutputId = DrvOutput{*drvPath, outputName}; if (!store->queryRealisation(thisOutputId)) { debug("missing output %s", outputName); assert(optResult); @@ -362,7 +360,7 @@ static int main_build_remote(int argc, char ** argv) auto i = success.builtOutputs.find(outputName); assert(i != success.builtOutputs.end()); auto & newRealisation = i->second; - missingRealisations.insert(newRealisation); + missingRealisations.insert({newRealisation, thisOutputId}); missingPaths.insert(newRealisation.outPath); } } diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 6c4bf2e7051d..b0f70be3a9e8 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -163,10 +163,13 @@ StoreWrapper::queryPathInfo(char * path, int base32) } SV * -StoreWrapper::queryRawRealisation(char * outputId) +StoreWrapper::queryRawRealisation(char * drvPath, char * outputName) PPCODE: try { - auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); + auto realisation = THIS->store->queryRealisation(DrvOutput{ + .drvPath = THIS->store->parseStorePath(drvPath), + .outputName = outputName, + }); if (realisation) XPUSHs(sv_2mortal(newSVpv(static_cast(*realisation).dump().c_str(), 0))); else diff --git a/tests/functional/ca/common.sh b/tests/functional/ca/common.sh index dc8e650fd682..10298d9d87dd 100644 --- a/tests/functional/ca/common.sh +++ b/tests/functional/ca/common.sh @@ -1,6 +1,9 @@ # shellcheck shell=bash source ../common.sh +# Need backend to support revamped CA +requireDaemonNewerThan "2.34.0pre20251217" + enableFeatures "ca-derivations" TODO_NixOS diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index 2f6ebcef5c11..022bea5cfced 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -49,14 +49,14 @@ fi clearStore nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore" clearStore -rm -r "$REMOTE_STORE_DIR/realisations" +rm -r "$REMOTE_STORE_DIR/build-trace-v2" nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # There's no easy way to check whether a realisation is present on the local # store − short of manually querying the db, but the build environment doesn't # have the sqlite binary − so we instead push things again, and check that the # realisations have correctly been pushed to the remote store nix copy --to "$REMOTE_STORE" --file ../simple.nix -if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then +if [[ -z "$(ls "$REMOTE_STORE_DIR/build-trace-v2")" ]]; then echo "Realisations not rebuilt" exit 1 fi @@ -71,5 +71,5 @@ buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # Try rebuilding, but remove the realisations from the remote cache to force # using the cachecache clearStore -rm "$REMOTE_STORE_DIR"/realisations/* +rm -r "$REMOTE_STORE_DIR"/build-trace-v2/* buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 From ca42db38a3e16445bf04ee59f4124d1138807ae0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 2 Mar 2026 21:24:05 +0300 Subject: [PATCH 020/555] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index dc1591835cd3..aa5388f63762 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.34.0 +2.35.0 From 855208ba24badfee6e3677090a156394bf020d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 2 Mar 2026 19:22:36 +0100 Subject: [PATCH 021/555] docs: add release notes for build trace rework The previous commit changed CA derivation realisations to be keyed by store path instead of hash modulo, affecting binary cache and wire protocols. Users and tool authors need to understand the impact on binary cache layout, protocol negotiation, and what migration looks like. This documents the key changes: the new build-trace-v2/ cache directory, the split key/value JSON format, worker protocol feature negotiation, serve protocol version bump, and the fact that non-CA users are unaffected. --- doc/manual/rl-next/build-trace-rework.md | 75 ++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 doc/manual/rl-next/build-trace-rework.md diff --git a/doc/manual/rl-next/build-trace-rework.md b/doc/manual/rl-next/build-trace-rework.md new file mode 100644 index 000000000000..fd3541c5b346 --- /dev/null +++ b/doc/manual/rl-next/build-trace-rework.md @@ -0,0 +1,75 @@ +--- +synopsis: "Content-addressed derivations: realisations keyed by store path instead of hash modulo" +issues: [11897] +prs: [12464] +--- + +The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. This affects the **binary cache protocol** and the **wire protocols**. + +### What changed + +Previously, a build trace entry (realisation) was keyed by the **hash modulo** of the derivation. +A SHA-256 hash computed via the complex "derivation hash modulo" algorithm. +This required implementations to understand ATerm serialisation and the full derivation hashing scheme just to look up or store build results. + +Now, build trace entries are keyed by the **regular derivation store path** plus the output name. For example, instead of: + +``` +sha256:ba7816bf8f01...!out +``` + +The key is now: + +``` +/nix/store/abc...-foo.drv^out +``` + +This is simpler, more intuitive, and means that third-party tools implementing CA derivation support (e.g., Hydra) +no longer need to implement the derivation hash modulo algorithm. + +### Binary cache protocol + +- The directory for build traces moved from `realisations/` to `build-trace-v2/`. +- File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. +- The JSON format of build trace entries is now split into `key` and `value` objects: + ```json + { + "key": { + "drvPath": "abc...-foo.drv", + "outputName": "out" + }, + "value": { + "outPath": "xyz...-foo", + "signatures": ["..."] + } + } + ``` + Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. +- The deprecated `dependentRealisations` field has been removed. + +Existing binary caches will need to be re-populated with the new format for CA derivation build traces. +Old build traces at the previous URLs are simply abandoned. +Non-CA builds are unaffected. + +### Wire protocols + +- **Worker protocol**: + A new feature flag `realisation-with-path-not-hash` is negotiated during the handshake. + Clients and daemons that both support this feature use the new binary serialisation for `DrvOutput`, `UnkeyedRealisation`, and related types. + Fallback to older protocol versions gracefully degrades (realisations are unavailable). +- **Serve protocol**: + Bumped from 2.7 to 2.8 with native serialisers for the new types. + Fallback to older protocol versions gracefully degrades in the same way. + +Stable code paths do use the realization fields (`BuildResult::Success::builtOutputs`), but only the output name and outpath parts of that. +For older protocols, we can fake enough of the realisation format to provide those two parts forthat map, which keeps operations like `--print-output-paths` working. + +### Impact + +- **Non-CA derivation users**: No impact. This only affects the experimental `ca-derivations` feature. +- **Binary cache operators**: + Binary caches serving CA derivation build traces will need to be repopulated. + Existing NARs and narinfo files are unaffected. +- **Tool authors**: + Implementations interfacing with the CA derivations protocol are simplified. + The derivation hash modulo algorithm is no longer required to form build trace keys. From 62a78c86b0a16e71c29856d542af5b7fbea94e2d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 2 Mar 2026 20:11:49 +0100 Subject: [PATCH 022/555] release notes 2.34: add entries for additional PRs --- doc/manual/source/release-notes/rl-2.34.md | 49 +++++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/release-notes/rl-2.34.md b/doc/manual/source/release-notes/rl-2.34.md index 854f1d1ed6af..c415041120cd 100644 --- a/doc/manual/source/release-notes/rl-2.34.md +++ b/doc/manual/source/release-notes/rl-2.34.md @@ -153,6 +153,8 @@ When these options are configured, Nix will use this certificate/private key pair to authenticate to the server. +- `nix store gc --dry-run` and `nix-collect-garbage --dry-run` now report the number of paths that would be freed [#15229](https://github.com/NixOS/nix/pull/15229) [#5704](https://github.com/NixOS/nix/issues/5704) + ## Performance improvements - Unpacking tarballs to `~/.cache/nix/tarball-cache-v2` is now multithreaded [#12087](https://github.com/NixOS/nix/pull/12087) @@ -168,9 +170,11 @@ - `nix nar ls` and other NAR listing operations have been optimised further [#15163](https://github.com/NixOS/nix/pull/15163) +- Evaluator hot-path optimizations [#15270](https://github.com/NixOS/nix/pull/15270) [#15271](https://github.com/NixOS/nix/pull/15271) + ## C API Changes -- New store API methods [#14766](https://github.com/NixOS/nix/pull/14766) +- New store API methods [#14766](https://github.com/NixOS/nix/pull/14766) [#14768](https://github.com/NixOS/nix/pull/14768) The C API now includes additional methods: @@ -204,7 +208,7 @@ Due to a bug in how Nix handled Boost.Coroutine2 suspension and resumption, copying from `ssh-ng://` stores would drop the SSH connection for each copied path. This issue has been fixed, which improves performance by avoiding multiple SSH/Nix Worker Protocol handshakes. -- S3 binary caches now use virtual-hosted-style addressing by default [#15208](https://github.com/NixOS/nix/issues/15208) +- S3 binary caches now use virtual-hosted-style addressing by default [#15208](https://github.com/NixOS/nix/issues/15208) [#15216](https://github.com/NixOS/nix/pull/15216) S3 binary caches now use virtual-hosted-style URLs (`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs @@ -243,6 +247,34 @@ When run as root, Nix doesn't run builds via the daemon and is a parent of the forked build processes. Prior versions of Nix failed to preserve the `PR_SET_PDEATHSIG` parent-death signal across `setuid` calls. This could lead to build processes being reparented and continue running in the background. This has been fixed. +- Fix crash when interrupting `--log-format internal-json` [#15335](https://github.com/NixOS/nix/pull/15335) + + Pressing Ctrl-C during `--log-format internal-json` (used by [nix-output-monitor](https://github.com/maralorn/nix-output-monitor)) no longer causes a spurious "Nix crashed. This is a bug." report. + +- Fix percent-encoding in `file://` and `local://` store URIs [#15280](https://github.com/NixOS/nix/pull/15280) + + Store URIs with special characters like `+` in the path (e.g. `file:///tmp/a+b`) no longer incorrectly create percent-encoded directories (e.g. `/tmp/a%2Bb`). + +- Fix crash during tab completion in `nix repl` [#15255](https://github.com/NixOS/nix/pull/15255) + +- Fix "Too many open files" on macOS [#15205](https://github.com/NixOS/nix/pull/15205) + + Nix now raises the open file soft limit to the hard limit at startup, fixing "Too many open files" errors on macOS where the default soft limit is low. + +- `nix develop` no longer fails when `inputs.nixpkgs` has `flake = false` [#15175](https://github.com/NixOS/nix/pull/15175) + +- `builtins.flakeRefToString` no longer fails with "attribute is a thunk" [#15160](https://github.com/NixOS/nix/pull/15160) + +- Fix `QueryPathInfo` throwing on invalid paths in the daemon [#15134](https://github.com/NixOS/nix/pull/15134) + +- `nix-store --generate-binary-cache-key` now fsyncs key files to prevent corruption [#15107](https://github.com/NixOS/nix/pull/15107) + +- Fix `build-hook` setting in `nix.conf` being ignored [#15083](https://github.com/NixOS/nix/pull/15083) + +- Fix empty error messages when builds are cancelled due to a dependency failure [#14972](https://github.com/NixOS/nix/pull/14972) + + When a build fails without `--keep-going`, other in-progress builds are cancelled. Previously, these cancelled builds were incorrectly reported as failed with empty error messages. This affected `buildPathsWithResults` callers such as `nix flake check`. + ## Miscellaneous changes - Content-Encoding decompression is now handled by libcurl [#14324](https://github.com/NixOS/nix/issues/14324) [#15336](https://github.com/NixOS/nix/pull/15336) @@ -254,6 +286,19 @@ - Static builds now support S3 features (`libstore:s3-aws-auth` meson option) [#15076](https://github.com/NixOS/nix/pull/15076) +- Improved package-related error messages [#15349](https://github.com/NixOS/nix/pull/15349) + + Store path context is now rendered in the user-facing `hash^out` format instead of the internal `!out!hash` format. + A misleading error message in `nix-env` that incorrectly blamed content-addressed derivations has been fixed. + +- Improved error message for empty derivation files [#15298](https://github.com/NixOS/nix/pull/15298) + + Parsing an empty `.drv` file (e.g. due to store corruption after an unclean shutdown) now produces a clear error message instead of the cryptic `expected string 'D'`. + +- Relative `file:` paths for tarballs are now rejected with a clear error [#14983](https://github.com/NixOS/nix/pull/14983) + +- Continued progress on the Windows port, including build fixes, CI improvements, and platform abstractions. + ## Contributors This release was made possible by the following 43 contributors: From 5f666eff14529cc3794cc195f4ced876799d247f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 2 Mar 2026 21:18:22 +0300 Subject: [PATCH 023/555] Add release note for GHCR Docker images (cherry picked from commit 5d054fe91ef95650ba7a7925b15ed9acfc0e7006) --- doc/manual/source/release-notes/rl-2.34.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/source/release-notes/rl-2.34.md b/doc/manual/source/release-notes/rl-2.34.md index c415041120cd..335e88ee89a9 100644 --- a/doc/manual/source/release-notes/rl-2.34.md +++ b/doc/manual/source/release-notes/rl-2.34.md @@ -299,6 +299,10 @@ - Continued progress on the Windows port, including build fixes, CI improvements, and platform abstractions. +- Nix docker images are now uploaded to [GHCR](https://github.com/NixOS/nix/pkgs/container/nix) as part of the release process + + Historically, only pre-release builds of `amd64` docker images have been uploaded to ghcr.io with the `latest` tag pointing to the last built image from `master` branch. This has been fixed and going forward, will include the same images as that are built by [Hydra](https://hydra.nixos.org/project/nix) for [arm64](https://hydra.nixos.org/job/nix/maintenance-2.34/dockerImage.aarch64-linux) and [amd64](https://hydra.nixos.org/job/nix/maintenance-2.34/dockerImage.x86_64-linux). Pre-release versions are no longer pushed to the registry. + ## Contributors This release was made possible by the following 43 contributors: From 4f8f581830e74ec7502d97feeea6bdaaaad4349c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 2 Mar 2026 10:37:21 -0500 Subject: [PATCH 024/555] libutil: Reimplement `*stat` functions on Windows for symlink support Reimplement `lstat` and `maybeLstat` from first principles on Windows so they work with symlinks. Properly define `S_IFLNK` and `S_ISLNK`. Use `GetFileAttributesExW` instead of `_wstat64` since the latter doesn't properly detect symlinks (reparse points) on Windows. Key changes: - Add `S_IFLNK` (0120000) and proper `S_ISLNK` macro for Windows - Add `windows::fileTimeToUnixTime` to convert FILETIME to Unix time_t - Add `windows::statFromFileInfo` to populate PosixStat from Windows file attributes - Move `lstat` and `maybeLstat` to platform-specific files (unix/file-system.cc and windows/file-system.cc) - Move `fstat` from file-system.hh to file-system-at.hh, changing signature from `int fd` to `Descriptor fd` for cross-platform support - Windows `lstat` now detects symlinks via FILE_ATTRIBUTE_REPARSE_POINT and sets S_IFLNK in st_mode - Windows `fstat` uses GetFileInformationByHandle Co-authored-by: Sergei Zimmerman --- src/libstore/unix/pathlocks.cc | 2 +- src/libutil/file-system.cc | 42 +-------- .../include/nix/util/file-system-at.hh | 9 ++ src/libutil/include/nix/util/file-system.hh | 53 +++++++++-- src/libutil/unix/file-descriptor.cc | 1 + src/libutil/unix/file-system-at.cc | 16 +++- src/libutil/unix/file-system.cc | 19 ++++ src/libutil/windows/file-system-at.cc | 20 +++++ src/libutil/windows/file-system.cc | 90 +++++++++++++++++++ 9 files changed, 199 insertions(+), 53 deletions(-) diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc index ce2adf74debc..47e33853c920 100644 --- a/src/libstore/unix/pathlocks.cc +++ b/src/libstore/unix/pathlocks.cc @@ -1,5 +1,5 @@ #include "nix/store/pathlocks.hh" -#include "nix/util/file-system.hh" +#include "nix/util/file-system-at.hh" #include "nix/util/util.hh" #include "nix/util/sync.hh" #include "nix/util/signals.hh" diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 5dec2decb39c..d810e4a06c30 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -171,10 +171,8 @@ bool isDirOrInDir(const std::filesystem::path & path, const std::filesystem::pat #ifdef _WIN32 # define STAT _wstat64 -# define LSTAT _wstat64 #else # define STAT stat -# define LSTAT lstat #endif PosixStat stat(const std::filesystem::path & path) @@ -185,54 +183,18 @@ PosixStat stat(const std::filesystem::path & path) return st; } -PosixStat lstat(const std::filesystem::path & path) -{ - PosixStat st; - if (LSTAT(path.c_str(), &st)) - throw SysError("getting status of %s", PathFmt(path)); - return st; -} - -PosixStat fstat(int fd) -{ - PosixStat st; - if ( -#ifdef _WIN32 - _fstat64 -#else - ::fstat -#endif - (fd, &st)) - throw SysError("getting status of fd %d", fd); - return st; -} - std::optional maybeStat(const std::filesystem::path & path) { std::optional st{std::in_place}; if (STAT(path.c_str(), &*st)) { if (errno == ENOENT || errno == ENOTDIR) - st.reset(); - else - throw SysError("getting status of %s", PathFmt(path)); - } - return st; -} - -std::optional maybeLstat(const std::filesystem::path & path) -{ - std::optional st{std::in_place}; - if (LSTAT(path.c_str(), &*st)) { - if (errno == ENOENT || errno == ENOTDIR) - st.reset(); - else - throw SysError("getting status of %s", PathFmt(path)); + return std::nullopt; + throw SysError("getting status of %s", PathFmt(path)); } return st; } #undef STAT -#undef LSTAT bool pathExists(const std::filesystem::path & path) { diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh index 4558f0e0ef88..a8cb17b95e1f 100644 --- a/src/libutil/include/nix/util/file-system-at.hh +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -15,6 +15,7 @@ */ #include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" #include @@ -25,6 +26,14 @@ namespace nix { +/** + * Get status of an open file/directory handle. + * + * @param fd File descriptor/handle + * @throws SystemError on I/O errors. + */ +PosixStat fstat(Descriptor fd); + /** * Read a symlink relative to a directory file descriptor. * diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 58aecade75a2..aae7b7bf37e5 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -30,11 +30,20 @@ * Polyfill for MinGW * * Windows does in fact support symlinks, but the C runtime interfaces predate this. - * - * @todo get rid of this, and stop using `stat` when we want `lstat` too. - */ + * We define S_IFLNK and S_ISLNK so that our lstat implementation can properly + * indicate symlinks by setting these mode bits when it detects a reparse point. + */ +#ifndef S_IFLNK +# ifndef _WIN32 +# error "S_IFLNK should be defined on non-Windows platforms" +# endif +# define S_IFLNK 0120000 +#endif #ifndef S_ISLNK -# define S_ISLNK(m) false +# ifndef _WIN32 +# error "S_ISLNK should be defined on non-Windows platforms" +# endif +# define S_ISLNK(m) (((m) & S_IFMT) == S_IFLNK) #endif namespace nix { @@ -97,6 +106,38 @@ using PosixStat = #endif ; +#ifdef _WIN32 +namespace windows { + +/** + * Convert Windows FILETIME to Unix time_t. + */ +time_t fileTimeToUnixTime(const FILETIME & ft); + +/** + * Fill a PosixStat structure from file attributes and timestamps. + * + * @param dwFileAttributes File attributes (FILE_ATTRIBUTE_*) + * @param ftCreationTime Creation time + * @param ftLastAccessTime Last access time + * @param ftLastWriteTime Last write time + * @param nFileSizeHigh High 32 bits of file size + * @param nFileSizeLow Low 32 bits of file size + * @param nNumberOfLinks Number of hard links (default 1) + */ +void statFromFileInfo( + PosixStat & st, + DWORD dwFileAttributes, + const FILETIME & ftCreationTime, + const FILETIME & ftLastAccessTime, + const FILETIME & ftLastWriteTime, + DWORD nFileSizeHigh, + DWORD nFileSizeLow, + DWORD nNumberOfLinks = 1); + +} // namespace windows +#endif + /** * Get status of `path`. */ @@ -105,10 +146,6 @@ PosixStat lstat(const std::filesystem::path & path); * Get status of `path` following symlinks. */ PosixStat stat(const std::filesystem::path & path); -/** - * Get status of an open file descriptor. - */ -PosixStat fstat(int fd); /** * `lstat` the given path if it exists. * @return std::nullopt if the path doesn't exist, or an optional containing the result of `lstat` otherwise diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index 05ec38a67cc5..f2c69f2a17c9 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -1,4 +1,5 @@ #include "nix/util/file-system.hh" +#include "nix/util/file-system-at.hh" #include "nix/util/signals.hh" #include "nix/util/finally.hh" #include "nix/util/serialise.hh" diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index 739cac0177ca..fee985fd5d65 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -89,10 +89,9 @@ void unix::fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t }); } - struct ::stat st; - /* Possible since https://github.com/torvalds/linux/commit/55815f70147dcfa3ead5738fd56d3574e2e3c1c2 (3.6) */ - if (::fstat(pathFd.get(), &st) == -1) - throw SysError("statting '%s' relative to parent directory via O_PATH file descriptor", path.rel()); + /* Possible to use with O_PATH fd since + * https://github.com/torvalds/linux/commit/55815f70147dcfa3ead5738fd56d3574e2e3c1c2 (3.6) */ + auto st = fstat(pathFd.get()); if (S_ISLNK(st.st_mode)) throw SysError(EOPNOTSUPP, "can't change mode of symlink %s", PathFmt(descriptorToPath(dirFd) / path.rel())); @@ -222,4 +221,13 @@ OsString readLinkAt(Descriptor dirFd, const CanonPath & path) } } +PosixStat fstat(Descriptor fd) +{ + PosixStat st; + if (::fstat(fd, &st)) { + throw SysError([&] { return HintFmt("getting status of %s", PathFmt(descriptorToPath(fd))); }); + } + return st; +} + } // namespace nix diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index 9423b76a1e9c..e709c5298a0d 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -77,6 +77,25 @@ std::filesystem::path defaultTempDir() return getEnvNonEmpty("TMPDIR").value_or("/tmp"); } +PosixStat lstat(const std::filesystem::path & path) +{ + PosixStat st; + if (::lstat(path.c_str(), &st)) + throw SysError("getting status of %s", PathFmt(path)); + return st; +} + +std::optional maybeLstat(const std::filesystem::path & path) +{ + std::optional st{std::in_place}; + if (::lstat(path.c_str(), &*st)) { + if (errno == ENOENT || errno == ENOTDIR) + return std::nullopt; + throw SysError("getting status of %s", PathFmt(path)); + } + return st; +} + void setWriteTime( const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) { diff --git a/src/libutil/windows/file-system-at.cc b/src/libutil/windows/file-system-at.cc index ebbe840007d0..a10dedbb886f 100644 --- a/src/libutil/windows/file-system-at.cc +++ b/src/libutil/windows/file-system-at.cc @@ -210,6 +210,26 @@ bool isReparsePoint(HANDLE handle) } // namespace windows +PosixStat fstat(Descriptor fd) +{ + BY_HANDLE_FILE_INFORMATION info; + if (!GetFileInformationByHandle(fd, &info)) + throw WinError("getting file information for %s", PathFmt(descriptorToPath(fd))); + + PosixStat st; + windows::statFromFileInfo( + st, + info.dwFileAttributes, + info.ftCreationTime, + info.ftLastAccessTime, + info.ftLastWriteTime, + info.nFileSizeHigh, + info.nFileSizeLow, + info.nNumberOfLinks); + + return st; +} + AutoCloseFD openFileEnsureBeneathNoSymlinks( Descriptor dirFd, const CanonPath & path, ACCESS_MASK desiredAccess, ULONG createOptions, ULONG createDisposition) { diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index c739ed041384..3e9680ff2469 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -6,6 +6,13 @@ #include #include +#include + +// Verify that our S_IFLNK polyfill doesn't conflict with Windows file type constants +static_assert((S_IFLNK & S_IFMT) == S_IFLNK, "S_IFLNK must fit within S_IFMT mask"); +static_assert(S_IFLNK != S_IFDIR, "S_IFLNK must not equal S_IFDIR"); +static_assert(S_IFLNK != S_IFREG, "S_IFLNK must not equal S_IFREG"); +static_assert(S_IFLNK != S_IFCHR, "S_IFLNK must not equal S_IFCHR"); namespace nix { @@ -105,4 +112,87 @@ std::filesystem::path descriptorToPath(Descriptor handle) return std::filesystem::path{std::wstring{buf.data(), dw}}; } +time_t windows::fileTimeToUnixTime(const FILETIME & ft) +{ + ULARGE_INTEGER ull; + ull.LowPart = ft.dwLowDateTime; + ull.HighPart = ft.dwHighDateTime; + + // file_clock on Windows matches FILETIME's epoch (1601-01-01) and resolution (100ns) + auto fileTP = std::chrono::file_clock::time_point{std::chrono::file_clock::duration{ull.QuadPart}}; + + auto sysTP = std::chrono::clock_cast(fileTP); + return std::chrono::system_clock::to_time_t(sysTP); +} + +void windows::statFromFileInfo( + PosixStat & st, + DWORD dwFileAttributes, + const FILETIME & ftCreationTime, + const FILETIME & ftLastAccessTime, + const FILETIME & ftLastWriteTime, + DWORD nFileSizeHigh, + DWORD nFileSizeLow, + DWORD nNumberOfLinks) +{ + memset(&st, 0, sizeof(st)); + + /* Determine file type */ + if (dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + st.st_mode = S_IFLNK | 0777; + } else if (dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + st.st_mode = S_IFDIR | 0755; + } else { + st.st_mode = S_IFREG | 0644; + } + + /* File size (only meaningful for regular files) */ + st.st_size = (static_cast(nFileSizeHigh) << 32) | nFileSizeLow; + + /* Timestamps */ + st.st_atime = fileTimeToUnixTime(ftLastAccessTime); + st.st_mtime = fileTimeToUnixTime(ftLastWriteTime); + st.st_ctime = fileTimeToUnixTime(ftCreationTime); + + st.st_nlink = nNumberOfLinks; + /* https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/stat-functions?view=msvc-170 + matches the regular windows stat functions. */ + st.st_uid = 0; + st.st_gid = 0; +} + +static PosixStat statFromFileInfo(const WIN32_FILE_ATTRIBUTE_DATA & attrData) +{ + PosixStat st; + windows::statFromFileInfo( + st, + attrData.dwFileAttributes, + attrData.ftCreationTime, + attrData.ftLastAccessTime, + attrData.ftLastWriteTime, + attrData.nFileSizeHigh, + attrData.nFileSizeLow); + return st; +} + +PosixStat lstat(const std::filesystem::path & path) +{ + WIN32_FILE_ATTRIBUTE_DATA attrData; + if (!GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &attrData)) + throw WinError("getting status of %s", PathFmt(path)); + return statFromFileInfo(attrData); +} + +std::optional maybeLstat(const std::filesystem::path & path) +{ + WIN32_FILE_ATTRIBUTE_DATA attrData; + if (!GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &attrData)) { + auto lastError = GetLastError(); + if (lastError == ERROR_FILE_NOT_FOUND || lastError == ERROR_PATH_NOT_FOUND) + return std::nullopt; + throw WinError(lastError, "getting status of %s", PathFmt(path)); + } + return statFromFileInfo(attrData); +} + } // namespace nix From 6633c55f8a1f8b9943f9c5bd18693ad4f9c03b6f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 2 Mar 2026 20:40:10 +0100 Subject: [PATCH 025/555] fix(libexpr-c): pass valid EvalState to primop callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Due to an erroneous cast, the wrong pointer was passed to these callbacks, leading to a crash. We now create a lightweight temporary EvalState wrapper on the stack in each callback bridge. This also eliminates the need for unsafe_new_with_self for EvalState construction. Co-authored-by: Jörg Thalheim --- doc/manual/rl-next/fix-primop-eval-state.md | 10 +++++ src/libexpr-c/nix_api_expr.cc | 16 ++++--- src/libexpr-c/nix_api_expr_internal.h | 9 ++-- src/libexpr-c/nix_api_external.cc | 13 ++---- src/libexpr-c/nix_api_external.h | 1 + src/libexpr-c/nix_api_value.cc | 3 +- src/libexpr-tests/nix_api_expr.cc | 46 +++++++++++++++++++++ src/libexpr-tests/nix_api_external.cc | 40 ++++++++++++++++++ 8 files changed, 116 insertions(+), 22 deletions(-) create mode 100644 doc/manual/rl-next/fix-primop-eval-state.md diff --git a/doc/manual/rl-next/fix-primop-eval-state.md b/doc/manual/rl-next/fix-primop-eval-state.md new file mode 100644 index 000000000000..1a3bc287538f --- /dev/null +++ b/doc/manual/rl-next/fix-primop-eval-state.md @@ -0,0 +1,10 @@ +--- +synopsis: "C API: Fix `EvalState` pointer passed to primop callbacks" +prs: [15300, 15383] +--- + +The `EvalState *` passed to C API primop callbacks was incorrectly pointing to +the internal `nix::EvalState` rather than the C API wrapper struct. This caused +a segfault when the callback used the pointer with C API functions such as +`nix_alloc_value()`. The same issue affected `printValueAsJSON` and +`printValueAsXML` callbacks on external values. diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 61edf9bc3515..97680ac6bfe7 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -181,15 +181,13 @@ EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder if (context) context->last_err_code = NIX_OK; try { - return unsafe_new_with_self([&](auto * self) { - return EvalState{ - .fetchSettings = std::move(builder->fetchSettings), - .settings = std::move(builder->settings), - .statePtr = std::make_shared( - builder->lookupPath, builder->store, self->fetchSettings, self->settings), - .state = *self->statePtr, - }; - }); + auto fetchSettings = std::make_unique(std::move(builder->fetchSettings)); + auto settings = std::make_unique(std::move(builder->settings)); + auto ownedState = + std::make_shared(builder->lookupPath, builder->store, *fetchSettings, *settings); + auto & stateRef = *ownedState; + void * p = ::operator new(sizeof(EvalState), static_cast(alignof(EvalState))); + return new (p) EvalState{stateRef, std::move(fetchSettings), std::move(settings), std::move(ownedState)}; } NIXC_CATCH_ERRS_NULL } diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 633599fb412b..b38aeaf7b498 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -1,6 +1,8 @@ #ifndef NIX_API_EXPR_INTERNAL_H #define NIX_API_EXPR_INTERNAL_H +#include + #include "nix/fetchers/fetch-settings.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" @@ -22,10 +24,11 @@ struct nix_eval_state_builder struct EvalState { - nix::fetchers::Settings fetchSettings; - nix::EvalSettings settings; - std::shared_ptr statePtr; nix::EvalState & state; + // Owned resources; null for temporary wrappers created in C API callbacks. + std::unique_ptr ownedFetchSettings; + std::unique_ptr ownedSettings; + std::shared_ptr ownedState; }; struct BindingsBuilder diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index ee500619fc12..a874d9a0861e 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -137,7 +137,8 @@ class NixCExternalValue : public nix::ExternalValueBase } nix_string_context ctx{context}; nix_string_return res{""}; - desc.printValueAsJSON(v, (EvalState *) &state, strict, &ctx, copyToStore, &res); + EvalState wrapper{state}; + desc.printValueAsJSON(v, &wrapper, strict, &ctx, copyToStore, &res); if (res.str.empty()) { return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore); } @@ -160,15 +161,9 @@ class NixCExternalValue : public nix::ExternalValueBase return nix::ExternalValueBase::printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); } nix_string_context ctx{context}; + EvalState wrapper{state}; desc.printValueAsXML( - v, - (EvalState *) &state, - strict, - location, - &doc, - &ctx, - &drvsSeen, - *reinterpret_cast(&pos)); + v, &wrapper, strict, location, &doc, &ctx, &drvsSeen, *reinterpret_cast(&pos)); } virtual ~NixCExternalValue() override {}; diff --git a/src/libexpr-c/nix_api_external.h b/src/libexpr-c/nix_api_external.h index 96c479d57697..1ae4c22f89bc 100644 --- a/src/libexpr-c/nix_api_external.h +++ b/src/libexpr-c/nix_api_external.h @@ -145,6 +145,7 @@ typedef struct NixCExternalValueDesc * Optional, the default is to throw an error * @todo The mechanisms for this call are incomplete. There are no C * bindings to work with XML, pathsets and positions. + * This callback also has no test coverage. * @param[in] self the void* passed to nix_create_external_value * @param[in] state The evaluator state * @param[in] strict boolean Whether to force the value before printing diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index c6b96213ea98..589ebf9a8ec2 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -105,7 +105,8 @@ static void nix_c_primop_wrapper( nix_value * external_arg = new_nix_value(args[i], state.mem); external_args.push_back(external_arg); } - f(userdata, &ctx, (EvalState *) &state, external_args.data(), vTmpPtr); + EvalState wrapper{state}; + f(userdata, &ctx, &wrapper, external_args.data(), vTmpPtr); if (ctx.last_err_code != NIX_OK) { if (ctx.last_err_code == NIX_ERR_RECOVERABLE) { diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index 56363038d0ca..c3a3f2dd53b1 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -476,6 +476,52 @@ TEST_F(nix_api_expr_test, nix_expr_primop_nix_err_key_conversion) nix_gc_decref(ctx, result); } +static void +primop_alloc_value(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ + assert(context); + assert(state); + + // Regression test: nix_c_primop_wrapper previously cast the inner + // nix::EvalState* directly to EvalState* (C wrapper). C API functions + // like nix_alloc_value() then accessed state->state at the wrong offset, + // causing a segfault. + nix_value * v = nix_alloc_value(context, state); + assert(v != nullptr); + nix_init_int(context, v, 42); + nix_copy_value(context, ret, v); + nix_gc_decref(nullptr, v); +} + +TEST_F(nix_api_expr_test, nix_primop_can_use_state_in_callback) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_alloc_value, 1, "allocValue", nullptr, "test alloc_value in callback", nullptr); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * dummy = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, dummy, 0); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, primopValue, dummy, result); + assert_ctx_ok(); + + auto r = nix_get_int(ctx, result); + ASSERT_EQ(42, r); + + nix_gc_decref(ctx, dummy); + nix_gc_decref(ctx, result); + nix_gc_decref(ctx, primopValue); + nix_gc_decref(ctx, primop); +} + TEST_F(nix_api_expr_test, nix_value_call_multi_no_args) { nix_value * n = nix_alloc_value(ctx, state); diff --git a/src/libexpr-tests/nix_api_external.cc b/src/libexpr-tests/nix_api_external.cc index ec19f1212e9e..885b9c1d2dd3 100644 --- a/src/libexpr-tests/nix_api_external.cc +++ b/src/libexpr-tests/nix_api_external.cc @@ -66,4 +66,44 @@ TEST_F(nix_api_expr_test, nix_expr_eval_external) nix_state_free(stateFn); } +static void print_value_as_json_using_state( + void * self, EvalState * state, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res) +{ + // Regression test: same cast bug as in nix_c_primop_wrapper (see primop_alloc_value). + nix_value * v = nix_alloc_value(nullptr, state); + assert(v != nullptr); + nix_gc_decref(nullptr, v); + + nix_set_string_return(res, "42"); +} + +TEST_F(nix_api_expr_test, nix_external_printValueAsJSON_can_use_state) +{ + NixCExternalValueDesc desc{}; + desc.print = [](void *, nix_printer *) {}; + desc.showType = [](void *, nix_string_return *) {}; + desc.typeOf = [](void *, nix_string_return *) {}; + desc.printValueAsJSON = print_value_as_json_using_state; + + ExternalValue * val = nix_create_external_value(ctx, &desc, nullptr); + assert_ctx_ok(); + nix_init_external(ctx, value, val); + assert_ctx_ok(); + + nix_value * toJsonFn = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "builtins.toJSON", ".", toJsonFn); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + nix_value_call(ctx, state, toJsonFn, value, result); + assert_ctx_ok(); + + std::string json_str; + nix_get_string(ctx, result, OBSERVE_STRING(json_str)); + ASSERT_EQ("42", json_str); + + nix_gc_decref(ctx, result); + nix_gc_decref(ctx, toJsonFn); +} + } // namespace nixC From faa16841b6fbd594f5449135d92b4ebfee062ab5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 23:39:07 +0000 Subject: [PATCH 026/555] build(deps): bump actions/download-artifact from 7.0.0 to 8.0.0 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 7.0.0 to 8.0.0. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/37930b1c2abaa49bbe596cd826c3c89aef350131...70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: 8.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e37193966a25..67f8c517835b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,7 +182,7 @@ jobs: steps: - uses: actions/checkout@v6 - name: Download installer tarball - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: installer-${{matrix.os}} path: out @@ -232,7 +232,7 @@ jobs: repository: NixOS/flake-regressions-data path: flake-regressions/tests - name: Download installer tarball - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: installer-linux path: out From 539c6a1aaf9ecbb538ca8803256542dcfa036578 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 23:39:12 +0000 Subject: [PATCH 027/555] build(deps): bump actions/upload-artifact from 6 to 7 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 6 to 7. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e37193966a25..5df158911b7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,13 +125,13 @@ jobs: cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY if: ${{ matrix.instrumented }} - name: Upload coverage reports - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: coverage-reports path: coverage-reports/ if: ${{ matrix.instrumented }} - name: Upload installer tarball - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: installer-${{matrix.os}} path: out/* From d7245ff8aca13e1ac3810199706a7fe1e4114643 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Tue, 20 Jan 2026 13:55:09 -0500 Subject: [PATCH 028/555] protocol: update JSON output with structured `Signature` type This commit updates the JSON output with a Signature type containing keyName and sig fields. JSON parsing accepts both formats for backwards compatibility. Co-authored-by: John Ericson --- doc/manual/rl-next/build-trace-rework.md | 8 ++- doc/manual/source/SUMMARY.md.in | 1 + .../protocols/json/build-trace-entry.md | 8 +-- doc/manual/source/protocols/json/meson.build | 5 +- .../json/schema/build-result-v1.yaml | 2 +- ...ld-trace-entry-v2 => build-trace-entry-v3} | 0 ...ntry-v2.yaml => build-trace-entry-v3.yaml} | 13 ++--- .../source/protocols/json/schema/nar-info-v3 | 1 + .../protocols/json/schema/signature-v2.yaml | 33 ++++++++++++ .../json/schema/signature-v2/simple.json | 4 ++ .../json/schema/store-object-info-v2 | 1 - .../json/schema/store-object-info-v3 | 1 + ...info-v2.yaml => store-object-info-v3.yaml} | 14 +++--- .../protocols/json/schema/store-v1.yaml | 4 +- doc/manual/source/protocols/json/signature.md | 9 ++++ .../protocols/json/store-object-info.md | 16 +++--- src/json-schema-checks/meson.build | 47 ++++++++++-------- src/json-schema-checks/signature/simple.json | 4 ++ .../data/common-protocol/realisation.bin | Bin 560 -> 600 bytes .../data/common-protocol/realisation.json | 10 +++- .../data/dummy-store/one-flat-file.json | 2 +- .../data/nar-info/json-3/impure.json | 31 ++++++++++++ .../data/nar-info/json-3/pure.json | 14 ++++++ .../data/path-info/json-3/empty_impure.json | 12 +++++ .../data/path-info/json-3/empty_pure.json | 8 +++ .../data/path-info/json-3/impure.json | 27 ++++++++++ .../data/path-info/json-3/pure.json | 14 ++++++ .../with-signature-structured.json | 15 ++++++ .../data/realisation/with-signature.json | 2 +- .../with-structured-signature.json | 15 ++++++ .../data/serve-protocol/realisation-2.8.json | 10 +++- .../data/serve-protocol/realisation.bin | Bin 560 -> 600 bytes .../data/serve-protocol/realisation.json | 10 +++- .../unkeyed-realisation-2.8.json | 10 +++- .../unkeyed-valid-path-info-2.3.json | 4 +- .../unkeyed-valid-path-info-2.4.json | 14 ++++-- ...sation-realisation-with-path-not-hash.json | 10 +++- .../data/worker-protocol/realisation.bin | Bin 560 -> 600 bytes .../data/worker-protocol/realisation.json | 10 +++- ...sation-realisation-with-path-not-hash.json | 10 +++- .../unkeyed-valid-path-info-1.15.json | 4 +- .../worker-protocol/valid-path-info-1.15.json | 4 +- .../worker-protocol/valid-path-info-1.16.json | 16 ++++-- .../ca-drv/store-after.json | 2 +- .../ca-drv/substituter.json | 2 +- .../issue-11928/store-after.json | 2 +- .../issue-11928/substituter.json | 4 +- .../single/substituter.json | 2 +- .../with-dep/substituter.json | 4 +- src/libstore-tests/nar-info.cc | 38 ++++++++++++++ src/libstore-tests/path-info.cc | 40 +++++++++++++++ src/libstore-tests/realisation.cc | 33 ++++++++---- src/libstore/include/nix/store/path-info.hh | 2 + src/libstore/path-info.cc | 14 ++++-- src/libutil/signature/local-keys.cc | 13 ++++- 55 files changed, 475 insertions(+), 104 deletions(-) rename doc/manual/source/protocols/json/schema/{build-trace-entry-v2 => build-trace-entry-v3} (100%) rename doc/manual/source/protocols/json/schema/{build-trace-entry-v2.yaml => build-trace-entry-v3.yaml} (92%) create mode 120000 doc/manual/source/protocols/json/schema/nar-info-v3 create mode 100644 doc/manual/source/protocols/json/schema/signature-v2.yaml create mode 100644 doc/manual/source/protocols/json/schema/signature-v2/simple.json delete mode 120000 doc/manual/source/protocols/json/schema/store-object-info-v2 create mode 120000 doc/manual/source/protocols/json/schema/store-object-info-v3 rename doc/manual/source/protocols/json/schema/{store-object-info-v2.yaml => store-object-info-v3.yaml} (97%) create mode 100644 doc/manual/source/protocols/json/signature.md create mode 100644 src/json-schema-checks/signature/simple.json create mode 100644 src/libstore-tests/data/nar-info/json-3/impure.json create mode 100644 src/libstore-tests/data/nar-info/json-3/pure.json create mode 100644 src/libstore-tests/data/path-info/json-3/empty_impure.json create mode 100644 src/libstore-tests/data/path-info/json-3/empty_pure.json create mode 100644 src/libstore-tests/data/path-info/json-3/impure.json create mode 100644 src/libstore-tests/data/path-info/json-3/pure.json create mode 100644 src/libstore-tests/data/realisation/with-signature-structured.json create mode 100644 src/libstore-tests/data/realisation/with-structured-signature.json diff --git a/doc/manual/rl-next/build-trace-rework.md b/doc/manual/rl-next/build-trace-rework.md index fd3541c5b346..1c4271b90378 100644 --- a/doc/manual/rl-next/build-trace-rework.md +++ b/doc/manual/rl-next/build-trace-rework.md @@ -40,7 +40,7 @@ no longer need to implement the derivation hash modulo algorithm. }, "value": { "outPath": "xyz...-foo", - "signatures": ["..."] + "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] } } ``` @@ -64,6 +64,12 @@ Non-CA builds are unaffected. Stable code paths do use the realization fields (`BuildResult::Success::builtOutputs`), but only the output name and outpath parts of that. For older protocols, we can fake enough of the realisation format to provide those two parts forthat map, which keeps operations like `--print-output-paths` working. +### Structured signatures + +[Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. +`nix path-info --json --json-format 3` opts into the new version for this command. +JSON parsing accepts both the old string format and new structured format for backwards compatibility. + ### Impact - **Non-CA derivation users**: No impact. This only affects the experimental `ca-derivations` feature. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 18d24788ca1c..2f78cb2d33db 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -125,6 +125,7 @@ - [Hash](protocols/json/hash.md) - [Content Address](protocols/json/content-address.md) - [Store Path](protocols/json/store-path.md) + - [Signature](protocols/json/signature.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation/index.md) - [Derivation Options](protocols/json/derivation/options.md) diff --git a/doc/manual/source/protocols/json/build-trace-entry.md b/doc/manual/source/protocols/json/build-trace-entry.md index 9eea93712601..2f8a4dbeeff6 100644 --- a/doc/manual/source/protocols/json/build-trace-entry.md +++ b/doc/manual/source/protocols/json/build-trace-entry.md @@ -1,21 +1,21 @@ -{{#include build-trace-entry-v2-fixed.md}} +{{#include build-trace-entry-v3-fixed.md}} ## Examples ### Simple build trace entry ```json -{{#include schema/build-trace-entry-v2/simple.json}} +{{#include schema/build-trace-entry-v3/simple.json}} ``` ### Build trace entry with signature ```json -{{#include schema/build-trace-entry-v2/with-signature.json}} +{{#include schema/build-trace-entry-v3/with-structured-signature.json}} ``` diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c7c48c4ed6e2..36e9220c028b 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -13,11 +13,12 @@ schemas = [ 'hash-v1', 'content-address-v1', 'store-path-v1', - 'store-object-info-v2', + 'signature-v2', + 'store-object-info-v3', 'derivation-v4', 'derivation-options-v1', 'deriving-path-v1', - 'build-trace-entry-v2', + 'build-trace-entry-v3', 'build-result-v1', 'store-v1', ] diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml index 7730f3d93999..d0d8d8a0c742 100644 --- a/doc/manual/source/protocols/json/schema/build-result-v1.yaml +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -83,7 +83,7 @@ properties: description: | A mapping from output names to their build trace entries. additionalProperties: - "$ref": "build-trace-entry-v2.yaml#/$defs/value" + "$ref": "build-trace-entry-v3.yaml#/$defs/value" failure: type: object diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v2 b/doc/manual/source/protocols/json/schema/build-trace-entry-v3 similarity index 100% rename from doc/manual/source/protocols/json/schema/build-trace-entry-v2 rename to doc/manual/source/protocols/json/schema/build-trace-entry-v3 diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml similarity index 92% rename from doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml rename to doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml index 2e8cd07da2a8..c3a27d2a6e0b 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml @@ -1,5 +1,5 @@ "$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v2.json" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v3.json" title: Build Trace Entry description: | A record of a successful build outcome for a specific derivation output. @@ -12,14 +12,17 @@ description: | > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) > and subject to change. - Verision history: + ## Version History - Version 1: Original format - Version 2: - - Use `drvPath` not `drvHash` to refer to derivation in a more conventional way. - Remove `dependentRealisations` + + - Version 3: + - Use `drvPath` not `drvHash` to refer to derivation in a more conventional way. - Separate into `key` and `value` + - Use 2nd version of signatures format (objects, not strings) type: object required: @@ -77,6 +80,4 @@ additionalProperties: false description: | A set of cryptographic signatures attesting to the authenticity of this build trace entry. items: - type: string - title: Signature - description: A single cryptographic signature + "$ref": "signature-v2.yaml" diff --git a/doc/manual/source/protocols/json/schema/nar-info-v3 b/doc/manual/source/protocols/json/schema/nar-info-v3 new file mode 120000 index 000000000000..378215b4156b --- /dev/null +++ b/doc/manual/source/protocols/json/schema/nar-info-v3 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/nar-info/json-3 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/signature-v2.yaml b/doc/manual/source/protocols/json/schema/signature-v2.yaml new file mode 100644 index 000000000000..e2b25dbc6fc0 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/signature-v2.yaml @@ -0,0 +1,33 @@ +"$schema": "http://json-schema.org/draft-07/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/signature-v2.json" +title: Signature +description: | + A cryptographic signature along with the name of the key that produced it. + + This schema describes the JSON representation of signatures as used in various Nix JSON APIs. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > and subject to change. + + ## Version History + + - Version 1: Colon-separated string in the format `:` + + - Version 2: Structured object with `keyName` and `sig` fields + +type: object +required: + - keyName + - sig +properties: + keyName: + type: string + title: Key Name + description: The name of the key used to produce this signature + sig: + type: string + title: Signature Data + description: The raw signature bytes, Base64-encoded diff --git a/doc/manual/source/protocols/json/schema/signature-v2/simple.json b/doc/manual/source/protocols/json/schema/signature-v2/simple.json new file mode 100644 index 000000000000..abf69dfa0f92 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/signature-v2/simple.json @@ -0,0 +1,4 @@ +{ + "keyName": "cache.nixos.org-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +} diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v2 b/doc/manual/source/protocols/json/schema/store-object-info-v2 deleted file mode 120000 index 36ca7f13db94..000000000000 --- a/doc/manual/source/protocols/json/schema/store-object-info-v2 +++ /dev/null @@ -1 +0,0 @@ -../../../../../../src/libstore-tests/data/path-info/json-2 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v3 b/doc/manual/source/protocols/json/schema/store-object-info-v3 new file mode 120000 index 000000000000..d5dae48fcf69 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/path-info/json-3 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml similarity index 97% rename from doc/manual/source/protocols/json/schema/store-object-info-v2.yaml rename to doc/manual/source/protocols/json/schema/store-object-info-v3.yaml index 3ed7e99e28d8..e0be716ef781 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml @@ -1,6 +1,6 @@ "$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v2.json" -title: Store Object Info v2 +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v3.json" +title: Store Object Info v3 description: | Information about a [store object](@docroot@/store/store-object.md). @@ -50,10 +50,10 @@ $defs: properties: version: type: integer - const: 2 - title: Format version (must be 2) + const: 3 + title: Format version (must be 3) description: | - Must be `2`. + Must be `3`. This is a guard that allows us to continue evolving this format. Here is the rough version history: @@ -63,6 +63,8 @@ $defs: - Version 2: Use structured JSON type for `ca` + - Version 3: Use structured JSON type for `signatures` + path: "$ref": "./store-path-v1.yaml" title: Store Path @@ -174,7 +176,7 @@ $defs: > This is an "impure" field that may not be included in certain contexts. items: - type: string + "$ref": "./signature-v2.yaml" # Computed closure fields closureSize: diff --git a/doc/manual/source/protocols/json/schema/store-v1.yaml b/doc/manual/source/protocols/json/schema/store-v1.yaml index ebe61d9cb227..b8ed3c8cc1c1 100644 --- a/doc/manual/source/protocols/json/schema/store-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-v1.yaml @@ -37,7 +37,7 @@ properties: - contents properties: info: - "$ref": "./store-object-info-v2.yaml#/$defs/impure" + "$ref": "./store-object-info-v3.yaml#/$defs/impure" title: Store Object Info description: | Metadata about the [store object](@docroot@/store/store-object.md) including hash, size, references, etc. @@ -70,7 +70,7 @@ properties: "^[A-Za-z0-9+/]{43}=$": type: object additionalProperties: - "$ref": "./build-trace-entry-v2.yaml#/$defs/value" + "$ref": "./build-trace-entry-v3.yaml#/$defs/value" additionalProperties: false "$defs": diff --git a/doc/manual/source/protocols/json/signature.md b/doc/manual/source/protocols/json/signature.md new file mode 100644 index 000000000000..363b4ef11e0a --- /dev/null +++ b/doc/manual/source/protocols/json/signature.md @@ -0,0 +1,9 @@ +{{#include signature-v2-fixed.md}} + +## Examples + +### Simple signature + +```json +{{#include schema/signature-v2/simple.json}} +``` diff --git a/doc/manual/source/protocols/json/store-object-info.md b/doc/manual/source/protocols/json/store-object-info.md index 4ad83de00b3f..ea1cc3a9d223 100644 --- a/doc/manual/source/protocols/json/store-object-info.md +++ b/doc/manual/source/protocols/json/store-object-info.md @@ -1,45 +1,45 @@ -{{#include store-object-info-v2-fixed.md}} +{{#include store-object-info-v3-fixed.md}} ## Examples ### Minimal store object (content-addressed) ```json -{{#include schema/store-object-info-v2/pure.json}} +{{#include schema/store-object-info-v3/pure.json}} ``` ### Store object with impure fields ```json -{{#include schema/store-object-info-v2/impure.json}} +{{#include schema/store-object-info-v3/impure.json}} ``` ### Minimal store object (empty) ```json -{{#include schema/store-object-info-v2/empty_pure.json}} +{{#include schema/store-object-info-v3/empty_pure.json}} ``` ### Store object with all impure fields ```json -{{#include schema/store-object-info-v2/empty_impure.json}} +{{#include schema/store-object-info-v3/empty_impure.json}} ``` ### NAR info (minimal) ```json -{{#include schema/nar-info-v2/pure.json}} +{{#include schema/nar-info-v3/pure.json}} ``` ### NAR info (with binary cache fields) ```json -{{#include schema/nar-info-v2/impure.json}} +{{#include schema/nar-info-v3/impure.json}} ``` diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index a4d946160969..46078deec510 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -51,6 +51,13 @@ schemas = [ 'simple.json', ], }, + { + 'stem' : 'signature', + 'schema' : schema_dir / 'signature-v2.yaml', + 'files' : [ + 'simple.json', + ], + }, { 'stem' : 'deriving-path', 'schema' : schema_dir / 'deriving-path-v1.yaml', @@ -62,10 +69,10 @@ schemas = [ }, { 'stem' : 'build-trace-entry', - 'schema' : schema_dir / 'build-trace-entry-v2.yaml', + 'schema' : schema_dir / 'build-trace-entry-v3.yaml', 'files' : [ 'simple.json', - 'with-signature.json', + 'with-structured-signature.json', ], }, { @@ -151,20 +158,20 @@ schemas += [ # Match overall { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml', + 'schema' : schema_dir / 'store-object-info-v3.yaml', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'impure.json', - 'json-2' / 'empty_pure.json', - 'json-2' / 'empty_impure.json', + 'json-3' / 'pure.json', + 'json-3' / 'impure.json', + 'json-3' / 'empty_pure.json', + 'json-3' / 'empty_impure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml', + 'schema' : schema_dir / 'store-object-info-v3.yaml', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'impure.json', + 'json-3' / 'pure.json', + 'json-3' / 'impure.json', ], }, { @@ -179,32 +186,32 @@ schemas += [ # Match exact variant { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/base', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'empty_pure.json', + 'json-3' / 'pure.json', + 'json-3' / 'empty_pure.json', ], }, { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/impure', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/impure', 'files' : [ - 'json-2' / 'impure.json', - 'json-2' / 'empty_impure.json', + 'json-3' / 'impure.json', + 'json-3' / 'empty_impure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/base', 'files' : [ - 'json-2' / 'pure.json', + 'json-3' / 'pure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/narInfo', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/narInfo', 'files' : [ - 'json-2' / 'impure.json', + 'json-3' / 'impure.json', ], }, ] diff --git a/src/json-schema-checks/signature/simple.json b/src/json-schema-checks/signature/simple.json new file mode 100644 index 000000000000..abf69dfa0f92 --- /dev/null +++ b/src/json-schema-checks/signature/simple.json @@ -0,0 +1,4 @@ +{ + "keyName": "cache.nixos.org-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +} diff --git a/src/libstore-tests/data/common-protocol/realisation.bin b/src/libstore-tests/data/common-protocol/realisation.bin index 44885cbdafdab24143f74aeef0ef65649339efb5..cbd942ed477a3f77de93e970f3033b22d456dfb8 100644 GIT binary patch delta 92 zcmdnMa)V{U4#w#dcW#xeR?1GT^h?Z5RkBh_EKW&N(orhTOb2o%H!^xop1{Z}XKSle UtAnhpuspR0Lm4|rS!^u>0LmI2jQ{`u delta 52 zcmcb?vVmp74#xb6JGZhaB^IZoSxshS3Yffrk&VmNR!K*xuspR0F2n{Dimhb;0NcC| A@c;k- diff --git a/src/libstore-tests/data/common-protocol/realisation.json b/src/libstore-tests/data/common-protocol/realisation.json index 034d620306f0..ad1ae3ab5f4e 100644 --- a/src/libstore-tests/data/common-protocol/realisation.json +++ b/src/libstore-tests/data/common-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/dummy-store/one-flat-file.json b/src/libstore-tests/data/dummy-store/one-flat-file.json index 804bbf07da6f..f7563abe6a68 100644 --- a/src/libstore-tests/data/dummy-store/one-flat-file.json +++ b/src/libstore-tests/data/dummy-store/one-flat-file.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/nar-info/json-3/impure.json b/src/libstore-tests/data/nar-info/json-3/impure.json new file mode 100644 index 000000000000..eb6d4d48b83b --- /dev/null +++ b/src/libstore-tests/data/nar-info/json-3/impure.json @@ -0,0 +1,31 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "compression": "xz", + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "downloadSize": 4029176, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ], + "storeDir": "/nix/store", + "ultimate": true, + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", + "version": 3 +} diff --git a/src/libstore-tests/data/nar-info/json-3/pure.json b/src/libstore-tests/data/nar-info/json-3/pure.json new file mode 100644 index 000000000000..32fc916489e1 --- /dev/null +++ b/src/libstore-tests/data/nar-info/json-3/pure.json @@ -0,0 +1,14 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/empty_impure.json b/src/libstore-tests/data/path-info/json-3/empty_impure.json new file mode 100644 index 000000000000..47d5030318b3 --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/empty_impure.json @@ -0,0 +1,12 @@ +{ + "ca": null, + "deriver": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/empty_pure.json b/src/libstore-tests/data/path-info/json-3/empty_pure.json new file mode 100644 index 000000000000..6cc032782a4b --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/empty_pure.json @@ -0,0 +1,8 @@ +{ + "ca": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/impure.json b/src/libstore-tests/data/path-info/json-3/impure.json new file mode 100644 index 000000000000..f7ec907dcef3 --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/impure.json @@ -0,0 +1,27 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ], + "storeDir": "/nix/store", + "ultimate": true, + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/pure.json b/src/libstore-tests/data/path-info/json-3/pure.json new file mode 100644 index 000000000000..32fc916489e1 --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/pure.json @@ -0,0 +1,14 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/realisation/with-signature-structured.json b/src/libstore-tests/data/realisation/with-signature-structured.json new file mode 100644 index 000000000000..a9efc97ca9e6 --- /dev/null +++ b/src/libstore-tests/data/realisation/with-signature-structured.json @@ -0,0 +1,15 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] + } +} diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json index cca29ef3e45e..7952e43697b0 100644 --- a/src/libstore-tests/data/realisation/with-signature.json +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -4,7 +4,7 @@ "outputName": "foo" }, "value": { - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", "signatures": [ "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ] diff --git a/src/libstore-tests/data/realisation/with-structured-signature.json b/src/libstore-tests/data/realisation/with-structured-signature.json new file mode 100644 index 000000000000..1b98554a0d11 --- /dev/null +++ b/src/libstore-tests/data/realisation/with-structured-signature.json @@ -0,0 +1,15 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] + } +} diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.json b/src/libstore-tests/data/serve-protocol/realisation-2.8.json index 1977d8485b99..d6dde7327670 100644 --- a/src/libstore-tests/data/serve-protocol/realisation-2.8.json +++ b/src/libstore-tests/data/serve-protocol/realisation-2.8.json @@ -6,8 +6,14 @@ "value": { "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } } diff --git a/src/libstore-tests/data/serve-protocol/realisation.bin b/src/libstore-tests/data/serve-protocol/realisation.bin index 44885cbdafdab24143f74aeef0ef65649339efb5..cbd942ed477a3f77de93e970f3033b22d456dfb8 100644 GIT binary patch delta 92 zcmdnMa)V{U4#w#dcW#xeR?1GT^h?Z5RkBh_EKW&N(orhTOb2o%H!^xop1{Z}XKSle UtAnhpuspR0Lm4|rS!^u>0LmI2jQ{`u delta 52 zcmcb?vVmp74#xb6JGZhaB^IZoSxshS3Yffrk&VmNR!K*xuspR0F2n{Dimhb;0NcC| A@c;k- diff --git a/src/libstore-tests/data/serve-protocol/realisation.json b/src/libstore-tests/data/serve-protocol/realisation.json index 034d620306f0..ad1ae3ab5f4e 100644 --- a/src/libstore-tests/data/serve-protocol/realisation.json +++ b/src/libstore-tests/data/serve-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json index 2c2d264f9a17..9c8277922c5a 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json @@ -1,7 +1,13 @@ { "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json index 0f593f4248d1..55bbd81bf14a 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json @@ -9,7 +9,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -23,6 +23,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json index 9c1fa3134d49..5c0f436e1f60 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json @@ -11,7 +11,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": { @@ -27,11 +27,17 @@ ], "registrationTime": null, "signatures": [ - "fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "fake-sig-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "fake-sig-2", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json index 1977d8485b99..d6dde7327670 100644 --- a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json +++ b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json @@ -6,8 +6,14 @@ "value": { "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } } diff --git a/src/libstore-tests/data/worker-protocol/realisation.bin b/src/libstore-tests/data/worker-protocol/realisation.bin index 44885cbdafdab24143f74aeef0ef65649339efb5..cbd942ed477a3f77de93e970f3033b22d456dfb8 100644 GIT binary patch delta 92 zcmdnMa)V{U4#w#dcW#xeR?1GT^h?Z5RkBh_EKW&N(orhTOb2o%H!^xop1{Z}XKSle UtAnhpuspR0Lm4|rS!^u>0LmI2jQ{`u delta 52 zcmcb?vVmp74#xb6JGZhaB^IZoSxshS3Yffrk&VmNR!K*xuspR0F2n{Dimhb;0NcC| A@c;k- diff --git a/src/libstore-tests/data/worker-protocol/realisation.json b/src/libstore-tests/data/worker-protocol/realisation.json index 034d620306f0..ad1ae3ab5f4e 100644 --- a/src/libstore-tests/data/worker-protocol/realisation.json +++ b/src/libstore-tests/data/worker-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json index 2c2d264f9a17..9c8277922c5a 100644 --- a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json +++ b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json @@ -1,7 +1,13 @@ { "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json index 9cc53c6804e9..fb2fbeef0c1d 100644 --- a/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json +++ b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json @@ -9,7 +9,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -23,6 +23,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json index 427c286ddfbc..e62fd7a236bb 100644 --- a/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json @@ -10,7 +10,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -26,6 +26,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json index 2c377145a10c..8bffd4410f6c 100644 --- a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json @@ -10,7 +10,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": true, - "version": 2 + "version": 3 }, { "ca": null, @@ -24,12 +24,18 @@ ], "registrationTime": 23423, "signatures": [ - "fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "fake-sig-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "fake-sig-2", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": { @@ -48,6 +54,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json index 3271cb985987..ce95439d6e01 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json @@ -30,7 +30,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json index f10653a3a7f3..c596ae7f88fa 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json @@ -30,7 +30,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json index c4d3901236c2..6263797a71eb 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json @@ -30,7 +30,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json index ecd228214691..41b1238679c2 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json @@ -36,7 +36,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } }, "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": { @@ -58,7 +58,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/single/substituter.json b/src/libstore-tests/data/worker-substitution/single/substituter.json index f22d4c7dfbf1..55e16a6d38f8 100644 --- a/src/libstore-tests/data/worker-substitution/single/substituter.json +++ b/src/libstore-tests/data/worker-substitution/single/substituter.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/with-dep/substituter.json b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json index 3f2994dfb455..765340031047 100644 --- a/src/libstore-tests/data/worker-substitution/with-dep/substituter.json +++ b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } }, "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": { @@ -47,7 +47,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index 9b0f6018cdeb..9c8a93e16b77 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -31,6 +31,16 @@ class NarInfoTestV2 : public CharacterizationTest, public LibStoreTest } }; +class NarInfoTestV3 : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "nar-info" / "json-3"; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / (testStem + ".json"); + } +}; + static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { auto info = NarInfo::makeFromCA( @@ -122,6 +132,31 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) JSON_READ_TEST_V2(STEM, PURE) \ JSON_WRITE_TEST_V2(STEM, PURE) +#define JSON_READ_TEST_V3(STEM, PURE) \ + TEST_F(NarInfoTestV3, NarInfo_##STEM##_from_json) \ + { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + auto got = UnkeyedNarInfo::fromJSON(nullptr, encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } + +#define JSON_WRITE_TEST_V3(STEM, PURE) \ + TEST_F(NarInfoTestV3, NarInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> json { return makeNarInfo(*store, PURE).toJSON(nullptr, PURE, PathInfoJsonFormat::V3); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +#define JSON_TEST_V3(STEM, PURE) \ + JSON_READ_TEST_V3(STEM, PURE) \ + JSON_WRITE_TEST_V3(STEM, PURE) + JSON_TEST_V1(pure, false) JSON_TEST_V1(impure, true) @@ -131,4 +166,7 @@ JSON_READ_TEST_V1(pure_noversion, false) JSON_TEST_V2(pure, false) JSON_TEST_V2(impure, true) +JSON_TEST_V3(pure, false) +JSON_TEST_V3(impure, true) + } // namespace nix diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index 97ad4b270ade..9141d06cd671 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -30,6 +30,16 @@ class PathInfoTestV2 : public CharacterizationTest, public LibStoreTest } }; +class PathInfoTestV3 : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "path-info" / "json-3"; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / (testStem + ".json"); + } +}; + static UnkeyedValidPathInfo makeEmpty() { return { @@ -129,6 +139,31 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo JSON_READ_TEST_V2(STEM, OBJ) \ JSON_WRITE_TEST_V2(STEM, OBJ, PURE) +#define JSON_READ_TEST_V3(STEM, OBJ) \ + TEST_F(PathInfoTestV3, PathInfo_##STEM##_from_json) \ + { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON(nullptr, encoded); \ + auto expected = OBJ; \ + ASSERT_EQ(got, expected); \ + }); \ + } + +#define JSON_WRITE_TEST_V3(STEM, OBJ, PURE) \ + TEST_F(PathInfoTestV3, PathInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> json { return OBJ.toJSON(nullptr, PURE, PathInfoJsonFormat::V3); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +#define JSON_TEST_V3(STEM, OBJ, PURE) \ + JSON_READ_TEST_V3(STEM, OBJ) \ + JSON_WRITE_TEST_V3(STEM, OBJ, PURE) + JSON_TEST_V1(empty_pure, makeEmpty(), false) JSON_TEST_V1(empty_impure, makeEmpty(), true) JSON_TEST_V1(pure, makeFull(*store, false), false) @@ -142,6 +177,11 @@ JSON_TEST_V2(empty_impure, makeEmpty(), true) JSON_TEST_V2(pure, makeFull(*store, false), false) JSON_TEST_V2(impure, makeFull(*store, true), true) +JSON_TEST_V3(empty_pure, makeEmpty(), false) +JSON_TEST_V3(empty_impure, makeEmpty(), true) +JSON_TEST_V3(pure, makeFull(*store, false), false) +JSON_TEST_V3(impure, makeFull(*store, true), true) + TEST_F(PathInfoTestV2, PathInfo_full_shortRefs) { ValidPathInfo it = makeFullKeyed(*store, true); diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index 9c0ebbe8abf9..1bc7708e7c41 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -54,6 +54,20 @@ Realisation simple{ }, }; +Realisation withSignature{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + }, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, + .outputName = "foo", + }, +}; + INSTANTIATE_TEST_SUITE_P( RealisationJSON, RealisationJsonTest, @@ -63,15 +77,16 @@ INSTANTIATE_TEST_SUITE_P( simple, }, std::pair{ - "with-signature", - [&] { - auto r = simple; - // FIXME actually sign properly - r.signatures = { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - }; - return r; - }(), + "with-signature-structured", + withSignature, })); +/** + * Old signature format (string) should still be parseable. + */ +TEST_F(RealisationTest, with_signature_from_json) +{ + readJsonTest("with-signature", withSignature); +} + } // namespace nix diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index 44840b0e5e45..c94a0d0936f0 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -22,6 +22,8 @@ enum class PathInfoJsonFormat { V1 = 1, /// New format with structured hashes and store path base names V2 = 2, + /// New format with structured signatures + V3 = 3, }; /** diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 1c631cf147fa..1ffd4d52fbd8 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -15,8 +15,10 @@ PathInfoJsonFormat parsePathInfoJsonFormat(uint64_t version) return PathInfoJsonFormat::V1; case 2: return PathInfoJsonFormat::V2; + case 3: + return PathInfoJsonFormat::V3; default: - throw Error("unsupported path info JSON format version %d; supported versions are 1 and 2", version); + throw Error("unsupported path info JSON format version %d; supported versions are 1, 2 and 3", version); } } @@ -211,7 +213,13 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig * store, bool includeImpureInf jsonObject["ultimate"] = ultimate; - jsonObject["signatures"] = sigs; + if (format == PathInfoJsonFormat::V3) { + jsonObject["signatures"] = sigs; + } else { + auto & sigsObj = jsonObject["signatures"] = json::array(); + for (auto & sig : sigs) + sigsObj.push_back(sig.to_string()); + } } return jsonObject; @@ -313,7 +321,7 @@ UnkeyedValidPathInfo adl_serializer::from_json(const json void adl_serializer::to_json(json & json, const UnkeyedValidPathInfo & c) { - json = c.toJSON(nullptr, true, PathInfoJsonFormat::V2); + json = c.toJSON(nullptr, true, PathInfoJsonFormat::V3); } ValidPathInfo adl_serializer::from_json(const json & json0) diff --git a/src/libutil/signature/local-keys.cc b/src/libutil/signature/local-keys.cc index 51f94cee006a..8afa3eb728f2 100644 --- a/src/libutil/signature/local-keys.cc +++ b/src/libutil/signature/local-keys.cc @@ -178,12 +178,21 @@ bool verifyDetached(std::string_view data, const Signature & sig, const PublicKe namespace nlohmann { void adl_serializer::to_json(json & j, const Signature & s) { - j = s.to_string(); + j = { + {"keyName", s.keyName}, + {"sig", base64::encode(std::as_bytes(std::span{s.sig}))}, + }; } Signature adl_serializer::from_json(const json & j) { - return Signature::parse(getString(j)); + if (j.is_string()) + return Signature::parse(getString(j)); + auto obj = getObject(j); + return Signature{ + .keyName = getString(valueAt(obj, "keyName")), + .sig = base64::decode(getString(valueAt(obj, "sig"))), + }; } } // namespace nlohmann From 3df8dbc07c6b1f14c1b234a20761f909ca1c47fc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 2 Mar 2026 10:22:40 -0500 Subject: [PATCH 029/555] libutil: Add dupDescriptor function Add a cross-platform function for duplicating file descriptors: - Unix: uses fcntl(F_DUPFD_CLOEXEC) - Windows: uses DuplicateHandle This is useful when code needs to take ownership of a borrowed descriptor. --- src/libutil/include/nix/util/file-descriptor.hh | 8 ++++++++ src/libutil/unix/file-descriptor.cc | 8 ++++++++ src/libutil/windows/file-descriptor.cc | 9 +++++++++ 3 files changed, 25 insertions(+) diff --git a/src/libutil/include/nix/util/file-descriptor.hh b/src/libutil/include/nix/util/file-descriptor.hh index 1e0d726adbad..fbfe5e4f8a66 100644 --- a/src/libutil/include/nix/util/file-descriptor.hh +++ b/src/libutil/include/nix/util/file-descriptor.hh @@ -285,6 +285,14 @@ public: void startFsync() const; }; +/** + * Duplicate a file descriptor. + * + * Returns a new file descriptor that refers to the same open file + * description as the original. + */ +AutoCloseFD dupDescriptor(Descriptor fd); + class Pipe { public: diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index f2c69f2a17c9..75e7e12c876e 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -56,6 +56,14 @@ size_t write(Descriptor fd, std::span buffer, bool allowInterru return static_cast(n); } +AutoCloseFD dupDescriptor(Descriptor fd) +{ + int newFd = fcntl(fd, F_DUPFD_CLOEXEC, 0); + if (newFd == -1) + throw SysError("duplicating file descriptor"); + return AutoCloseFD{newFd}; +} + ////////////////////////////////////////////////////////////////////// void Pipe::create() diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 726859135193..0020623e31a5 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -68,6 +68,15 @@ size_t write(Descriptor fd, std::span buffer, bool allowInterru return static_cast(n); } +AutoCloseFD dupDescriptor(Descriptor fd) +{ + HANDLE newHandle; + if (!DuplicateHandle(GetCurrentProcess(), fd, GetCurrentProcess(), &newHandle, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + throw WinError("duplicating handle"); + } + return AutoCloseFD{newHandle}; +} + ////////////////////////////////////////////////////////////////////// void Pipe::create() From 72abbc4164893bba7bf7d28e2be59960de4078b2 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 17:17:00 -0500 Subject: [PATCH 030/555] libstore: use `urlPathToPath` for store URL path parsing `store-registration.hh` constructed a `std::filesystem::path` directly from `percentDecode(uri)`, which broke on Windows because file:// URLs like `/C:/foo` lack a drive-letter root. Replace with `splitString` + `urlPathToPath`, which handles Windows drive-letter stripping. --- src/libstore-tests/nix_api_store.cc | 5 +++-- src/libstore/include/nix/store/store-registration.hh | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 9bfbf8453a9e..b029b2b42599 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -251,8 +251,9 @@ TEST_F(nix_api_util_context, nix_store_real_path_relocated) TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) { - Store * store = - nix_store_open(ctx, nix::fmt("file://%s/binary-cache", nix::createTempDir().string()).c_str(), nullptr); + auto tmpDir = nix::createTempDir() / "binary-cache"; + auto url = nix::ParsedURL{.scheme = "file", .path = nix::pathToUrlPath(tmpDir)}; + Store * store = nix_store_open(ctx, url.to_string().c_str(), nullptr); assert_ctx_ok(); ASSERT_NE(store, nullptr); diff --git a/src/libstore/include/nix/store/store-registration.hh b/src/libstore/include/nix/store/store-registration.hh index 51db33e12854..e805523e08f2 100644 --- a/src/libstore/include/nix/store/store-registration.hh +++ b/src/libstore/include/nix/store/store-registration.hh @@ -66,8 +66,11 @@ struct Implementations .experimentalFeature = TConfig::experimentalFeature(), .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { if constexpr (std::is_constructible_v) { - std::filesystem::path path = percentDecode(uri); - return make_ref(path.empty() ? std::filesystem::path{} : canonPath(path), params); + auto path = + uri.empty() + ? std::filesystem::path{} + : canonPath(urlPathToPath(splitString>(percentDecode(uri), "/"))); + return make_ref(std::move(path), params); } else if constexpr (std::is_constructible_v) { return make_ref(parseURL(concatStrings(scheme, "://", uri)), params); } else if constexpr (std::is_constructible_v) { From 08efb2b4541dbd218c7a395822a99a54e2374be9 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 17:18:31 -0500 Subject: [PATCH 031/555] tests: skip symlink-dependent nix_api tests on Windows --- src/libstore-tests/nix_api_store.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index b029b2b42599..fc9b50d2d777 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -212,6 +212,9 @@ TEST_F(nix_api_store_test, nix_store_real_path) TEST_F(nix_api_util_context, nix_store_real_path_relocated) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine/Windows does not support symlinks needed for local store gcroots"; +#endif auto tmp = nix::createTempDir(); auto storeRoot = (tmp / "store").string(); auto stateDir = (tmp / "state").string(); From 9534543860ce92fb3de0ddf49cf385401c50830a Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 17:36:44 -0500 Subject: [PATCH 032/555] tests: skip and fix nix_api_store tests for Windows/Wine --- .../include/nix/expr/tests/nix_api_expr.hh | 9 ++++---- .../include/nix/store/tests/nix_api_store.hh | 22 +++++++++---------- src/libstore-tests/meson.build | 15 ++++++++----- src/libstore-tests/nix_api_store.cc | 15 +++++++++++++ 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh b/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh index 376761d76325..4aa55688def2 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh @@ -12,21 +12,22 @@ class nix_api_expr_test : public nix_api_store_test { protected: - nix_api_expr_test() + void SetUp() override { + nix_api_store_test::SetUp(); nix_libexpr_init(ctx); state = nix_state_create(nullptr, nullptr, store); value = nix_alloc_value(nullptr, state); } - ~nix_api_expr_test() + void TearDown() override { nix_gc_decref(nullptr, value); nix_state_free(state); } - EvalState * state; - nix_value * value; + EvalState * state = nullptr; + nix_value * value = nullptr; }; } // namespace nixC diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index bb9e5a3038fd..15df329cb2b1 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -79,23 +79,23 @@ class nix_api_store_test : public nix_api_store_test_base { public: nix_api_store_test() - : nix_api_store_test_base{} - { - init_local_store(); - }; + : nix_api_store_test_base{} {}; - ~nix_api_store_test() override + void SetUp() override { - nix_store_free(store); +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif + store = open_local_store(); } - Store * store; - -protected: - void init_local_store() + ~nix_api_store_test() override { - store = open_local_store(); + if (store) + nix_store_free(store); } + + Store * store = nullptr; }; } // namespace nixC diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 2d12b14d3cb5..3d448678b975 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -105,14 +105,19 @@ this_exe = executable( cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [], ) +test_env = { + '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', +} + +if host_machine.system() != 'windows' + test_env += {'NIX_REMOTE' : meson.current_build_dir() / 'test-home' / 'store'} +endif + test( meson.project_name(), this_exe, - env : { - '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', - 'HOME' : meson.current_build_dir() / 'test-home', - 'NIX_REMOTE' : meson.current_build_dir() / 'test-home' / 'store', - }, + env : test_env, protocol : 'gtest', ) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index fc9b50d2d777..468eac1b957f 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -301,6 +301,9 @@ class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base void SetUp() override { nix_api_store_test_base::SetUp(); +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; @@ -356,6 +359,9 @@ class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base TEST_F(nix_api_store_test_base, build_from_json) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // FIXME get rid of these nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; @@ -403,6 +409,9 @@ TEST_F(nix_api_store_test_base, build_from_json) TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when the system is invalid nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; @@ -448,6 +457,9 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system) TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when the builder fails nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; @@ -493,6 +505,9 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails) TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when builder succeeds but produces no output nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; From f17d9dee9e1584922c93020720e11bf38bd0b5ad Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 17:49:36 -0500 Subject: [PATCH 033/555] tests: use empty temp file instead of `/dev/null` in machines test --- src/libstore-tests/machines.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index f0a334c1b613..c4cbf47e4e22 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -185,7 +185,10 @@ TEST(machines, getMachinesWithCorrectFileReference) TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { - std::filesystem::path path = "/dev/null"; + auto tmpDir = nix::createTempDir(); + AutoDelete delTmpDir(tmpDir); + auto path = tmpDir / "empty-machines"; + nix::writeFile(path, ""); ASSERT_TRUE(std::filesystem::exists(path)); auto actual = Machine::parseConfig({}, "@" + path.string()); From 70c6e0a399a0efa523367788aa2ef2ec85863e75 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 17:49:36 -0500 Subject: [PATCH 034/555] tests: use Windows-absolute paths in LocalStore/LocalOverlayStore config tests --- src/libstore-tests/local-overlay-store.cc | 18 ++++++++++++++---- src/libstore-tests/local-store.cc | 20 +++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/libstore-tests/local-overlay-store.cc b/src/libstore-tests/local-overlay-store.cc index c207564255f3..4c12f6256aba 100644 --- a/src/libstore-tests/local-overlay-store.cc +++ b/src/libstore-tests/local-overlay-store.cc @@ -6,24 +6,34 @@ namespace nix { TEST(LocalOverlayStore, constructConfig_rootQueryParam) { +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif LocalOverlayStoreConfig config{ "", { { "root", - "/foo/bar", + std::string{root}, }, }, }; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalOverlayStore, constructConfig_rootPath) { - LocalOverlayStoreConfig config{"/foo/bar", {}}; +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif + LocalOverlayStoreConfig config{std::string{root}, {}}; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } } // namespace nix diff --git a/src/libstore-tests/local-store.cc b/src/libstore-tests/local-store.cc index 554d42efc811..a094774f8fc8 100644 --- a/src/libstore-tests/local-store.cc +++ b/src/libstore-tests/local-store.cc @@ -12,24 +12,34 @@ namespace nix { TEST(LocalStore, constructConfig_rootQueryParam) { +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif LocalStoreConfig config{ "", { { "root", - "/foo/bar", + std::string{root}, }, }, }; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalStore, constructConfig_rootPath) { - LocalStoreConfig config{"/foo/bar", {}}; - - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif + LocalStoreConfig config{std::string{root}, {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalStore, constructConfig_to_string) From 5bedf61b83fbf075dcc22abd08b522abadd11816 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 19:21:35 -0500 Subject: [PATCH 035/555] tests: fix two assertion crashes during Windows test teardown --- src/libstore-test-support/https-store.cc | 4 +++- .../include/nix/store/tests/https-store.hh | 2 ++ src/libstore-tests/outputs-spec.cc | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libstore-test-support/https-store.cc b/src/libstore-test-support/https-store.cc index 79548f61f3fe..549e3ce36172 100644 --- a/src/libstore-test-support/https-store.cc +++ b/src/libstore-test-support/https-store.cc @@ -60,7 +60,7 @@ void HttpsBinaryCacheStoreTest::SetUp() openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"}); // clang-format on -#ifndef _WIN32 /* FIXME: Can't yet start processes on windows */ +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ auto args = serverArgs(); serverPid = startProcess( [&] { @@ -91,7 +91,9 @@ void HttpsBinaryCacheStoreTest::SetUp() void HttpsBinaryCacheStoreTest::TearDown() { +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ serverPid.kill(); +#endif delTmpDir.reset(); testFileTransferSettings.reset(); } diff --git a/src/libstore-test-support/include/nix/store/tests/https-store.hh b/src/libstore-test-support/include/nix/store/tests/https-store.hh index 3aa2cd3455e9..19ecd62e9f49 100644 --- a/src/libstore-test-support/include/nix/store/tests/https-store.hh +++ b/src/libstore-test-support/include/nix/store/tests/https-store.hh @@ -65,7 +65,9 @@ protected: std::filesystem::path tmpDir, cacheDir; std::filesystem::path caCert, caKey, serverCert, serverKey; std::filesystem::path clientCert, clientKey; +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ Pid serverPid; +#endif uint16_t port = 8443; std::shared_ptr localCacheStore; diff --git a/src/libstore-tests/outputs-spec.cc b/src/libstore-tests/outputs-spec.cc index 1fac222fccba..d791709ac258 100644 --- a/src/libstore-tests/outputs-spec.cc +++ b/src/libstore-tests/outputs-spec.cc @@ -33,7 +33,16 @@ class ExtendedOutputsSpecTest : public virtual CharacterizationTest TEST_F(OutputsSpecTest, no_empty_names) { +#ifndef _WIN32 ASSERT_DEATH(OutputsSpec::Names{StringSet{}}, ""); +#else + // ASSERT_DEATH relies on CreateProcess, which under Wine leaks the + // child's stderr into the parent console. Test both sides of the + // invariant without a death test instead. + ASSERT_FALSE(OutputsSpec::parseOpt("")); + OutputsSpec::Names names{StringSet{"out"}}; + ASSERT_FALSE(names.empty()); +#endif } #define TEST_DONT_PARSE(NAME, STR) \ From 6bedefea9e610ba739efcc7a3ab8f9cb6618da7b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 3 Mar 2026 16:59:49 +0300 Subject: [PATCH 036/555] Improve formatting of various errors and warnings Removes unnecessary `error: warning:` prefixes by using .message() and not .msg(). Also includes attempt counts in the filetransfer warnings. Removes a redundant trace from the filetransfer errors with FileTransferError (which is already descriptive enough) and contains the same information basically. --- src/libcmd/installables.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libmain/common-args.cc | 2 +- src/libstore/daemon.cc | 2 +- src/libstore/filetransfer.cc | 34 +++++++++++++++++++++++++--------- src/libstore/local-store.cc | 2 +- src/libstore/store-api.cc | 2 +- src/nix/develop.cc | 7 ++++--- 8 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index b9366a275904..8f8309bd96d7 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -400,7 +400,7 @@ void completeFlakeRefWithFragment( } } } catch (Error & e) { - warn(e.msg()); + logWarning(e.info()); } } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b415a13428b6..065174433110 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -51,7 +51,7 @@ DownloadFileResult downloadFile( res = getFileTransfer()->download(request); } catch (FileTransferError & e) { if (cached) { - warn("%s; using cached version", e.msg()); + warn("%s; using cached version", e.message()); return useCached(); } else throw; diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 6055ec0e7524..42b814ee5b8b 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -48,7 +48,7 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) globalConfig.set(name, value); } catch (UsageError & e) { if (!getRoot().completions) - warn(e.what()); + logWarning(e.info()); } }}, .completer = diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 3e5eac517008..38eaad189122 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -305,7 +305,7 @@ struct ClientSettings "ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { - warn(e.what()); + logWarning(e.info()); } } } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index ba86e9ebbf2c..099f436ae65b 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -221,6 +221,8 @@ struct curlFileTransfer : public FileTransfer done = true; try { std::rethrow_exception(ex); + } catch (FileTransferError & e) { + /* Already descriptive enough. */ } catch (nix::Error & e) { /* Add more context to the error message. */ e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.uri.to_string()); @@ -628,7 +630,7 @@ struct curlFileTransfer : public FileTransfer debug( "finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", - request.noun(), + Uncolored(request.noun()), request.uri, code, httpStatus, @@ -723,14 +725,14 @@ struct curlFileTransfer : public FileTransfer Interrupted, std::move(response), "%s of '%s' was interrupted", - request.noun(), + Uncolored(request.noun()), request.uri) : httpStatus != 0 ? FileTransferError( err, std::move(response), "unable to %s '%s': HTTP error %d%s", - request.verb(), + Uncolored(request.verb()), request.uri, httpStatus, code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) @@ -738,7 +740,7 @@ struct curlFileTransfer : public FileTransfer err, std::move(response), "unable to %s '%s': %s (%d) %s", - request.verb(), + Uncolored(request.verb()), request.uri, curl_easy_strerror(code), code, @@ -753,10 +755,24 @@ struct curlFileTransfer : public FileTransfer int ms = retryTimeMs * std::pow( 2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); - if (writtenToSink) - warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); - else - warn("%s; retrying in %d ms", exc.what(), ms); + + if (writtenToSink) { + warn( + "%s; retrying from offset %d in %d ms (attempt %d/%d)", + exc.message(), + writtenToSink, + ms, + attempt, + fileTransfer.settings.tries); + } else { + warn( + "%s; retrying in %d ms (attempt %d/%d)", + exc.message(), + ms, + attempt, + fileTransfer.settings.tries); + } + errorSink.reset(); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); try { @@ -937,7 +953,7 @@ struct curlFileTransfer : public FileTransfer } for (auto & item : incoming) { - debug("starting %s of %s", item->request.noun(), item->request.uri); + debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.uri); item->init(); curl_multi_add_handle(curlm.get(), item->req); item->active = true; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8631df218418..649ad70dd393 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1407,7 +1407,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (isValidPath(i)) logError(e.info()); else - warn(e.msg()); + logWarning(e.info()); errors = true; } } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 845e7bcb37af..5b5d41623893 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1201,7 +1201,7 @@ static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, return parseDerivation(store, std::move(contents), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { - throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); + throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.message()); } } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d093c7f27ffe..b898e99a4fe3 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -676,9 +676,10 @@ struct CmdDevelop : Common, MixEnvironment // Override SHELL with the one chosen for this environment. // This is to make sure the system shell doesn't leak into the build environment. - setEnv("SHELL", shell.string().c_str()); - // https://github.com/NixOS/nix/issues/5873 - script += fmt("SHELL=%s\n", PathFmt(shell)); + setEnvOs(OS_STR("SHELL"), shell.c_str()); + /* See: https://github.com/NixOS/nix/issues/5873 + Format via .string() and not PathFmt intentionally. */ + script += fmt("SHELL=\"%s\"\n", shell.string()); if (foundInteractive) script += fmt("PATH=\"%s${PATH:+:$PATH}\"\n", std::filesystem::path(shell).parent_path().string()); writeFull(rcFileFd.get(), script); From 5e84b575945f0344867f6294a47b43b46c852e35 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 3 Mar 2026 09:59:50 -0500 Subject: [PATCH 037/555] Introduce `Source::drainInto` with explicit length This is used to deduplicate two spots in the code: - `git::parseBlob` - `parseContents` in `archive.cc` Implementation is a bit more hardened than either of the two originals. --- src/libutil/archive.cc | 13 +------------ src/libutil/git.cc | 12 +----------- src/libutil/include/nix/util/serialise.hh | 12 ++++++++++++ src/libutil/serialise.cc | 15 +++++++++++++++ 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 847e339b6d2d..53aebde70aa1 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -135,18 +135,7 @@ static void parseContents(CreateRegularFileSink & sink, Source & source) return; } - uint64_t left = size; - std::array buf; - - while (left) { - checkInterrupt(); - auto n = buf.size(); - if ((uint64_t) n > left) - n = left; - source(buf.data(), n); - sink({buf.data(), n}); - left -= n; - } + source.drainInto(sink, size); readPadding(size, source); } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 1b6c65c2e01f..2646babfd50c 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -68,17 +68,7 @@ void parseBlob( crf.preallocateContents(size); - unsigned long long left = size; - std::string buf; - buf.reserve(65536); - - while (left) { - checkInterrupt(); - buf.resize(std::min((unsigned long long) buf.capacity(), left)); - source(buf); - crf(buf); - left -= buf.size(); - } + source.drainInto(crf, size); }); }; diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index f8b545f49b9a..0598104e87e6 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -96,8 +96,20 @@ struct Source return true; } + /** + * Read the rest of this `Source` into `sink`. + */ void drainInto(Sink & sink); + /** + * Read exactly 'len' bytes and write them to 'sink'. + * + * Virtual in anticipation that some `Source` implementations, like + * `FdSource` may eventually be able to provide more performant + * implementations of this function. + */ + virtual void drainInto(Sink & sink, uint64_t len); + std::string drain(); virtual void skip(size_t len); diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8adbe63841e0..11c9d1e3e602 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -104,6 +105,20 @@ void Source::drainInto(Sink & sink) } } +void Source::drainInto(Sink & sink, uint64_t len) +{ + std::array buf; + while (len) { + checkInterrupt(); + // Until std::saturate_cast is available (C++26) + auto lenTrunc = static_cast(std::min(len, std::numeric_limits::max())); + auto n = read(buf.data(), std::min(lenTrunc, buf.size())); + sink({buf.data(), n}); + assert(n <= len); + len -= n; + } +} + std::string Source::drain() { StringSink s; From 17ced6f80c4f1a26a26f5a11715a401d0361ebfa Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 3 Mar 2026 21:46:40 +0300 Subject: [PATCH 038/555] libutil: Add finalSymlink argument to openFileReadonly We have a couple of places where this would end up being useful. --- src/libutil/include/nix/util/file-system.hh | 2 +- src/libutil/unix/file-system.cc | 5 +++-- src/libutil/windows/file-system.cc | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index aae7b7bf37e5..8dcddc38e256 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -220,7 +220,7 @@ AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink final * * @note For directories use @ref openDirectory. */ -AutoCloseFD openFileReadonly(const std::filesystem::path & path); +AutoCloseFD openFileReadonly(const std::filesystem::path & path, FinalSymlink finalSymlink = FinalSymlink::Follow); struct OpenNewFileForWriteParams { diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index e709c5298a0d..cc99c9ba2010 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -29,9 +29,10 @@ AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink final path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC | (finalSymlink == FinalSymlink::Follow ? 0 : O_NOFOLLOW))}; } -AutoCloseFD openFileReadonly(const std::filesystem::path & path) +AutoCloseFD openFileReadonly(const std::filesystem::path & path, FinalSymlink finalSymlink) { - return AutoCloseFD{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; + return AutoCloseFD{ + open(path.c_str(), O_RDONLY | O_CLOEXEC | (finalSymlink == FinalSymlink::Follow ? 0 : O_NOFOLLOW))}; } AutoCloseFD openNewFileForWrite(const std::filesystem::path & path, mode_t mode, OpenNewFileForWriteParams params) diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 3e9680ff2469..9dab6f2b7a04 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -42,7 +42,7 @@ AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink final /*hTemplateFile=*/nullptr)}; } -AutoCloseFD openFileReadonly(const std::filesystem::path & path) +AutoCloseFD openFileReadonly(const std::filesystem::path & path, FinalSymlink finalSymlink) { return AutoCloseFD{CreateFileW( path.c_str(), @@ -50,7 +50,7 @@ AutoCloseFD openFileReadonly(const std::filesystem::path & path) FILE_SHARE_READ | FILE_SHARE_DELETE, /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, + FILE_ATTRIBUTE_NORMAL | (finalSymlink == FinalSymlink::Follow ? 0 : FILE_FLAG_OPEN_REPARSE_POINT), /*hTemplateFile=*/nullptr)}; } From a8d51a0c59346c9326acfcfb0b7c9f9d4e17d9ce Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 3 Mar 2026 21:49:06 +0300 Subject: [PATCH 039/555] libutil: Address TODO O_NOFOLLOW in recursiveSync --- src/libutil/file-system.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index d810e4a06c30..c269e81b1b3b 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -338,7 +338,7 @@ void recursiveSync(const std::filesystem::path & path) /* If it's a file or symlink, just fsync and return. */ auto st = lstat(path); if (S_ISREG(st.st_mode)) { - AutoCloseFD fd = openFileReadonly(path); /* TODO: O_NOFOLLOW? */ + AutoCloseFD fd = openFileReadonly(path, FinalSymlink::DontFollow); if (!fd) throw NativeSysError("opening file %s", PathFmt(path)); fd.fsync(); @@ -359,7 +359,7 @@ void recursiveSync(const std::filesystem::path & path) if (std::filesystem::is_directory(st)) { dirsToEnumerate.emplace_back(entry.path()); } else if (std::filesystem::is_regular_file(st)) { - AutoCloseFD fd = openFileReadonly(entry.path()); /* TODO: O_NOFOLLOW? */ + AutoCloseFD fd = openFileReadonly(entry.path(), FinalSymlink::DontFollow); if (!fd) throw NativeSysError("opening file %1%", PathFmt(entry.path())); fd.fsync(); From 26679828f74ee6e82a4100904e6361f993ff5390 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 3 Mar 2026 22:05:16 +0300 Subject: [PATCH 040/555] libutil,libstore: Expose FinalSymlink in writeFile arguments, use in LocalStore This exposes the knob for controlling final symlink following in writeFile with O_TRUNC (which I've previously marked with FIXMEs, since that was quite suspicious looking). Use DontFollow in restorePath for `flat` serialisation case. Callers already have the invariant that a regular file will be written to the destination path (which has been deleted recursively via deletePath for good measure). This just adds for guardrails against any possible SNAFUs. --- src/libutil/file-content-address.cc | 2 +- src/libutil/file-system.cc | 9 +++++---- src/libutil/include/nix/util/file-system.hh | 14 ++++++++++++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index caf12f81e6c0..e30560349087 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -79,7 +79,7 @@ void restorePath(const std::filesystem::path & path, Source & source, FileSerial { switch (method) { case FileSerialisationMethod::Flat: - writeFile(path, source, 0666, startFsync ? FsSync::Yes : FsSync::No); + writeFile(path, source, 0666, startFsync ? FsSync::Yes : FsSync::No, FinalSymlink::DontFollow); break; case FileSerialisationMethod::NixArchive: restorePath(path, source, startFsync); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index c269e81b1b3b..f56000ebee08 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -254,14 +254,15 @@ void readFile(const std::filesystem::path & path, Sink & sink, bool memory_map) drainFD(fd.get(), sink); } -void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode, FsSync sync) +void writeFile( + const std::filesystem::path & path, std::string_view s, mode_t mode, FsSync sync, FinalSymlink finalSymlink) { AutoCloseFD fd = openNewFileForWrite( path, mode, { .truncateExisting = true, - .followSymlinksOnTruncate = true, /* FIXME: Do we want this? */ + .followSymlinksOnTruncate = (finalSymlink == FinalSymlink::Follow), }); if (!fd) throw NativeSysError("opening file %s", PathFmt(path)); @@ -287,14 +288,14 @@ void writeFile(Descriptor fd, std::string_view s, FsSync sync, const std::filesy } } -void writeFile(const std::filesystem::path & path, Source & source, mode_t mode, FsSync sync) +void writeFile(const std::filesystem::path & path, Source & source, mode_t mode, FsSync sync, FinalSymlink finalSymlink) { AutoCloseFD fd = openNewFileForWrite( path, mode, { .truncateExisting = true, - .followSymlinksOnTruncate = true, /* FIXME: Do we want this? */ + .followSymlinksOnTruncate = (finalSymlink == FinalSymlink::Follow), }); if (!fd) throw NativeSysError("opening file %s", PathFmt(path)); diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 8dcddc38e256..5dce92907749 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -256,9 +256,19 @@ enum struct FsSync { Yes, No }; /** * Write a string to a file. */ -void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No); +void writeFile( + const std::filesystem::path & path, + std::string_view s, + mode_t mode = 0666, + FsSync sync = FsSync::No, + FinalSymlink finalSymlink = FinalSymlink::Follow); -void writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, FsSync sync = FsSync::No); +void writeFile( + const std::filesystem::path & path, + Source & source, + mode_t mode = 0666, + FsSync sync = FsSync::No, + FinalSymlink finalSymlink = FinalSymlink::Follow); void writeFile( Descriptor fd, std::string_view s, FsSync sync = FsSync::No, const std::filesystem::path * origPath = nullptr); From ae955594d695fe6641063a1ab3b3f3133c4ec250 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 3 Mar 2026 22:16:58 +0300 Subject: [PATCH 041/555] libutil: Use Source::drainInto with explicit size parameter in PosixSourceAccessor::readFile Slightly cleaner. We dont need any of the NONBLOCK logic from drainFD here since the source is a regular file + this would allow to stuff copy_file_range/sendfile into the FdSource for more efficient copying to a file descriptor. --- src/libutil/posix-source-accessor.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index e3731c9c5745..7d0b1df75a9a 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -58,8 +58,11 @@ void PosixSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun Date: Tue, 3 Mar 2026 22:15:07 +0100 Subject: [PATCH 042/555] fix: typographical error Modify `derivatons` to `derivations`. --- doc/manual/source/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index af5b45314d1d..871b9416cf91 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -39,7 +39,7 @@ This sandbox by default only allows reading from store objects specified as inputs, and only allows writing to designated [outputs][output] to be [captured as store objects](@docroot@/store/building.md#processing-outputs). A derivation is typically specified as a [derivation expression] in the [Nix language], and [instantiated][instantiate] to a [store derivation]. - There are multiple ways of obtaining store objects from store derivatons, collectively called [realisation][realise]. + There are multiple ways of obtaining store objects from store derivations, collectively called [realisation][realise]. [derivation]: #gloss-derivation From 93246e6d99649f562cad4103e4f24f5fcc845e0e Mon Sep 17 00:00:00 2001 From: Sander Date: Wed, 4 Mar 2026 15:42:02 +0100 Subject: [PATCH 043/555] libstore: forward trusted client settings to fileTransferSettings in daemon Client setting overrides from trusted users were not passed to fileTransferSettings, so settings like `netrc-file` had no effect when set by the client. --- src/libstore/daemon.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 043402c95040..0e5eb6aa5d91 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -6,6 +6,7 @@ #include "nix/store/build-result.hh" #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" +#include "nix/store/filetransfer.hh" #include "nix/store/gc-store.hh" #include "nix/store/log-store.hh" #include "nix/store/indirect-root-store.hh" @@ -296,9 +297,10 @@ struct ClientSettings trusted || name == settings.getWorkerSettings().buildTimeout.name || name == settings.getWorkerSettings().maxSilentTime.name || name == settings.getWorkerSettings().pollInterval.name || name == "connect-timeout" - || (name == "builders" && value == "")) + || (name == "builders" && value == "")) { settings.set(name, value); - else if (setSubstituters(settings.getWorkerSettings().substituters)) + fileTransferSettings.set(name, value); + } else if (setSubstituters(settings.getWorkerSettings().substituters)) ; else warn( From 96c76a6e4dcb5c849a5042835b8c50b9533abd28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20=C5=BDlender?= Date: Wed, 4 Mar 2026 15:11:24 +0100 Subject: [PATCH 044/555] fix: refresh tarball cache on nix profile upgrade Setting tarball-ttl to 0 prevent the use of stale flake inputs when upgrading a profile. Closes: #15396 --- src/nix/profile.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 4f563199c02c..39d51421a50a 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -12,6 +12,7 @@ #include "nix/store/names.hh" #include "nix/util/url.hh" #include "nix/flake/url-name.hh" +#include "nix/fetchers/fetch-settings.hh" #include #include @@ -688,6 +689,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf void run(ref store) override { + fetchSettings.tarballTtl = 0; ProfileManifest manifest(*getEvalState(), *profile); Installables installables; From c28203f0971295eacaebde0eb503d9df9824849e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 5 Mar 2026 00:29:47 +0300 Subject: [PATCH 045/555] libcmd,daemon: Reap children via a self-pipe trick Somewhat better solution for https://github.com/NixOS/nix/issues/15394. Instead of reaping immediately in the signal handler we instead punt that at the poll loop. This way Pid destructor has a chance to properly run instead of being rudely interrupted by the signal handler. See: https://man7.org/tlpi/code/online/dist/altio/self_pipe.c.html --- .../include/nix/cmd/unix-socket-server.hh | 13 ++++++ src/libcmd/unix/unix-socket-server.cc | 11 ++++- src/nix/unix/daemon.cc | 42 +++++++++++++++++-- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh index 3aeee5b55e1c..7a0d9fa79317 100644 --- a/src/libcmd/include/nix/cmd/unix-socket-server.hh +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -53,6 +53,19 @@ struct ServeUnixSocketOptions * Mode for the created socket file. */ mode_t socketMode = 0666; + +#ifndef _WIN32 + /** + * Additional file descriptor to poll. Useful for doing a self-pipe trick + * https://cr.yp.to/docs/selfpipe.html. + */ + Descriptor auxiliaryFd = INVALID_DESCRIPTOR; + + /** + * Optional callback invoked on POLLIN event for auxiliaryFd. + */ + std::function onAuxiliaryFdPollin = nullptr; +#endif }; /** diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index 8cc2bd713d07..e33f4317dad9 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -9,6 +9,8 @@ #include "nix/util/unix-domain-socket.hh" #include "nix/util/util.hh" +#include + #include #include #include @@ -83,6 +85,9 @@ PeerInfo getPeerInfo(Descriptor remote) for (auto & i : listeningSockets) fds.push_back({.fd = i.get(), .events = POLLIN}); + if (options.auxiliaryFd != INVALID_DESCRIPTOR) + fds.push_back({.fd = options.auxiliaryFd, .events = POLLIN}); + // Loop accepting connections. while (1) { try { @@ -95,7 +100,11 @@ PeerInfo getPeerInfo(Descriptor remote) throw SysError("polling for incoming connections"); } - for (auto & fd : fds) { + if (options.auxiliaryFd != INVALID_DESCRIPTOR && options.onAuxiliaryFdPollin && fds.back().revents & POLLIN) + /* Useful for reaping children. */ + options.onAuxiliaryFdPollin(); + + for (auto & fd : std::views::take(fds, listeningSockets.size())) { if (!fd.revents) continue; diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 7fce2e8f6752..3faaba6429d7 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -120,13 +120,20 @@ static ssize_t splice(int fd_in, void * off_in, int fd_out, void * off_out, size } #endif +static Pipe sigChldPipe; + static void sigChldHandler(int sigNo) { // Ensure we don't modify errno of whatever we've interrupted auto saved_errno = errno; - // Reap all dead children. - while (waitpid(-1, 0, WNOHANG) > 0) - ; + /* Write to the self-pipe that gets polled in the accept loop. Pipe + is non-blocking. https://man7.org/tlpi/code/online/dist/altio/self_pipe.c.html */ + auto res = ::write(sigChldPipe.writeSide.get(), "x", 1); + if (res == -1 && errno != EAGAIN) { + /* Something is deeply wrong. Can't call std::terminate here because our terminate + handler is not safe for that. */ + abort(); + } errno = saved_errno; } @@ -243,7 +250,13 @@ static void daemonLoop(ref storeConfig, std::optional storeConfig, std::optional buf; + + /* Drain the self-pipe. */ + while (true) { + if (::read(sigChldPipe.readSide.get(), buf.data(), buf.size()) == -1) { + if (errno == EAGAIN) + break; + else + throw SysError("reading from self-pipe for SIGCHLD"); + } + } + + /* Reap all dead children. */ + pid_t pid = -1; + int status; + while (pid = ::waitpid(/*pid (any child process)=*/-1, &status, WNOHANG), pid > 0) + printInfo("reaped child process %1%, status = %2%", pid, statusToString(status)); + }, }, [&](AutoCloseFD remote, std::function closeListeners) { unix::closeOnExec(remote.get()); From c38eacae2673c60071eec1a07e2181288a13eae7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 12 Sep 2025 21:14:18 +0300 Subject: [PATCH 046/555] libexpr: Use 16-byte atomic loads/stores MOVAPS/MOVDQA for ValueStorage<8> This would have to pair with CMPXCHG16B (or other double-width CAS approaches) to be able to implement concurrent thunk evaluation. See the extensive comment in the code for the reasons why we have to do it this way. The motivation for the change is to simplify the eventual implementation of parallel evaluation. Being able to atomically update the whole Value is much easier to reason about. --- src/libexpr/include/nix/expr/value.hh | 121 +++++++++++++++++++++++++- src/libexpr/meson.build | 4 + src/libexpr/package.nix | 4 +- src/libexpr/value.cc | 21 +++++ 4 files changed, 145 insertions(+), 5 deletions(-) diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index e1b2bc4e25a5..28ff71db8845 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -22,6 +22,10 @@ #include #include +#if defined(__x86_64__) && defined(__SSE2__) +# include +#endif + namespace nix { struct Value; @@ -555,6 +559,11 @@ protected: { return internalType; } + + static bool isAtomic() + { + return false; + } }; namespace detail { @@ -588,7 +597,11 @@ class alignas(16) using PackedPointer = typename PackedPointerTypeStruct::type; using Payload = std::array; - Payload payload = {}; +#if defined(__x86_64__) && defined(__SSE2__) + __m128i payloadWords; +#else + Payload payloadWords = {}; +#endif static constexpr int discriminatorBits = 3; static constexpr PackedPointer discriminatorMask = (PackedPointer(1) << discriminatorBits) - 1; @@ -632,6 +645,54 @@ class alignas(16) pdPairOfPointers, //< layout: Pair of pointers payload }; +#if defined(__x86_64__) && defined(__SSE2__) + /* Why do we even bother with hand-rolling these arch-specific intrinsics + * and don't use libatomic directly for 16 byte atomics? Here's why: + * - https://gcc.gnu.org/legacy-ml/gcc/2018-02/msg00224.html + * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84563 + * + * Basically, we don't really ever want to go through libatomic. As is + * so happens on x86_64 with AVX, MOVDQA/MOVAPS instructions (16-byte aligned + * 128-bit loads and stores) are atomic [^]. Note that + * these instructions are not part of AVX but rather SSE2, which is x86_64-v1. + * They are just not guaranteed to be atomic without AVX. + * + * For more details see: + * - [^] Intel® 64 and IA-32 Architectures Software Developer’s Manual (10.1.1 Guaranteed Atomic Operations). + * - https://patchwork.sourceware.org/project/gcc/patch/YhxkfzGEEQ9KHbBC@tucnak/ + * - https://ibraheem.ca/posts/128-bit-atomics/ + * - https://rigtorp.se/isatomic/ + */ + [[gnu::always_inline]] + void updatePayload(Payload payload) noexcept + { + /* This intrinsic corresponds to MOVAPS. Note that Value (and thus the first member + payloadWords) is 16 bytes aligned. */ + _mm_store_si128(&payloadWords, std::bit_cast<__m128i>(payload)); + } + + [[gnu::always_inline]] + Payload loadPayload() const noexcept + { + /* This intrinsic corresponds to MOVDQA. Note that Value (and thus the first member + payloadWords) is 16 bytes aligned. */ + __m128i res = _mm_load_si128(&payloadWords); + return std::bit_cast(res); + } +#else + [[gnu::always_inline]] + void updatePayload(Payload payload) noexcept + { + payloadWords = payload; + } + + [[gnu::always_inline]] + Payload loadPayload() const noexcept + { + return payloadWords; + } +#endif + template requires std::is_pointer_v static T untagPointer(PackedPointer val) noexcept @@ -639,9 +700,9 @@ class alignas(16) return std::bit_cast(val & ~discriminatorMask); } - PrimaryDiscriminator getPrimaryDiscriminator() const noexcept + PrimaryDiscriminator getPrimaryDiscriminator(PackedPointer firstDWord) const noexcept { - return static_cast(payload[0] & discriminatorMask); + return static_cast(firstDWord & discriminatorMask); } static void assertAligned(PackedPointer val) noexcept @@ -652,25 +713,30 @@ class alignas(16) template void setSingleDWordPayload(PackedPointer untaggedVal) noexcept { + Payload payload; /* There's plenty of free upper bits in the first dword, which is used only for the discriminator. */ payload[0] = static_cast(pdSingleDWord) | (static_cast(type) << discriminatorBits); payload[1] = untaggedVal; + updatePayload(payload); } template void setUntaggablePayload(T * firstPtrField, U untaggableField) noexcept { + Payload payload; static_assert(discriminator >= pdListN && discriminator <= pdPath); auto firstFieldPayload = std::bit_cast(firstPtrField); assertAligned(firstFieldPayload); payload[0] = static_cast(discriminator) | firstFieldPayload; payload[1] = std::bit_cast(untaggableField); + updatePayload(payload); } template void setPairOfPointersPayload(T * firstPtrField, U * secondPtrField) noexcept { + Payload payload; static_assert(type >= tFirstPairOfPointers && type <= tLastPairOfPointers); { auto firstFieldPayload = std::bit_cast(firstPtrField); @@ -682,21 +748,58 @@ class alignas(16) assertAligned(secondFieldPayload); payload[1] = (type - tFirstPairOfPointers) | secondFieldPayload; } + updatePayload(payload); } template requires std::is_pointer_v && std::is_pointer_v void getPairOfPointersPayload(T & firstPtrField, U & secondPtrField) const noexcept { + Payload payload = loadPayload(); firstPtrField = untagPointer(payload[0]); secondPtrField = untagPointer(payload[1]); } +public: + ValueStorage() + { + updatePayload({}); + } + + ValueStorage(const ValueStorage & other) + { + updatePayload(other.loadPayload()); + } + + ValueStorage & operator=(const ValueStorage & other) + { + updatePayload(other.loadPayload()); + return *this; + } + + ValueStorage(ValueStorage && other) noexcept + { + updatePayload(other.loadPayload()); + other.updatePayload({}); // Zero out rhs + } + + ValueStorage & operator=(ValueStorage && other) noexcept + { + updatePayload(other.loadPayload()); + other.updatePayload({}); // Zero out rhs + return *this; + } + + ~ValueStorage() noexcept {} + protected: + static bool isAtomic(); + /** Get internal type currently occupying the storage. */ InternalType getInternalType() const noexcept { - switch (auto pd = getPrimaryDiscriminator()) { + Payload payload = loadPayload(); + switch (auto pd = getPrimaryDiscriminator(payload[0])) { case pdUninitialized: /* Discriminator value of zero is used to distinguish uninitialized values. */ return tUninitialized; @@ -738,6 +841,7 @@ protected: void getStorage(NixInt & integer) const noexcept { + Payload payload = loadPayload(); /* PackedPointerType -> int64_t here is well-formed, since the standard requires this conversion to follow 2's complement rules. This is just a no-op. */ integer = NixInt(payload[1]); @@ -745,6 +849,7 @@ protected: void getStorage(bool & boolean) const noexcept { + Payload payload = loadPayload(); boolean = payload[1]; } @@ -752,44 +857,52 @@ protected: void getStorage(NixFloat & fpoint) const noexcept { + Payload payload = loadPayload(); fpoint = std::bit_cast(payload[1]); } void getStorage(ExternalValueBase *& external) const noexcept { + Payload payload = loadPayload(); external = std::bit_cast(payload[1]); } void getStorage(PrimOp *& primOp) const noexcept { + Payload payload = loadPayload(); primOp = std::bit_cast(payload[1]); } void getStorage(Bindings *& attrs) const noexcept { + Payload payload = loadPayload(); attrs = std::bit_cast(payload[1]); } void getStorage(List & list) const noexcept { + Payload payload = loadPayload(); list.elems = untagPointer(payload[0]); list.size = payload[1]; } void getStorage(StringWithContext & string) const noexcept { + Payload payload = loadPayload(); string.context = untagPointer(payload[0]); string.str = std::bit_cast(payload[1]); } void getStorage(Path & path) const noexcept { + Payload payload = loadPayload(); path.accessor = untagPointer(payload[0]); path.path = std::bit_cast(payload[1]); } void getStorage(Failed *& failed) const noexcept { + Payload payload = loadPayload(); failed = std::bit_cast(payload[1]); } diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 0b594dc0cf1f..893718cb82b7 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -91,6 +91,10 @@ configdata_priv.set( deps_other += toml11 +cpuid = dependency('libcpuid', 'cpuid', required : false) +configdata_priv.set('HAVE_LIBCPUID', cpuid.found().to_int()) +deps_private += cpuid + config_priv_h = configure_file( configuration : configdata_priv, output : 'expr-config-private.hh', diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index d0aef34e95de..e9d12035cde1 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -14,6 +14,7 @@ boehmgc, nlohmann_json, toml11, + libcpuid, # Configuration Options @@ -64,7 +65,8 @@ mkMesonLibrary (finalAttrs: { buildInputs = [ toml11 - ]; + ] + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; propagatedBuildInputs = [ nix-util diff --git a/src/libexpr/value.cc b/src/libexpr/value.cc index 07d036b0dd4c..8dbb277750ba 100644 --- a/src/libexpr/value.cc +++ b/src/libexpr/value.cc @@ -1,5 +1,11 @@ #include "nix/expr/value.hh" +#include "expr-config-private.hh" + +#if HAVE_LIBCPUID +# include +#endif + namespace nix { Value Value::vEmptyList = []() { @@ -26,4 +32,19 @@ Value Value::vFalse = []() { return res; }(); +template<> +bool ValueStorage<8>::isAtomic() +{ +#if HAVE_LIBCPUID + struct cpu_id_t data; + + if (cpu_identify(NULL, &data) < 0) + return false; + + return data.flags[CPU_FEATURE_AVX]; +#else + return false; // Can't tell +#endif +} + } // namespace nix From 877d1ef25904bf5077d16e4b8433f20db7ef906c Mon Sep 17 00:00:00 2001 From: Audrey Dutcher Date: Fri, 6 Mar 2026 16:42:27 +0000 Subject: [PATCH 047/555] Use per-platform signal for internal interrupts instead of SIGUSR1 FreeBSD bdwgc uses SIGUSR1 internally for its runtime. Use a different signal. See [1] for documentation and the suggestion to use SIGTSTP on FreeBSD. [1] https://github.com/bdwgc/bdwgc/blob/af374f921837af641f2f61698ab90767befd8996/include/private/gc_priv.h#L4557 --- src/libmain/shared.cc | 6 +++--- src/libutil/include/nix/util/signals.hh | 9 ++++++++- src/libutil/unix/include/nix/util/signals-impl.hh | 4 ++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index ae5281e42821..a65fc773aa44 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -164,10 +164,10 @@ void initNix(bool loadConfig) if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD"); - /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ + /* Install a dummy NIX_SIG_MULTI_INT handler for use with pthread_kill(). */ act.sa_handler = sigHandler; - if (sigaction(SIGUSR1, &act, 0)) - throw SysError("handling SIGUSR1"); + if (sigaction(NIX_SIG_MULTI_INT, &act, 0)) + throw SysError("handling multiplexed interrupt"); #endif #ifdef __APPLE__ diff --git a/src/libutil/include/nix/util/signals.hh b/src/libutil/include/nix/util/signals.hh index 0062e8dcd70f..c7b9d4e1324a 100644 --- a/src/libutil/include/nix/util/signals.hh +++ b/src/libutil/include/nix/util/signals.hh @@ -8,6 +8,13 @@ #include +#if defined(__FreeBSD__) +// SIGUSR1 is used by bdwgc +# define NIX_SIG_MULTI_INT SIGTSTP +#elif !defined(_WIN32) +# define NIX_SIG_MULTI_INT SIGUSR1 +#endif + namespace nix { /* User interruption. */ @@ -51,7 +58,7 @@ struct InterruptCallback std::unique_ptr createInterruptCallback(fun callback); /** - * A RAII class that causes the current thread to receive SIGUSR1 when + * A RAII class that causes the current thread to receive NIX_SIG_MULTI_INT when * the signal handler thread receives SIGINT. That is, this allows * SIGINT to be multiplexed to multiple threads. * diff --git a/src/libutil/unix/include/nix/util/signals-impl.hh b/src/libutil/unix/include/nix/util/signals-impl.hh index e51ce082a780..dab0b09ed2dc 100644 --- a/src/libutil/unix/include/nix/util/signals-impl.hh +++ b/src/libutil/unix/include/nix/util/signals-impl.hh @@ -94,7 +94,7 @@ inline void checkInterrupt() } /** - * A RAII class that causes the current thread to receive SIGUSR1 when + * A RAII class that causes the current thread to receive NIX_SIG_MULTI_INT when * the signal handler thread receives SIGINT. That is, this allows * SIGINT to be multiplexed to multiple threads. */ @@ -105,7 +105,7 @@ struct ReceiveInterrupts ReceiveInterrupts() : target(pthread_self()) - , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + , callback(createInterruptCallback([&]() { pthread_kill(target, NIX_SIG_MULTI_INT); })) { } }; From 82704a5bfeb5e4b65f8c0c38364045dbe24c5124 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 7 Mar 2026 00:42:08 +0300 Subject: [PATCH 048/555] Disable flake-regressions with failing dependency --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02a9fe146fc1..fc153fa53749 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,7 +245,9 @@ jobs: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} - name: Run flake regressions tests - run: MAX_FLAKES=25 flake-regressions/eval-all.sh + run: | + touch flake-regressions/tests/{0utkarsh/nixconfig,HariAmoor-professional/nixos-config}/disabled + MAX_FLAKES=25 flake-regressions/eval-all.sh profile_build: needs: tests From 77d27d8c2030c8f69fb328c67e0cbc6f523d2b8d Mon Sep 17 00:00:00 2001 From: Audrey Dutcher Date: Sun, 18 Aug 2024 16:22:47 -0700 Subject: [PATCH 049/555] tests/functional/restricted: Don't use a process substitution The <() process substitution syntax doesn't work for this one testcase in bash for FreeBSD. The exact reason for this is unknown, possibly to do with pipe vs file vs fifo EOF behavior. The prior behavior was this test hanging forever, with no children of the bash process. Change-Id: I71822a4b9dea6059b34300568256c5b7848109ac (cherry picked from commit ae628d4af29a3b89340a91187a6e7d4d242e759d) --- tests/functional/restricted.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index c86abebee3d0..9c2b860a9e2d 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -6,7 +6,7 @@ clearStoreIfPossible nix-instantiate --restrict-eval --eval -E '1 + 2' (! nix-instantiate --eval --restrict-eval ./restricted.nix) -(! nix-instantiate --eval --restrict-eval <(echo '1 + 2')) +TMPFILE=$(mktemp); echo '1 + 2' >"$TMPFILE"; (! nix-instantiate --eval --restrict-eval "$TMPFILE"); rm "$TMPFILE" mkdir -p "$TEST_ROOT/nix" cp ./simple.nix "$TEST_ROOT/nix" From 14359a1f5be1fe80e59a9ef97f16a08faa4e48c5 Mon Sep 17 00:00:00 2001 From: Audrey Dutcher Date: Wed, 31 Jan 2024 20:12:40 -0700 Subject: [PATCH 050/555] Add support for sandboxing on FreeBSD New FreeBSD sandboxes are based on jails and chroots. They provide fairly similar capabilities to sandboxes on Linux and allow for pure builds of FreeBSD nixpkgs. Although it would also be possble to use jails for Linux emulation, that is not supported with this commit. Change-Id: I619e1e34c56de7aaa64a38408210a410bb13adba Co-Authored-By: Artemis Tosini Co-Authored-By: John Ericson --- src/libstore/meson.build | 6 + src/libstore/package.nix | 16 +- .../unix/build/chroot-derivation-builder.cc | 2 +- src/libstore/unix/build/derivation-builder.cc | 12 +- .../unix/build/freebsd-derivation-builder.cc | 483 ++++++++++++++++++ .../unix/build/linux-derivation-builder.cc | 3 + src/libutil/file-system.cc | 53 +- src/libutil/freebsd/freebsd-jail.cc | 19 + .../freebsd/include/nix/util/freebsd-jail.hh | 3 +- src/libutil/include/nix/util/file-system.hh | 52 +- 10 files changed, 550 insertions(+), 99 deletions(-) create mode 100644 src/libstore/unix/build/freebsd-derivation-builder.cc diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 9e52517d4865..63eba3125531 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -142,6 +142,12 @@ endif configdata_priv.set('HAVE_SECCOMP', seccomp.found().to_int()) deps_private += seccomp +if host_machine.system() == 'freebsd' + # libjail is not in freebsd.libc, but there's no discovery mechanism + libjail = declare_dependency(link_args : [ '-ljail' ]) + deps_other += libjail +endif + nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 1f7b56f2238e..4fa2abaa722b 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -5,6 +5,7 @@ unixtools, darwin, + freebsd, nix-util, boost, @@ -17,6 +18,7 @@ cmake, # for resolving aws-crt-cpp dep busybox-sandbox-shell ? null, + pkgsStatic, # Configuration Options @@ -24,6 +26,15 @@ embeddedSandboxShell ? stdenv.hostPlatform.isStatic, + withSandboxShell ? stdenv.hostPlatform.isLinux || stdenv.hostPlatform.isFreeBSD, + sandboxShell ? + if stdenv.hostPlatform.isLinux then + "${busybox-sandbox-shell}/bin/busybox" + else if stdenv.hostPlatform.isFreeBSD then + "${pkgsStatic.bash}/bin/bash" + else + null, + withAWS ? # Default is this way because there have been issues building this dependency (lib.meta.availableOn stdenv.hostPlatform aws-c-common), @@ -67,6 +78,7 @@ mkMesonLibrary (finalAttrs: { sqlite ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp + ++ lib.optional stdenv.hostPlatform.isFreeBSD freebsd.libjail ++ lib.optional withAWS aws-crt-cpp; propagatedBuildInputs = [ @@ -79,8 +91,8 @@ mkMesonLibrary (finalAttrs: { (lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell) (lib.mesonEnable "s3-aws-auth" withAWS) ] - ++ lib.optionals stdenv.hostPlatform.isLinux [ - (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox") + ++ lib.optionals withSandboxShell [ + (lib.mesonOption "sandbox-shell" sandboxShell) ]; meta = { diff --git a/src/libstore/unix/build/chroot-derivation-builder.cc b/src/libstore/unix/build/chroot-derivation-builder.cc index d54aa4bbe25f..4c3ed21d8635 100644 --- a/src/libstore/unix/build/chroot-derivation-builder.cc +++ b/src/libstore/unix/build/chroot-derivation-builder.cc @@ -1,4 +1,4 @@ -#ifdef __linux__ +#if defined(__linux__) || defined(__FreeBSD__) namespace nix { diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index cd9237051350..5fc4d05b4895 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1324,9 +1324,6 @@ void DerivationBuilderImpl::runChild(RunChildArgs args) if (chdir(tmpDirInSandbox().c_str()) == -1) throw SysError("changing into %1%", PathFmt(tmpDir)); - /* Close all other file descriptors. */ - unix::closeExtraFDs(); - /* Disable core dumps by default. */ struct rlimit limit = {0, RLIM_INFINITY}; setrlimit(RLIMIT_CORE, &limit); @@ -2045,6 +2042,7 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path) // FIXME: do this properly #include "chroot-derivation-builder.cc" #include "linux-derivation-builder.cc" +#include "freebsd-derivation-builder.cc" #include "darwin-derivation-builder.cc" #include "external-derivation-builder.cc" @@ -2093,7 +2091,7 @@ std::unique_ptr makeDerivationBuild } if (store.storeDir != store.config->realStoreDir.get()) { -#ifdef __linux__ +#if defined(__linux__) || defined(__FreeBSD__) useSandbox = true; #else throw Error("building using a diverted store is not supported on this platform"); @@ -2123,6 +2121,12 @@ std::unique_ptr makeDerivationBuild new ChrootLinuxDerivationBuilder(store, std::move(miscMethods), std::move(params))); return DerivationBuilderUnique(new LinuxDerivationBuilder(store, std::move(miscMethods), std::move(params))); +#elif defined(__FreeBSD__) + if (useSandbox) + return DerivationBuilderUnique( + new ChrootFreeBSDDerivationBuilder(store, std::move(miscMethods), std::move(params))); + + return DerivationBuilderUnique(new FreeBSDDerivationBuilder(store, std::move(miscMethods), std::move(params))); #else if (useSandbox) throw Error("sandboxing builds is not supported on this platform"); diff --git a/src/libstore/unix/build/freebsd-derivation-builder.cc b/src/libstore/unix/build/freebsd-derivation-builder.cc new file mode 100644 index 000000000000..3a42f230727f --- /dev/null +++ b/src/libstore/unix/build/freebsd-derivation-builder.cc @@ -0,0 +1,483 @@ +#ifdef __FreeBSD__ + +# include +# include + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include "nix/util/freebsd-jail.hh" +# include "nix/util/util.hh" + +namespace nix { + +namespace { + +struct PasswordEntry +{ + std::string name; + uid_t uid; + gid_t gid; + std::string description; + std::filesystem::path home; + std::filesystem::path shell; +}; + +static void free_db(DB * db) +{ + if (db != nullptr) { + (db->close)(db); + } +} + +// Database open flags from FreeBSD, in case they're necessary for compatibility +static const HASHINFO db_flags = { + .bsize = 4096, + .ffactor = 32, + .nelem = 256, + .cachesize = 2 * 1024 * 1024, + .hash = nullptr, + .lorder = BIG_ENDIAN, +}; + +// Password database version +// Version 4 has been current since 2003 +static const uint8_t dbVersion = 4; + +static void serializeString(std::vector & buf, std::string const & str) +{ + buf.reserve(buf.size() + str.size() + 1); + buf.insert(buf.end(), str.begin(), str.end()); + buf.push_back(0); +} + +static void serializeInt(std::vector & buf, uint32_t num) +{ + buf.reserve(buf.size() + sizeof(num)); + // Always big endian + buf.push_back((num >> 24) & 0xff); + buf.push_back((num >> 16) & 0xff); + buf.push_back((num >> 8) & 0xff); + buf.push_back((num >> 0) & 0xff); +} + +static std::vector byNameKey(std::string const & name) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYNAME, dbVersion)}; + buf.reserve(1 + name.size()); + // We can't use serializeString since that's null terimated + buf.insert(buf.end(), name.begin(), name.end()); + + return buf; +} + +static std::vector byNumKey(uint32_t num) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYNUM, dbVersion)}; + serializeInt(buf, num); + + return buf; +} + +static std::vector byUidKey(uid_t uid) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYUID, dbVersion)}; + serializeInt(buf, uid); + + return buf; +} + +static void createPasswordFiles(std::filesystem::path & chrootRootDir, std::vector & users) +{ + std::unique_ptr db( + dbopen((chrootRootDir + "/etc/pwd.db").c_str(), O_CREAT | O_RDWR | O_EXCL, 0644, DB_HASH, &db_flags), &free_db); + + if (db == nullptr) { + throw SysError("Could not create password database"); + } + + auto dbInsert = [&db](std::vector key_buf, std::vector & value_buf) { + DBT key = {key_buf.data(), key_buf.size()}; + DBT value = {value_buf.data(), value_buf.size()}; + + if ((db->put)(db.get(), &key, &value, R_NOOVERWRITE) == -1) { + throw SysError("Could not write to password database"); + } + }; + + // Annoyingly DBT doesn't have const pointers so we need this whole shuffle + std::string versionKeyStr(_PWD_VERSION_KEY); + std::vector versionKey(versionKeyStr.begin(), versionKeyStr.end()); + std::vector versionValue{dbVersion}; + dbInsert(versionKey, versionValue); + + for (size_t i = 0; i < users.size(); i++) { + auto user = users[i]; + + // flags for non-empty fields + uint32_t fields = _PWF_NAME | _PWF_PASSWD | _PWF_UID | _PWF_GID | _PWF_GECOS | _PWF_DIR | _PWF_SHELL; + + std::vector buf; + serializeString(buf, user.name); + // pw_password is always "*" in the insecure database + serializeString(buf, std::string("*")); + serializeInt(buf, user.uid); + serializeInt(buf, user.gid); + // pw_change = 0 means no requirement to change password + serializeInt(buf, 0); + // pw_class is empty since we don't make a class database + serializeString(buf, std::string("")); + serializeString(buf, user.description); + serializeString(buf, user.home); + serializeString(buf, user.shell); + // pw_expire = 0 means password does not expire + serializeInt(buf, 0); + serializeInt(buf, fields); + + dbInsert(byNameKey(user.name), buf); + // _PW_KEYBYNUM is 1-indexed + dbInsert(byNumKey(i + 1), buf); + dbInsert(byUidKey(user.uid), buf); + } + + // FreeBSD libc doesn't use /etc/passwd, but some software might + std::string passwdContent = ""; + for (auto user : users) { + passwdContent.append( + fmt("%s:*:%d:%d:%s:%s:%s\n", + user.name, + user.uid, + user.gid, + user.description, + PathFmt(user.home), + PathFmt(user.shell))); + } + + writeFile(chrootRootDir + "/etc/passwd", passwdContent); + + // No need to make /etc/master.passwd or /etc/spwd.db, + // our build user wouldn't be able to read them anyway +} + +} // namespace + +struct FreeBSDDerivationBuilder : virtual DerivationBuilderImpl +{ + using DerivationBuilderImpl::DerivationBuilderImpl; +}; + +template +struct iovec iovFromStaticSizedString(const char (&array)[n]) +{ + return { + .iov_base = const_cast(static_cast(&array[0])), + .iov_len = n, + }; +} + +struct iovec iovFromDynamicSizeString(const std::string & s) +{ + return { + .iov_base = const_cast(static_cast(s.c_str())), + .iov_len = s.length() + 1, + }; +} + +struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivationBuilder +{ + std::shared_ptr autoDelJail = std::make_shared(); + + ChrootFreeBSDDerivationBuilder( + LocalStore & store, std::unique_ptr miscMethods, DerivationBuilderParams params) + : DerivationBuilderImpl{store, std::move(miscMethods), std::move(params)} + , ChrootDerivationBuilder{store, std::move(miscMethods), std::move(params)} + , FreeBSDDerivationBuilder{store, std::move(miscMethods), std::move(params)} + { + } + + virtual void cleanupBuild(bool force) override + { + autoDelJail->remove(); + ChrootDerivationBuilder::cleanupBuild(force); + } + + void prepareSandbox() override + { + ChrootDerivationBuilder::prepareSandbox(); + + std::vector users{ + { + .name = "root", + .uid = 0, + .gid = 0, + .description = "Nix build user", + .home = store.config->getLocalSettings().sandboxBuildDir, + .shell = "/noshell", + }, + { + .name = "nixbld", + .uid = buildUser->getUID(), + .gid = sandboxGid(), + .description = "Nix build user", + .home = store.config->getLocalSettings().sandboxBuildDir, + .shell = "/noshell", + }, + { + .name = "nobody", + .uid = 65534, + .gid = 65534, + .description = "Nobody", + .home = "/", + .shell = "/noshell", + }, + }; + + createPasswordFiles(chrootRootDir, users); + + // FreeBSD doesn't have a group database, just write a text file + writeFile( + chrootRootDir + "/etc/group", + fmt("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n", + sandboxGid())); + + // Linux waits until after entering the child to start mounting so it doesn't + // pollute the root mount namespace. + // FreeBSD doesn't have mount namespaces, so there's no reason to wait. + + auto devpath = chrootRootDir + "/dev"; + mkdir(devpath.c_str(), 0555); + mkdir((chrootRootDir + "/bin").c_str(), 0555); + char errmsg[255] = ""; + struct iovec iov[8] = { + iovFromStaticSizedString("fstype"), + iovFromStaticSizedString("devfs"), + iovFromStaticSizedString("fspath"), + iovFromDynamicSizeString(devpath), + iovFromStaticSizedString("ruleset"), + iovFromStaticSizedString("4"), + iovFromStaticSizedString("errmsg"), + iovFromStaticSizedString(errmsg), + }; + if (nmount(iov, 6, 0) < 0) { + throw SysError("Failed to mount jail /dev: %1%", PathFmt(errmsg)); + } + autoDelJail->childrenMounts.emplace_back(devpath); + + for (auto & i : pathsInChroot) { + char errmsg[255]; + errmsg[0] = 0; + + if (i.second.source == "/proc") { + continue; // backwards compatibility + } + std::filesystem::path path = chrootRootDir + i.first.c_str(); + + struct stat stat_buf; + if (stat(i.second.source.c_str(), &stat_buf) < 0) { + throw SysError("stat"); + } + + // mount points must exist and be the right type + if (S_ISDIR(stat_buf.st_mode)) { + createDirs(path); + } else { + createDirs(path.parent_path()); + writeFile(path, ""); + } + + struct iovec iov[8] = { + iovFromStaticSizedString("fstype"), + iovFromStaticSizedString("nullfs"), + iovFromStaticSizedString("fspath"), + iovFromDynamicSizeString(path), + iovFromStaticSizedString("target"), + iovFromDynamicSizeString(i.second.source), + iovFromStaticSizedString("errmsg"), + iovFromStaticSizedString(errmsg), + }; + if (nmount(iov, 8, 0) < 0) { + throw SysError("Failed to mount nullfs for %1% - %2%", PathFmt(path), errmsg); + } + autoDelJail->childrenMounts.emplace_back(path); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (!derivationType.isSandboxed()) { + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed-outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + + /* N.B. it is realistic that these paths might not exist. It + happens when testing Nix building fixed-output derivations + within a pure derivation. */ + for (std::filesystem::path path : {"/etc/resolv.conf", "/etc/services", "/etc/hosts"}) { + if (pathExists(path)) { + // This means if your network config changes during a FOD build, + // the DNS in the sandbox will be wrong. However, this is pretty unlikely + // to actually be a problem, because FODs are generally pretty fast, + // and machines with often-changing network configurations probably + // want to run resolved or some other local resolver anyway. + // + // There's also just no simple way to do this correctly, you have to manually + // inotify watch the files for changes on the outside and update the sandbox + // while the build is running (or at least that's what Flatpak does). + // + // I also just generally feel icky about modifying sandbox state under a build, + // even though it really shouldn't be a big deal. -K900 + copyFile(path, chrootRootDir / path.relative_path(), false, true); + } + } + + if (fileTransferSettings.caFile.get() && pathExists(fileTransferSettings.caFile.get().value())) { + // For the same reasons as above, copy the CA certificates file too. + // It should be even less likely to change during the build than resolv.conf. + createDirs(chrootRootDir + "/etc/ssl/certs"); + copyFile( + fileTransferSettings.caFile.get().value(), + chrootRootDir + "/etc/ssl/certs/ca-certificates.crt", + false, + true); + } + } + } + + void startChild() override + { + int jid; + + RunChildArgs args{ +# if NIX_WITH_AWS_AUTH + .awsCredentials = preResolveAwsCredentials(), +# endif + }; + + if (derivationType.isSandboxed()) { + jid = jail_setv( + JAIL_CREATE, + "persist", + "true", + "path", + chrootRootDir.c_str(), + "host.hostname", + "localhost", + // TODO: Make our own ruleset + "vnet", + "new", + NULL); + if (jid < 0) { + throw SysError("Failed to create jail (isolated network): %1%", jail_errmsg); + } + autoDelJail->jid = jid; + + // Everything from here to the end of the block is setting up the network + // code adapted from freebsd/sbin/ifconfig/af_inet.c, in_exec_nl + Pid helper = startProcess([&]() { + unix::closeExtraFDs(); + enterChroot(); + + struct snl_state ss = {}; + if (!snl_init(&ss, NETLINK_ROUTE)) { + throw SysError("Failed to init netlink connection"); + } + + struct snl_writer nw = {}; + snl_init_writer(&ss, &nw); + struct nlmsghdr * hdr = snl_create_msg_request(&nw, NL_RTM_NEWADDR); + struct ifaddrmsg * ifahdr = snl_reserve_msg_object(&nw, struct ifaddrmsg); + + ifahdr->ifa_family = AF_INET; + ifahdr->ifa_prefixlen = 8; + ifahdr->ifa_index = if_nametoindex("lo0"); + snl_add_msg_attr_ip4(&nw, IFA_LOCAL, (const struct in_addr *) "\x7f\x00\x00\x01"); + + int off = snl_add_msg_attr_nested(&nw, IFA_FREEBSD); + snl_add_msg_attr_u32(&nw, IFAF_FLAGS, IFF_LOOPBACK | IFF_UP); + snl_end_attr_nested(&nw, off); + + if (!(hdr = snl_finalize_msg(&nw)) || !snl_send_message(&ss, hdr)) { + snl_free(&ss); + throw SysError("Failed to sendoff netlink message"); + } + + struct snl_errmsg_data e = {}; + snl_read_reply_code(&ss, hdr->nlmsg_seq, &e); + if (e.error_str != NULL) { + snl_free(&ss); + throw SysError("Failed to configure loopback interface: %1%", e.error_str); + } + snl_free(&ss); + _exit(0); + }); + + if (helper.wait() != 0) { + throw SysError("Failed to configure loopback address"); + } + } else { + jid = jail_setv( + JAIL_CREATE, + "persist", + "true", + // 4 is the most restrictive devfs ruleset that meets our needs + // which is found in the default installation. Trying to add + // another one is a huge pain... + "devfs_ruleset", + "4", + "path", + chrootRootDir.c_str(), + "host.hostname", + "localhost", + "ip4", + "inherit", + "ip6", + "inherit", + "allow.raw_sockets", + "true", + NULL); + if (jid < 0) { + throw SysError("Failed to create jail (networked): %1%", jail_errmsg); + } + autoDelJail->jid = jid; + } + + pid = startProcess([&]() { + openSlave(); + runChild(args); + }); + } + + void enterChroot() override + { + /* Close all other file descriptors. This must happen before + jail_attach for FreeBSD. */ + unix::closeExtraFDs(); + + if (jail_attach(autoDelJail->jid) < 0) { + throw SysError("Failed to attach to jail"); + } + } + + void addDependency(const StorePath & path) + { + auto [source, target] = ChrootDerivationBuilder::addDependencyPrep(path); + throw Error("Unimplemented"); + } +}; + +} // namespace nix + +#endif diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 0ffcf0ccb657..6af4a0028039 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -171,6 +171,9 @@ struct LinuxDerivationBuilder : virtual DerivationBuilderImpl .system = drv.platform, .impersonateLinux26 = localSettings.impersonateLinux26, }); + + /* Close all other file descriptors. */ + unix::closeExtraFDs(); } }; diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index f56000ebee08..d2983ec7d321 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -440,47 +440,6 @@ void AutoDelete::cancel() noexcept del = false; } -////////////////////////////////////////////////////////////////////// - -#ifdef __FreeBSD__ -AutoUnmount::AutoUnmount() - : del{false} -{ -} - -AutoUnmount::AutoUnmount(const std::filesystem::path & p) - : path(p) - , del(true) -{ -} - -AutoUnmount::~AutoUnmount() -{ - try { - unmount(); - } catch (...) { - ignoreExceptionInDestructor(); - } -} - -void AutoUnmount::cancel() noexcept -{ - del = false; -} - -void AutoUnmount::unmount() -{ - if (del) { - if (::unmount(path.c_str(), 0) < 0) { - throw SysError("Failed to unmount path %1%", PathFmt(path)); - } - } - cancel(); -} -#endif - -////////////////////////////////////////////////////////////////////// - std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const std::string & prefix, mode_t mode) { while (1) { @@ -620,7 +579,7 @@ void setWriteTime(const std::filesystem::path & path, const PosixStat & st) setWriteTime(path, st.st_atime, st.st_mtime, S_ISLNK(st.st_mode)); } -void copyFile(const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete) +void copyFile(const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete, bool contents) { auto fromStatus = std::filesystem::symlink_status(from); @@ -633,8 +592,14 @@ void copyFile(const std::filesystem::path & from, const std::filesystem::path & } if (std::filesystem::is_symlink(fromStatus) || std::filesystem::is_regular_file(fromStatus)) { - std::filesystem::copy( - from, to, std::filesystem::copy_options::copy_symlinks | std::filesystem::copy_options::overwrite_existing); + if (contents) { + std::filesystem::copy_file(from, to, std::filesystem::copy_options::overwrite_existing); + } else { + std::filesystem::copy( + from, + to, + std::filesystem::copy_options::copy_symlinks | std::filesystem::copy_options::overwrite_existing); + } } else if (std::filesystem::is_directory(fromStatus)) { std::filesystem::create_directory(to); for (auto & entry : DirectoryIterator(from)) { diff --git a/src/libutil/freebsd/freebsd-jail.cc b/src/libutil/freebsd/freebsd-jail.cc index 1961c9a13f50..9273a50b6e93 100644 --- a/src/libutil/freebsd/freebsd-jail.cc +++ b/src/libutil/freebsd/freebsd-jail.cc @@ -8,6 +8,7 @@ # include "nix/util/error.hh" # include "nix/util/util.hh" +# include "nix/util/logging.hh" namespace nix { @@ -24,6 +25,24 @@ void AutoRemoveJail::remove() } } cancel(); + + bool failed = false; + for (auto & path : childrenMounts) { + int r = unmount(path.c_str(), 0); + if (r < 0 && errno == EBUSY) { + sleep(1); + r = unmount(path.c_str(), 0); + } + if (r < 0) { + warn("Failed to unmount path %1%", PathFmt(path)); + failed = true; + } + } + childrenMounts.clear(); + + if (failed) { + throw SysError("Failed to unmount some jail paths"); + } } AutoRemoveJail::~AutoRemoveJail() diff --git a/src/libutil/freebsd/include/nix/util/freebsd-jail.hh b/src/libutil/freebsd/include/nix/util/freebsd-jail.hh index b4b1cc75a067..aad51f856a84 100644 --- a/src/libutil/freebsd/include/nix/util/freebsd-jail.hh +++ b/src/libutil/freebsd/include/nix/util/freebsd-jail.hh @@ -8,8 +8,9 @@ namespace nix { class AutoRemoveJail { static constexpr int INVALID_JAIL = -1; - int jid = INVALID_JAIL; public: + int jid = INVALID_JAIL; + std::vector childrenMounts; AutoRemoveJail() = default; AutoRemoveJail(int jid); AutoRemoveJail(const AutoRemoveJail &) = delete; diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 5dce92907749..2601b5fdaa0a 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -353,8 +353,12 @@ void moveFile(const std::filesystem::path & src, const std::filesystem::path & d * `true`, then also remove `oldPath` (making this equivalent to `moveFile`, but * with the guaranty that the destination will be “fresh”, with no stale inode * or file descriptor pointing to it). + * + * If contents is set, always create a regular file, even if the source is a + * link. */ -void copyFile(const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete); +void copyFile( + const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete, bool contents = false); /** * Automatic cleanup of resources. @@ -598,50 +602,4 @@ private: std::filesystem::directory_iterator it_; }; -#ifdef __FreeBSD__ -class AutoUnmount -{ - std::filesystem::path path; - bool del; -public: - AutoUnmount(); - AutoUnmount(const std::filesystem::path &); - AutoUnmount(const AutoUnmount &) = delete; - - AutoUnmount(AutoUnmount && other) noexcept - : path(std::move(other.path)) - , del(std::exchange(other.del, false)) - { - } - - AutoUnmount & operator=(AutoUnmount && other) noexcept - { - swap(*this, other); - return *this; - } - - friend void swap(AutoUnmount & lhs, AutoUnmount & rhs) noexcept - { - using std::swap; - swap(lhs.path, rhs.path); - swap(lhs.del, rhs.del); - } - - ~AutoUnmount(); - - /** - * Cancel the unmounting - */ - void cancel() noexcept; - - /** - * Unmount the mountpoint right away (if it exists), resetting the - * `AutoUnmount` - * - * The destructor calls this, but ignoring any exception. - */ - void unmount(); -}; -#endif - } // namespace nix From c608b07746f1210886f747fd56619a6aa29cb957 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 6 Mar 2026 19:40:57 +0300 Subject: [PATCH 051/555] libutil: Make ChunkedVector thread-safe --- src/libexpr/include/nix/expr/symbol-table.hh | 9 +- src/libutil-tests/chunked-vector.cc | 61 ++++++++-- .../include/nix/util/chunked-vector.hh | 112 ++++++++++++++---- 3 files changed, 141 insertions(+), 41 deletions(-) diff --git a/src/libexpr/include/nix/expr/symbol-table.hh b/src/libexpr/include/nix/expr/symbol-table.hh index f0220376c53f..dab095287088 100644 --- a/src/libexpr/include/nix/expr/symbol-table.hh +++ b/src/libexpr/include/nix/expr/symbol-table.hh @@ -84,8 +84,11 @@ class SymbolStr { friend class SymbolTable; - constexpr static size_t chunkSize{8192}; - using SymbolValueStore = ChunkedVector; + constexpr static size_t chunkSize{65536}; + using SymbolValueStore = ChunkedVector< + SymbolValue, + chunkSize, + /*MaxChunks=*/std::numeric_limits::max() / chunkSize>; const SymbolValue * s; @@ -265,7 +268,7 @@ private: * During its lifetime the monotonic buffer holds all strings and nodes, if the symbol set is node based. */ std::pmr::monotonic_buffer_resource buffer; - SymbolStr::SymbolValueStore store{16}; + SymbolStr::SymbolValueStore store; /** * Transparent lookup of string view for a pointer to a ChunkedVector entry -> return offset into the store. diff --git a/src/libutil-tests/chunked-vector.cc b/src/libutil-tests/chunked-vector.cc index 52f87a0d5f4c..da87d7e3571e 100644 --- a/src/libutil-tests/chunked-vector.cc +++ b/src/libutil-tests/chunked-vector.cc @@ -2,16 +2,20 @@ #include +#include +#include +#include + namespace nix { TEST(ChunkedVector, InitEmpty) { - auto v = ChunkedVector(100); + auto v = ChunkedVector(); ASSERT_EQ(v.size(), 0u); } TEST(ChunkedVector, GrowsCorrectly) { - auto v = ChunkedVector(100); + auto v = ChunkedVector(); for (uint32_t i = 1; i < 20; i++) { v.add(i); ASSERT_EQ(v.size(), i); @@ -20,7 +24,7 @@ TEST(ChunkedVector, GrowsCorrectly) TEST(ChunkedVector, AddAndGet) { - auto v = ChunkedVector(100); + auto v = ChunkedVector(); for (auto i = 1; i < 20; i++) { auto [i2, idx] = v.add(i); auto & i3 = v[idx]; @@ -31,7 +35,7 @@ TEST(ChunkedVector, AddAndGet) TEST(ChunkedVector, ForEach) { - auto v = ChunkedVector(100); + auto v = ChunkedVector(); for (auto i = 1; i < 20; i++) { v.add(i); } @@ -40,17 +44,48 @@ TEST(ChunkedVector, ForEach) ASSERT_EQ(count, v.size()); } -TEST(ChunkedVector, OverflowOK) +TEST(ChunkedVector, NonTrivialType) { - // Similar to the AddAndGet, but intentionnally use a small - // initial ChunkedVector to force it to overflow - auto v = ChunkedVector(2); - for (auto i = 1; i < 20; i++) { - auto [i2, idx] = v.add(i); - auto & i3 = v[idx]; - ASSERT_EQ(i, i2); - ASSERT_EQ(&i2, &i3); + auto v = ChunkedVector, 2, 10>(); + v.add(std::vector{1, 2}); + v.add(std::vector{3, 4}); + v.add(std::vector{5, 6}); + ASSERT_EQ(v.size(), 3); +} + +TEST(ChunkedVector, ConcurrentAdd) +{ + ChunkedVector v; + constexpr int nThreads = 8; + constexpr int perThread = 8192; + + std::vector threads; + std::vector> indices(nThreads); + + for (int t = 0; t < nThreads; t++) { + threads.emplace_back([&, t]() { + for (int i = 0; i < perThread; ++i) { + auto [val, idx] = v.add(static_cast(t * perThread + i)); + indices[t].push_back(idx); + } + }); } + + for (auto & t : threads) + t.join(); + + ASSERT_EQ(v.size(), nThreads * perThread); + + // All indices unique + std::set all; + for (auto & vec : indices) + for (auto idx : vec) + ASSERT_TRUE(all.insert(idx).second); + + // All values correct + for (int t = 0; t < nThreads; t++) + for (int i = 0; i < perThread; i++) + ASSERT_EQ(v[indices[t][i]], static_cast(t * perThread + i)); } } // namespace nix diff --git a/src/libutil/include/nix/util/chunked-vector.hh b/src/libutil/include/nix/util/chunked-vector.hh index 38e53c7f54c4..f3452139f4c9 100644 --- a/src/libutil/include/nix/util/chunked-vector.hh +++ b/src/libutil/include/nix/util/chunked-vector.hh @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include "nix/util/error.hh" @@ -18,50 +20,101 @@ namespace nix { * on large data sets by on average (growth factor)/2, mostly * eliminates copies within the vector during resizing, and provides stable * references to its elements. + * + * Thread-safe to append and read concurrently. Caps the maximum number of chunks + * to MaxChunks for ease of implementation. */ -template +template class ChunkedVector { + static_assert(alignof(T) <= __STDCPP_DEFAULT_NEW_ALIGNMENT__, "ChunkedVector does not support over-aligned types"); + private: - uint32_t size_ = 0; - std::vector> chunks; + /* Put on a separate cache line to avoid false sharing. */ + alignas(std::hardware_destructive_interference_size) std::atomic size_ = 0; + + using ChunksArray = std::array, MaxChunks>; + + std::unique_ptr chunks = std::make_unique(); + + [[gnu::noinline]] + T * allocChunk() + { + return static_cast(::operator new(ChunkSize * sizeof(T))); + } /** * Keep this out of the ::add hot path */ [[gnu::noinline]] - auto & addChunk() + T * ensureChunk(size_t chunkIdx) { - if (size_ >= std::numeric_limits::max() - ChunkSize) + if (chunkIdx >= MaxChunks) unreachable(); - chunks.emplace_back(); - chunks.back().reserve(ChunkSize); - return chunks.back(); + + /* Want synchronises-with relation with the CAS. */ + auto * p = (*chunks)[chunkIdx].load(std::memory_order_acquire); + if (p) [[likely]] + return p; + + auto * newChunk = allocChunk(); + T * expected = nullptr; + /* Try to allocate the chunk ourselves. If CAS fails then somebody beat + us to it. Release semantics for the winner and acquire semantics for + the loser. */ + if ((*chunks)[chunkIdx].compare_exchange_strong( + expected, newChunk, /*success=*/std::memory_order_release, /*failure=*/std::memory_order_acquire)) + return newChunk; /* We succeeded. */ + + /* Somebody beat us to it. */ + ::operator delete(newChunk); + assert(expected); + return expected; + } + + std::size_t numActiveChunks() const noexcept + { + return (size_.load(std::memory_order_relaxed) + ChunkSize - 1) / ChunkSize; } public: - ChunkedVector(uint32_t reserve) + ChunkedVector() = default; + + ChunkedVector(ChunkedVector &&) = delete; + ChunkedVector(const ChunkedVector &) = delete; + ChunkedVector & operator=(ChunkedVector &&) = delete; + ChunkedVector & operator=(const ChunkedVector &) = delete; + + ~ChunkedVector() { - chunks.reserve(reserve); - addChunk(); + auto len = size(); + for (auto & chunkPtr : std::ranges::views::take(*chunks, numActiveChunks())) { + auto * chunk = chunkPtr.load(std::memory_order_relaxed); + auto count = std::min(len, ChunkSize); + if constexpr (!std::is_trivially_destructible_v) { + std::destroy_n(chunk, count); + } + ::operator delete(chunk); + len -= count; + } } + /** + * Get the current size. + */ uint32_t size() const noexcept { - return size_; + return size_.load(std::memory_order_relaxed); } template std::pair add(Args &&... args) { - const auto idx = size_++; - auto & chunk = [&]() -> auto & { - if (auto & back = chunks.back(); back.size() < ChunkSize) - return back; - return addChunk(); - }(); - auto & result = chunk.emplace_back(std::forward(args)...); - return {result, idx}; + /* Get some unique index. Doesn't need any synchronises-with relation. */ + const auto idx = size_.fetch_add(1, std::memory_order_relaxed); + auto * chunk = ensureChunk(idx / ChunkSize); + auto * p = new (&chunk[idx % ChunkSize]) T(std::forward(args)...); + return {*p, idx}; } /** @@ -69,17 +122,26 @@ public: * @pre add must have been called at least idx + 1 times. * @throws nothing */ - const T & operator[](uint32_t idx) const noexcept + const T & operator[](uint32_t idx) const & noexcept { - return chunks[idx / ChunkSize][idx % ChunkSize]; + /* Synchronises-with relation with add(). */ + auto * chunk = (*chunks)[idx / ChunkSize].load(std::memory_order_acquire); + return chunk[idx % ChunkSize]; } + /** + * Iterate over all elements. Do not use when there might be concurrent writers. + */ template void forEach(Fn fn) const { - for (const auto & c : chunks) - for (const auto & e : c) - fn(e); + auto len = size(); + for (auto & chunkPtr : std::ranges::views::take(*chunks, numActiveChunks())) { + auto * chunk = chunkPtr.load(std::memory_order_relaxed); + auto count = std::min(len, ChunkSize); + std::for_each_n(chunk, count, fn); + len -= count; + } } }; } // namespace nix From 5fbf4f90e84a21ae0a8dd081c4f4c4649d6c73bc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 6 Mar 2026 19:41:07 +0300 Subject: [PATCH 052/555] packaging: Add build with thread sanitizer If we are going to go all-in on threading then we might as well utilise all the tooling we have to get it right. I'm not super confident in acquire/release semantics in the previous commit, but TSan would barf if I had gotten it wrong. Only enabled on 64 bit linux systems - i686 doesn't support it. If warranted, we could probably enable it on darwin too, but I haven't tried. --- packaging/components.nix | 12 +++++++++++- packaging/hydra.nix | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/packaging/components.nix b/packaging/components.nix index b74484b6e9a2..72707a094ba9 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -209,8 +209,13 @@ let enableSanitizersLayer = finalAttrs: prevAttrs: let - sanitizers = lib.optional scope.withASan "address" ++ lib.optional scope.withUBSan "undefined"; + sanitizers = + lib.optional scope.withASan "address" + ++ lib.optional scope.withUBSan "undefined" + ++ lib.optional scope.withTSan "thread"; in + # Thread sanitizer can't be used with ASan or UBSan + assert scope.withTSan -> !(scope.withASan || scope.withUBSan); { mesonFlags = (prevAttrs.mesonFlags or [ ]) @@ -277,6 +282,11 @@ in */ withUBSan = false; + /** + Whether meson components are built with [ThreadSanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html). + */ + withTSan = false; + /** A user-provided extension function to apply to each component derivation. */ diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 9b9b33d6798b..ce50edd1ea30 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -187,6 +187,26 @@ rec { ) (forAllSystems (system: components.${system}.${pkgName})) ); + # Separate build because one cannot mix ASan + UBSan with TSan. + buildWithTSan = + let + components = + system: + let + pkgs = nixpkgsFor.${system}.native; + in + pkgs.nixComponents2.overrideScope ( + self: super: { + withTSan = true; + # Dies at startup. + nix-perl-bindings = null; + # TSan has issues with fork and threads. + nix-functional-tests = super.nix-functional-tests.overrideAttrs { doCheck = false; }; + } + ); + in + forAllPackages (pkgName: lib.genAttrs linux64BitSystems (system: (components system).${pkgName})); + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-cli); # Toggles some settings for better coverage. Windows needs these From f7dd04d1002a0c309d682e06de62633321658614 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 7 Mar 2026 21:43:02 +0300 Subject: [PATCH 053/555] libstore: Fix issues with FreeBSD derivation builder Various cleanup and bugfixes on the PR that was merged in quite a rough state. In the order of significance. 1. Stop following symlinks when mounting things in the jail. This was busted. 2. Restore missing unix::closeExtraFDs() for darwin/unsandboxed freebsd builder by putting it back in runChild. This has silently regressed a lot of derivation builders like the Darwin/Linux ones which is a non-starter. 3. Handle optional chrootPaths. 4. More helpful exception in addDependency. 5. Fix nmount call ignoring errmsg argument because it had a hardcoded count of 6 iovec. 6. Various code style and std::filesystem::path cleanups. --- src/libstore/unix/build/derivation-builder.cc | 3 + .../unix/build/freebsd-derivation-builder.cc | 164 +++++++++--------- .../unix/build/linux-derivation-builder.cc | 3 - 3 files changed, 89 insertions(+), 81 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 5fc4d05b4895..f81e8a42fd23 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1324,6 +1324,9 @@ void DerivationBuilderImpl::runChild(RunChildArgs args) if (chdir(tmpDirInSandbox().c_str()) == -1) throw SysError("changing into %1%", PathFmt(tmpDir)); + /* Close all other file descriptors. */ + unix::closeExtraFDs(); + /* Disable core dumps by default. */ struct rlimit limit = {0, RLIM_INFINITY}; setrlimit(RLIMIT_CORE, &limit); diff --git a/src/libstore/unix/build/freebsd-derivation-builder.cc b/src/libstore/unix/build/freebsd-derivation-builder.cc index 3a42f230727f..c59e96e864a8 100644 --- a/src/libstore/unix/build/freebsd-derivation-builder.cc +++ b/src/libstore/unix/build/freebsd-derivation-builder.cc @@ -1,10 +1,12 @@ #ifdef __FreeBSD__ +# include "nix/util/freebsd-jail.hh" +# include "nix/util/util.hh" + # include # include # include -# include # include # include # include @@ -15,9 +17,6 @@ # include # include -# include "nix/util/freebsd-jail.hh" -# include "nix/util/util.hh" - namespace nix { namespace { @@ -32,15 +31,13 @@ struct PasswordEntry std::filesystem::path shell; }; -static void free_db(DB * db) -{ - if (db != nullptr) { - (db->close)(db); - } -} +using UniqueDB = std::unique_ptr<::DB, decltype([](::DB * db) { + if (db) + (db->close)(db); + })>; // Database open flags from FreeBSD, in case they're necessary for compatibility -static const HASHINFO db_flags = { +static constexpr HASHINFO dbFlags = { .bsize = 4096, .ffactor = 32, .nelem = 256, @@ -55,14 +52,12 @@ static const uint8_t dbVersion = 4; static void serializeString(std::vector & buf, std::string const & str) { - buf.reserve(buf.size() + str.size() + 1); buf.insert(buf.end(), str.begin(), str.end()); buf.push_back(0); } static void serializeInt(std::vector & buf, uint32_t num) { - buf.reserve(buf.size() + sizeof(num)); // Always big endian buf.push_back((num >> 24) & 0xff); buf.push_back((num >> 16) & 0xff); @@ -74,7 +69,7 @@ static std::vector byNameKey(std::string const & name) { std::vector buf{_PW_VERSIONED(_PW_KEYBYNAME, dbVersion)}; buf.reserve(1 + name.size()); - // We can't use serializeString since that's null terimated + // We can't use serializeString since that's null terminated buf.insert(buf.end(), name.begin(), name.end()); return buf; @@ -98,19 +93,18 @@ static std::vector byUidKey(uid_t uid) static void createPasswordFiles(std::filesystem::path & chrootRootDir, std::vector & users) { - std::unique_ptr db( - dbopen((chrootRootDir + "/etc/pwd.db").c_str(), O_CREAT | O_RDWR | O_EXCL, 0644, DB_HASH, &db_flags), &free_db); + auto db = + UniqueDB(::dbopen((chrootRootDir / "etc/pwd.db").c_str(), O_CREAT | O_RDWR | O_EXCL, 0644, DB_HASH, &dbFlags)); - if (db == nullptr) { - throw SysError("Could not create password database"); - } + if (!db) + throw SysError("could not create password database"); - auto dbInsert = [&db](std::vector key_buf, std::vector & value_buf) { - DBT key = {key_buf.data(), key_buf.size()}; - DBT value = {value_buf.data(), value_buf.size()}; + auto dbInsert = [&db](std::vector keyBuf, std::vector & valueBuf) { + DBT key = {keyBuf.data(), keyBuf.size()}; + DBT value = {valueBuf.data(), valueBuf.size()}; if ((db->put)(db.get(), &key, &value, R_NOOVERWRITE) == -1) { - throw SysError("Could not write to password database"); + throw SysError("could not write to password database"); } }; @@ -120,9 +114,7 @@ static void createPasswordFiles(std::filesystem::path & chrootRootDir, std::vect std::vector versionValue{dbVersion}; dbInsert(versionKey, versionValue); - for (size_t i = 0; i < users.size(); i++) { - auto user = users[i]; - + for (const auto & [i, user] : enumerate(users)) { // flags for non-empty fields uint32_t fields = _PWF_NAME | _PWF_PASSWD | _PWF_UID | _PWF_GID | _PWF_GECOS | _PWF_DIR | _PWF_SHELL; @@ -150,19 +142,19 @@ static void createPasswordFiles(std::filesystem::path & chrootRootDir, std::vect } // FreeBSD libc doesn't use /etc/passwd, but some software might - std::string passwdContent = ""; - for (auto user : users) { + std::string passwdContent; + for (const auto & user : users) { passwdContent.append( fmt("%s:*:%d:%d:%s:%s:%s\n", user.name, user.uid, user.gid, user.description, - PathFmt(user.home), - PathFmt(user.shell))); + user.home.native(), + user.shell.native())); } - writeFile(chrootRootDir + "/etc/passwd", passwdContent); + writeFile(chrootRootDir / "etc/passwd", passwdContent); // No need to make /etc/master.passwd or /etc/spwd.db, // our build user wouldn't be able to read them anyway @@ -175,12 +167,21 @@ struct FreeBSDDerivationBuilder : virtual DerivationBuilderImpl using DerivationBuilderImpl::DerivationBuilderImpl; }; -template -struct iovec iovFromStaticSizedString(const char (&array)[n]) +template +struct iovec iovFromMutableBuffer(std::array & array) { return { - .iov_base = const_cast(static_cast(&array[0])), - .iov_len = n, + .iov_base = static_cast(array.data()), + .iov_len = N, + }; +} + +template +struct iovec iovFromStaticSizedString(const char (&array)[N]) +{ + return { + .iov_base = const_cast(static_cast(array)), + .iov_len = N, }; } @@ -245,7 +246,7 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati // FreeBSD doesn't have a group database, just write a text file writeFile( - chrootRootDir + "/etc/group", + chrootRootDir / "etc/group", fmt("root:x:0:\n" "nixbld:!:%1%:\n" "nogroup:x:65534:\n", @@ -255,60 +256,66 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati // pollute the root mount namespace. // FreeBSD doesn't have mount namespaces, so there's no reason to wait. - auto devpath = chrootRootDir + "/dev"; - mkdir(devpath.c_str(), 0555); - mkdir((chrootRootDir + "/bin").c_str(), 0555); - char errmsg[255] = ""; - struct iovec iov[8] = { + auto devpath = chrootRootDir / "dev"; + createDir(devpath, 0555); + createDir(chrootRootDir / "bin", 0555); + + std::array errmsg{}; + std::array<::iovec, 8> iov = { iovFromStaticSizedString("fstype"), iovFromStaticSizedString("devfs"), iovFromStaticSizedString("fspath"), - iovFromDynamicSizeString(devpath), + iovFromDynamicSizeString(devpath.native()), iovFromStaticSizedString("ruleset"), iovFromStaticSizedString("4"), iovFromStaticSizedString("errmsg"), - iovFromStaticSizedString(errmsg), + iovFromMutableBuffer(errmsg), }; - if (nmount(iov, 6, 0) < 0) { - throw SysError("Failed to mount jail /dev: %1%", PathFmt(errmsg)); - } - autoDelJail->childrenMounts.emplace_back(devpath); - for (auto & i : pathsInChroot) { - char errmsg[255]; - errmsg[0] = 0; + if (nmount(iov.data(), iov.size(), 0) < 0) + throw SysError("failed to mount jail /dev: %1%", std::string_view(errmsg.data())); - if (i.second.source == "/proc") { - continue; // backwards compatibility - } - std::filesystem::path path = chrootRootDir + i.first.c_str(); + autoDelJail->childrenMounts.emplace_back(devpath); - struct stat stat_buf; - if (stat(i.second.source.c_str(), &stat_buf) < 0) { - throw SysError("stat"); + for (const auto & [target, chrootPath] : pathsInChroot) { + std::filesystem::path path = chrootRootDir / target.relative_path(); + + auto maybeSt = maybeLstat(chrootPath.source); + if (!maybeSt) { + if (chrootPath.optional) + continue; /* Skip mounting this path. */ + else + throw SysError("getting attributes of path %1%", PathFmt(chrootPath.source)); } - // mount points must exist and be the right type - if (S_ISDIR(stat_buf.st_mode)) { + /* Mount points must exist and be the right type. */ + if (S_ISDIR(maybeSt->st_mode)) { createDirs(path); + } else if (S_ISLNK(maybeSt->st_mode)) { + createDirs(path.parent_path()); + copyFile(chrootPath.source, path, /*andDelete=*/false, /*contents=*/false); + continue; } else { createDirs(path.parent_path()); writeFile(path, ""); } - struct iovec iov[8] = { + std::array<::iovec, 8> iov = { iovFromStaticSizedString("fstype"), iovFromStaticSizedString("nullfs"), iovFromStaticSizedString("fspath"), - iovFromDynamicSizeString(path), + iovFromDynamicSizeString(path.native()), iovFromStaticSizedString("target"), - iovFromDynamicSizeString(i.second.source), + iovFromDynamicSizeString(chrootPath.source.native()), iovFromStaticSizedString("errmsg"), - iovFromStaticSizedString(errmsg), + iovFromMutableBuffer(errmsg), }; - if (nmount(iov, 8, 0) < 0) { - throw SysError("Failed to mount nullfs for %1% - %2%", PathFmt(path), errmsg); - } + + debug("setting up a nullfs mount from %1% to %2%", PathFmt(chrootPath.source), PathFmt(path)); + + if (nmount(iov.data(), iov.size(), 0) < 0) + throw SysError("failed to mount nullfs for %1%: %2%", PathFmt(path), std::string_view(errmsg.data())); + autoDelJail->childrenMounts.emplace_back(path); } @@ -320,7 +327,7 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati // services. Don’t use it for anything else that may // be configured for this system. This limits the // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + writeFile(chrootRootDir / "etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); /* N.B. it is realistic that these paths might not exist. It happens when testing Nix building fixed-output derivations @@ -346,10 +353,10 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati if (fileTransferSettings.caFile.get() && pathExists(fileTransferSettings.caFile.get().value())) { // For the same reasons as above, copy the CA certificates file too. // It should be even less likely to change during the build than resolv.conf. - createDirs(chrootRootDir + "/etc/ssl/certs"); + createDirs(chrootRootDir / "etc/ssl/certs"); copyFile( fileTransferSettings.caFile.get().value(), - chrootRootDir + "/etc/ssl/certs/ca-certificates.crt", + chrootRootDir / "etc/ssl/certs/ca-certificates.crt", false, true); } @@ -378,9 +385,9 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati // TODO: Make our own ruleset "vnet", "new", - NULL); + nullptr); if (jid < 0) { - throw SysError("Failed to create jail (isolated network): %1%", jail_errmsg); + throw SysError("failed to create jail (isolated network): %1%", jail_errmsg); } autoDelJail->jid = jid; @@ -411,21 +418,22 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati if (!(hdr = snl_finalize_msg(&nw)) || !snl_send_message(&ss, hdr)) { snl_free(&ss); - throw SysError("Failed to sendoff netlink message"); + throw SysError("failed to sendoff netlink message"); } struct snl_errmsg_data e = {}; snl_read_reply_code(&ss, hdr->nlmsg_seq, &e); if (e.error_str != NULL) { snl_free(&ss); - throw SysError("Failed to configure loopback interface: %1%", e.error_str); + throw SysError("failed to configure loopback interface: %1%", e.error_str); } snl_free(&ss); _exit(0); }); + /* TODO: Capture the error from the helper? */ if (helper.wait() != 0) { - throw SysError("Failed to configure loopback address"); + throw SysError("failed to configure loopback address"); } } else { jid = jail_setv( @@ -449,7 +457,7 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati "true", NULL); if (jid < 0) { - throw SysError("Failed to create jail (networked): %1%", jail_errmsg); + throw SysError("failed to create jail (networked): %1%", jail_errmsg); } autoDelJail->jid = jid; } @@ -467,14 +475,14 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati unix::closeExtraFDs(); if (jail_attach(autoDelJail->jid) < 0) { - throw SysError("Failed to attach to jail"); + throw SysError("failed to attach to jail"); } } void addDependency(const StorePath & path) { - auto [source, target] = ChrootDerivationBuilder::addDependencyPrep(path); - throw Error("Unimplemented"); + throw UnimplementedError( + "adding store path '%s' to the sandbox is not implemented (recursive-nix)", store.printStorePath(path)); } }; diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 6af4a0028039..0ffcf0ccb657 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -171,9 +171,6 @@ struct LinuxDerivationBuilder : virtual DerivationBuilderImpl .system = drv.platform, .impersonateLinux26 = localSettings.impersonateLinux26, }); - - /* Close all other file descriptors. */ - unix::closeExtraFDs(); } }; From c0dbe8e42cee61e6249664f74cedd62c6c6256f3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 7 Mar 2026 21:48:52 +0300 Subject: [PATCH 054/555] libstore: Mount store paths readonly, nosuid in sandboxed freebsd builders This isn't done on Linux (supposedly for back-compat reasons). But since the FreeBSD sandbox is a new feature we can try to get it right from the start. --- src/libstore/unix/build/freebsd-derivation-builder.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libstore/unix/build/freebsd-derivation-builder.cc b/src/libstore/unix/build/freebsd-derivation-builder.cc index c59e96e864a8..cf31d4457a0e 100644 --- a/src/libstore/unix/build/freebsd-derivation-builder.cc +++ b/src/libstore/unix/build/freebsd-derivation-builder.cc @@ -313,7 +313,15 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati debug("setting up a nullfs mount from %1% to %2%", PathFmt(chrootPath.source), PathFmt(path)); - if (nmount(iov.data(), iov.size(), 0) < 0) + int flags = 0; + if (store.isInStore(target.native())) + /* While we are at it, enforce invariants about store paths. Anything located at the "logical" store + location must be readonly (file permission canonicalisation enforces this on the host filesystem). + Also the store must never contain setuid binaries for the same reason. This is just defense-in-depth. + */ + flags = MNT_RDONLY | MNT_NOSUID; + + if (nmount(iov.data(), iov.size(), flags) < 0) throw SysError("failed to mount nullfs for %1%: %2%", PathFmt(path), std::string_view(errmsg.data())); autoDelJail->childrenMounts.emplace_back(path); From 42a735000d9e70f0b7c108992cd7635aa9e0fd46 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 7 Mar 2026 21:51:11 +0300 Subject: [PATCH 055/555] libstore/freebsd-derivation-builder: NULL -> nullptr Let's not fix legacy C things in C++. --- src/libstore/unix/build/freebsd-derivation-builder.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/freebsd-derivation-builder.cc b/src/libstore/unix/build/freebsd-derivation-builder.cc index cf31d4457a0e..073daac566f4 100644 --- a/src/libstore/unix/build/freebsd-derivation-builder.cc +++ b/src/libstore/unix/build/freebsd-derivation-builder.cc @@ -431,7 +431,7 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati struct snl_errmsg_data e = {}; snl_read_reply_code(&ss, hdr->nlmsg_seq, &e); - if (e.error_str != NULL) { + if (e.error_str != nullptr) { snl_free(&ss); throw SysError("failed to configure loopback interface: %1%", e.error_str); } @@ -463,7 +463,7 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati "inherit", "allow.raw_sockets", "true", - NULL); + nullptr); if (jid < 0) { throw SysError("failed to create jail (networked): %1%", jail_errmsg); } From b523564eb3b5ec5070d5de82d19906a06726c9ea Mon Sep 17 00:00:00 2001 From: Luna Nova Date: Sat, 7 Mar 2026 19:09:27 -0800 Subject: [PATCH 056/555] libstore: handle root path in RemoteFSAccessor::maybeLstat RemoteFSAccessor::fetch() crashes when called with the root path. $ nix build nixpkgs#hello --store ssh-ng://host error: path '/nix/store/' is not in the Nix store Add path.isRoot() guard like LocalStoreAccessor::maybeLstat Fixes: eb643d034 ("`Store::getFSAccessor`: Do not include the store dir") --- src/libstore/remote-fs-accessor.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index fbb7ac22303c..31356a506a09 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -35,6 +35,8 @@ std::shared_ptr RemoteFSAccessor::accessObject(const StorePath & std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) { + if (path.isRoot()) + return Stat{.type = tDirectory}; auto res = fetch(path); return res.first->maybeLstat(res.second); } From f84a7d2a4d8a202e30481638fb360be88a12dea3 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 20 Feb 2026 19:28:55 -0500 Subject: [PATCH 057/555] libstore: use Windows known folders API for runtime path configuration On Windows the compile-time path constants (`NIX_CONF_DIR`, `NIX_STATE_DIR`, `NIX_LOG_DIR`) are meaningless because the install drive varies, so this resolves them at runtime via `FOLDERID_ProgramData` through the existing known-folders API. Co-authored-by: John Ericson --- src/libstore/globals.cc | 78 ++++++++++++++++++++++++++-------------- src/libstore/meson.build | 10 +++--- 2 files changed, 57 insertions(+), 31 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 2e262aec8a0a..ece19d8525ad 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -53,29 +53,23 @@ namespace nix { Nix daemon by setting the mode/ownership of the directory appropriately. (This wouldn't work on the socket itself since it must be deleted and recreated on startup.) */ -#define DEFAULT_SOCKET_PATH "daemon-socket/socket" +#define DEFAULT_SOCKET_PATH "daemon-socket" / "socket" -/** - * Helper to resolve the NIX_CONF_DIR at runtime on Windows. - * On Windows, NIX_CONF_DIR is not defined at compile time, so we determine - * the path at runtime using the Windows known folders API (FOLDERID_ProgramData). - * This allows Nix to work correctly regardless of which drive Windows is installed on. - */ -static std::filesystem::path resolveNixConfDir() -{ +LogFileSettings::LogFileSettings() + : nixLogDir(getEnvOsNonEmpty(OS_STR("NIX_LOG_DIR")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { #ifdef _WIN32 -# ifdef NIX_CONF_DIR - // On Windows, NIX_CONF_DIR should not be defined at compile time -# error "NIX_CONF_DIR should not be defined on Windows" +# ifdef NIX_LOG_DIR +# error "NIX_LOG_DIR should not be defined on Windows" # endif - return windows::known_folders::getProgramData() / "nix"; + return windows::known_folders::getProgramData() / "nix" / "log"; #else - return NIX_CONF_DIR; + return NIX_LOG_DIR; #endif -} - -LogFileSettings::LogFileSettings() - : nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) + }) + .transform([](auto && s) { return canonPath(s); }) + .value()) { } @@ -84,10 +78,26 @@ Settings settings; static GlobalConfig::Register rSettings(&settings); Settings::Settings() - : nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) - , nixDaemonSocketFile(canonPath(getEnvOsNonEmpty(OS_STR("NIX_DAEMON_SOCKET_PATH")) - .transform([](auto && s) { return std::filesystem::path(s); }) - .value_or(nixStateDir / DEFAULT_SOCKET_PATH))) + : nixStateDir(getEnvOsNonEmpty(OS_STR("NIX_STATE_DIR")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { +#ifdef _WIN32 +# ifdef NIX_STATE_DIR +# error "NIX_STATE_DIR should not be defined on Windows" +# endif + return windows::known_folders::getProgramData() / "nix" / "state"; +#else + return NIX_STATE_DIR; +#endif + }) + .transform([](auto && s) { return canonPath(s); }) + .value()) + , nixDaemonSocketFile( + getEnvOsNonEmpty(OS_STR("NIX_DAEMON_SOCKET_PATH")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([this]() -> std::optional { return nixStateDir / DEFAULT_SOCKET_PATH; }) + .transform([](auto && s) { return canonPath(s); }) + .value()) { #ifndef _WIN32 buildUsersGroup = isRootUser() ? "nixbld" : ""; @@ -151,12 +161,28 @@ void loadConfFile(AbstractConfig & config) } } +/** + * On Windows, NIX_CONF_DIR (and other directories like NIX_STATE_DIR, NIX_LOG_DIR) + * are not defined at compile time, so we determine paths at runtime using the + * Windows known folders API (FOLDERID_ProgramData). This allows Nix to work + * correctly regardless of which drive Windows is installed on. + */ const std::filesystem::path & nixConfDir() { - static const std::filesystem::path dir = - canonPath(getEnvOsNonEmpty(OS_STR("NIX_CONF_DIR")) - .transform([](auto && s) { return std::filesystem::path(s); }) - .value_or(resolveNixConfDir())); + static const std::filesystem::path dir = getEnvOsNonEmpty(OS_STR("NIX_CONF_DIR")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { +#ifdef _WIN32 +# ifdef NIX_CONF_DIR +# error "NIX_CONF_DIR should not be defined on Windows" +# endif + return windows::known_folders::getProgramData() / "nix" / "conf"; +#else + return NIX_CONF_DIR; +#endif + }) + .transform([](auto && s) { return canonPath(s); }) + .value(); return dir; } diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 63eba3125531..96fd9f69c616 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -249,14 +249,14 @@ endif # by joining it with prefix, unless it was already an absolute path # (which is the default for store-dir, localstatedir, and log-dir). configdata_priv.set_quoted('NIX_STORE_DIR', store_dir) -configdata_priv.set_quoted('NIX_STATE_DIR', localstatedir / 'nix') -configdata_priv.set_quoted('NIX_LOG_DIR', log_dir) -# On Windows, NIX_CONF_DIR is determined at runtime using the Windows known -# folders API (FOLDERID_ProgramData), so we don't define it at compile time. +# On Windows, NIX_STATE_DIR, NIX_LOG_DIR, and NIX_CONF_DIR are determined at +# runtime using the Windows known folders API (FOLDERID_ProgramData), so we +# don't define them at compile time. if host_machine.system() != 'windows' + configdata_priv.set_quoted('NIX_STATE_DIR', localstatedir / 'nix') + configdata_priv.set_quoted('NIX_LOG_DIR', log_dir) configdata_priv.set_quoted('NIX_CONF_DIR', sysconfdir / 'nix') endif -configdata_priv.set_quoted('NIX_MAN_DIR', mandir) lsof = find_program('lsof', required : false) configdata_priv.set_quoted( From 6de78b8095099db6d7245e97350b3ffb27a1224b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Jul 2025 19:44:13 +0200 Subject: [PATCH 058/555] AllowListSourceAccessor: Make thread-safe --- src/libfetchers/filtering-source-accessor.cc | 27 ++++++++++--------- src/libfetchers/git-utils.cc | 4 +-- .../nix/fetchers/filtering-source-accessor.hh | 7 ++--- src/libutil/include/nix/util/sync.hh | 7 +++++ 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 0daedcf7c78f..6fe7d2504ec3 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -1,6 +1,7 @@ #include "nix/fetchers/filtering-source-accessor.hh" +#include "nix/util/sync.hh" -#include +#include namespace nix { @@ -74,39 +75,39 @@ void FilteringSourceAccessor::checkAccess(const CanonPath & path) struct AllowListSourceAccessorImpl : AllowListSourceAccessor { - std::set allowedPrefixes; - boost::unordered_flat_set allowedPaths; + SharedSync> allowedPrefixes; + boost::concurrent_flat_set allowedPaths; AllowListSourceAccessorImpl( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError) : AllowListSourceAccessor(SourcePath(next), std::move(makeNotAllowedError)) - , allowedPrefixes(std::move(allowedPrefixes)) - , allowedPaths(std::move(allowedPaths)) + , allowedPrefixes(allowedPrefixes.begin(), allowedPrefixes.end()) + , allowedPaths(allowedPaths.begin(), allowedPaths.end()) { } bool isAllowed(const CanonPath & path) override { - return allowedPaths.contains(path) || path.isAllowed(allowedPrefixes); + /* Read lock is held for the duration of the full expression if the || doesn't short-circuit. */ + return allowedPaths.contains(path) || path.isAllowed(*allowedPrefixes.readLock()); } void allowPrefix(CanonPath prefix) override { - allowedPrefixes.insert(std::move(prefix)); + allowedPrefixes.lock()->insert(std::move(prefix)); } }; ref AllowListSourceAccessor::create( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError) { - return make_ref( - next, std::move(allowedPrefixes), std::move(allowedPaths), std::move(makeNotAllowedError)); + return make_ref(next, allowedPrefixes, allowedPaths, std::move(makeNotAllowedError)); } bool CachingFilteringSourceAccessor::isAllowed(const CanonPath & path) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 456d869983c2..fcffdcfa8aef 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1435,9 +1435,9 @@ ref GitRepoImpl::getAccessor( auto self = ref(shared_from_this()); ref fileAccessor = AllowListSourceAccessor::create( makeFSSourceAccessor(path), - std::set{wd.files}, + /*allowedPrefixes=*/wd.files, // Always allow access to the root, but not its children. - boost::unordered_flat_set{CanonPath::root}, + /*allowedPaths=*/{CanonPath::root}, std::move(makeNotAllowedError)) .cast(); if (options.exportIgnore) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 98532c4b14c2..13272719fe3d 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -2,7 +2,8 @@ #include "nix/util/source-path.hh" -#include +#include +#include namespace nix { @@ -79,8 +80,8 @@ struct AllowListSourceAccessor : public FilteringSourceAccessor static ref create( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError); using FilteringSourceAccessor::FilteringSourceAccessor; diff --git a/src/libutil/include/nix/util/sync.hh b/src/libutil/include/nix/util/sync.hh index 6f3298324ff8..71f4fa05628f 100644 --- a/src/libutil/include/nix/util/sync.hh +++ b/src/libutil/include/nix/util/sync.hh @@ -50,6 +50,13 @@ public: { } + template + SyncBase(Ts &&... args) + requires requires { T{std::forward(args)...}; } + : data(std::forward(args)...) + { + } + SyncBase(SyncBase && other) noexcept : data(std::move(*other.lock())) { From 30e71b1d7831a2522ba024bffb398aa8831d9326 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 9 Mar 2026 12:04:59 -0400 Subject: [PATCH 059/555] tests: add `UnkeyedRealisation` JSON characterisation tests This commit extracts `UnkeyedRealisation` test values from `Realisation` and reuses them, matching how `Realisation` composes `UnkeyedRealisation` + `DrvOutput`. The new unkeyed-simple and unkeyed-with-signature fixtures test the JSON format independently of the keyed wrapper. --- .../data/realisation/unkeyed-simple.json | 4 + .../realisation/unkeyed-with-signature.json | 9 ++ src/libstore-tests/realisation.cc | 112 +++++++++++++----- 3 files changed, 95 insertions(+), 30 deletions(-) create mode 100644 src/libstore-tests/data/realisation/unkeyed-simple.json create mode 100644 src/libstore-tests/data/realisation/unkeyed-with-signature.json diff --git a/src/libstore-tests/data/realisation/unkeyed-simple.json b/src/libstore-tests/data/realisation/unkeyed-simple.json new file mode 100644 index 000000000000..2cf61d215206 --- /dev/null +++ b/src/libstore-tests/data/realisation/unkeyed-simple.json @@ -0,0 +1,4 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] +} diff --git a/src/libstore-tests/data/realisation/unkeyed-with-signature.json b/src/libstore-tests/data/realisation/unkeyed-with-signature.json new file mode 100644 index 000000000000..c51c6127a765 --- /dev/null +++ b/src/libstore-tests/data/realisation/unkeyed-with-signature.json @@ -0,0 +1,9 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] +} diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index 1bc7708e7c41..204d33874097 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -11,6 +11,43 @@ namespace nix { +using nlohmann::json; + +/* ---------------------------------------------------------------------------- + * Test data + * --------------------------------------------------------------------------*/ + +UnkeyedRealisation unkeyedSimple{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, +}; + +UnkeyedRealisation unkeyedWithSignature{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + }, +}; + +DrvOutput testDrvOutput{ + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, + .outputName = "foo", +}; + +Realisation simple{ + unkeyedSimple, + testDrvOutput, +}; + +Realisation withSignature{ + unkeyedWithSignature, + testDrvOutput, +}; + +/* ---------------------------------------------------------------------------- + * Realisation JSON + * --------------------------------------------------------------------------*/ + class RealisationTest : public JsonCharacterizationTest, public LibStoreTest { std::filesystem::path unitTestData = getUnitTestData() / "realisation"; @@ -23,12 +60,6 @@ class RealisationTest : public JsonCharacterizationTest, public Lib } }; -/* ---------------------------------------------------------------------------- - * JSON - * --------------------------------------------------------------------------*/ - -using nlohmann::json; - struct RealisationJsonTest : RealisationTest, ::testing::WithParamInterface> {}; @@ -44,30 +75,6 @@ TEST_P(RealisationJsonTest, to_json) writeJsonTest(name, value); } -Realisation simple{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, - .outputName = "foo", - }, -}; - -Realisation withSignature{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - }, - }, - { - .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, - .outputName = "foo", - }, -}; - INSTANTIATE_TEST_SUITE_P( RealisationJSON, RealisationJsonTest, @@ -89,4 +96,49 @@ TEST_F(RealisationTest, with_signature_from_json) readJsonTest("with-signature", withSignature); } +/* ---------------------------------------------------------------------------- + * UnkeyedRealisation JSON + * --------------------------------------------------------------------------*/ + +class UnkeyedRealisationTest : public JsonCharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "realisation"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +struct UnkeyedRealisationJsonTest : UnkeyedRealisationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(UnkeyedRealisationJsonTest, from_json) +{ + const auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(UnkeyedRealisationJsonTest, to_json) +{ + const auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + UnkeyedRealisationJSON, + UnkeyedRealisationJsonTest, + ::testing::Values( + std::pair{ + "unkeyed-simple", + unkeyedSimple, + }, + std::pair{ + "unkeyed-with-signature", + unkeyedWithSignature, + })); + } // namespace nix From 02e4f4ad75fda707a594ffd8ad511691fc1a311f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 9 Mar 2026 15:31:14 +0300 Subject: [PATCH 060/555] libutil: Implement descriptor-based PosixFileSourceAccessor Changes makeFSSourceAccessor to open a file descriptor first and then operates on it. Symlinks get copied into a MemorySourceAccessor instead to avoid any accidental symlink following issues. This does mean that makeFSSourceAccessor now dies on a non-existent path, but all existing call-sites would already fail afterwards, so I think it's fine. --- src/libutil-tests/source-accessor.cc | 4 +- src/libutil/file-descriptor.cc | 3 +- .../include/nix/util/posix-source-accessor.hh | 54 ++++-- .../include/nix/util/source-accessor.hh | 2 + src/libutil/posix-source-accessor.cc | 179 +++++++++++++++--- 5 files changed, 201 insertions(+), 41 deletions(-) diff --git a/src/libutil-tests/source-accessor.cc b/src/libutil-tests/source-accessor.cc index fd4e2429e11a..1ee2c44cc6fc 100644 --- a/src/libutil-tests/source-accessor.cc +++ b/src/libutil-tests/source-accessor.cc @@ -127,9 +127,7 @@ TEST_F(FSSourceAccessorTest, works) } { - auto accessor = makeFSSourceAccessor(tmpDir / "nonexistent"); - EXPECT_FALSE(accessor->maybeLstat(CanonPath::root)); - EXPECT_THROW(accessor->readFile(CanonPath::root), SystemError); + EXPECT_THROW(makeFSSourceAccessor(tmpDir / "nonexistent"), SystemError); } { diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index 61c444d4fe78..acd273ca79d9 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -1,3 +1,4 @@ +#include "nix/util/file-system.hh" #include "nix/util/serialise.hh" #include "nix/util/util.hh" #include "nix/util/signals.hh" @@ -208,7 +209,7 @@ void copyFdRange(Descriptor fd, off_t offset, size_t nbytes, Sink & sink) auto limit = std::min(left, buf.size()); auto n = readOffset(fd, offset, std::span(buf.data(), limit)); if (n == 0) - throw EndOfFile("unexpected end-of-file"); + throw EndOfFile("unexpected end-of-file reading from %1%", PathFmt(descriptorToPath(fd))); assert(n <= left); sink(std::string_view(reinterpret_cast(buf.data()), n)); offset += n; diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index f9f0309365e7..d93b68e0afea 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -6,10 +6,49 @@ namespace nix { struct SourcePath; +namespace detail { + +/** + * Common base helper class for deduplicating common code paths for tracking mtime. + */ +class PosixSourceAccessorBase : virtual public SourceAccessor +{ +protected: + const bool trackLastModified = false; + + /** + * The most recent mtime seen by lstat(). This is a hack to + * support dumpPathAndGetMtime(). Should remove this eventually. + */ + time_t mtime = 0; + + void maybeUpdateMtime(time_t seenMTime) + { + /* The contract is that trackLastModified implies that the caller uses the accessor + from a single thread. Thus this is not a CAS loop. */ + if (trackLastModified) + mtime = std::max(mtime, seenMTime); + } + + PosixSourceAccessorBase(bool trackLastModified) + : trackLastModified(trackLastModified) + { + } + + virtual std::optional getLastModified() override + { + if (trackLastModified) + return mtime; + return std::nullopt; + } +}; + +} // namespace detail + /** * A source accessor that uses the Unix filesystem. */ -class PosixSourceAccessor : virtual public SourceAccessor +class PosixSourceAccessor : public detail::PosixSourceAccessorBase { /** * Optional root path to prefix all operations into the native file @@ -18,19 +57,11 @@ class PosixSourceAccessor : virtual public SourceAccessor */ const std::filesystem::path root; - const bool trackLastModified = false; - public: PosixSourceAccessor(); PosixSourceAccessor(std::filesystem::path && root, bool trackLastModified = false); - /** - * The most recent mtime seen by lstat(). This is a hack to - * support dumpPathAndGetMtime(). Should remove this eventually. - */ - time_t mtime = 0; - void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; using SourceAccessor::readFile; @@ -75,11 +106,6 @@ public: */ static SourcePath createAtRoot(const std::filesystem::path & path, bool trackLastModified = false); - std::optional getLastModified() override - { - return trackLastModified ? std::optional{mtime} : std::nullopt; - } - void invalidateCache(const CanonPath & path) override; private: diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index bcc25b99a5aa..c458cf8b5c35 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -259,6 +259,8 @@ ref getFSSourceAccessor(); * that it is not possible to escape `root` by appending `..` path * elements, and that absolute symlinks are resolved relative to * `root`. + * + * Symlinks in parents of `root` are resolved. Final symlink is not. */ ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified = false); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 7d0b1df75a9a..730ac8efaab5 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -1,4 +1,6 @@ #include "nix/util/posix-source-accessor.hh" +#include "nix/util/file-system-at.hh" +#include "nix/util/memory-source-accessor.hh" #include "nix/util/source-path.hh" #include "nix/util/signals.hh" #include "nix/util/sync.hh" @@ -7,9 +9,125 @@ namespace nix { +static SourceAccessor::Stat sourceAccessorStatFromPosixStat(const PosixStat & st) +{ + using enum SourceAccessor::Type; + return SourceAccessor::Stat{ + .type = S_ISREG(st.st_mode) ? tRegular + : S_ISDIR(st.st_mode) ? tDirectory + : S_ISLNK(st.st_mode) ? tSymlink + : S_ISCHR(st.st_mode) ? tChar + : S_ISBLK(st.st_mode) ? tBlock + : +#ifdef S_ISSOCK + S_ISSOCK(st.st_mode) ? tSocket + : +#endif + S_ISFIFO(st.st_mode) ? tFifo + : tUnknown, + .fileSize = S_ISREG(st.st_mode) ? std::optional(st.st_size) : std::nullopt, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR, + }; +} + +namespace { + +class PosixFileSourceAccessor : public detail::PosixSourceAccessorBase +{ + AutoCloseFD fd; + std::filesystem::path fsPath; + /** + * Stat is memoised once opened. This does mean that modifying the same file + * while we are reading is busted, but caching it might be considered an + * improvement. Since we have a file descriptor for a regular file it can't + * be swapped out for another file type. Thus, we are only really caching + * the file size and mtime, which shouldn't change. Providing a consistent + * size value here is also fine. If the file becomes smaller than we expect + * then readFile will barf with an EOF exception. If it becomes larger then + * we are just going to silently ignore the extra bytes. + */ + PosixStat st; + +public: + PosixFileSourceAccessor(AutoCloseFD fd, std::filesystem::path path, bool trackLastModified, const PosixStat & st_) + : PosixSourceAccessorBase(trackLastModified) + , fd(std::move(fd)) + , fsPath(std::move(path)) + , st(st_) + { + assert(S_ISREG(st.st_mode)); + assert(fsPath.is_absolute()); /* Only used for error messages, but still nice to enforce this invariant. */ + setPathDisplay(fsPath.generic_string()); + maybeUpdateMtime(st.st_mtime); + } + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; + + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override + { + if (path.isRoot()) + return fsPath; + return std::nullopt; /* Definitely doesn't exist. */ + } + + std::string showPath(const CanonPath & path) override + { + if (path.isRoot()) + return displayPrefix; /* No trailing slash. */ + return displayPrefix + path.abs(); + } +}; + +void PosixFileSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) +{ + if (!path.isRoot()) /* We are the parent and also a regular file. */ + throw NotADirectory("reading file '%1%': %2%", showPath(path), "Not a directory"); + + auto size = st.st_size; + sizeCallback(size); + /* The most important invariant we care about here is writing exactly size + bytes to the sink. copyFdRange should throw an EndOfFile if we fail to read + `size` bytes. */ + copyFdRange(fd.get(), /*offset=*/0, size, sink); +} + +bool PosixFileSourceAccessor::pathExists(const CanonPath & path) +{ + return path.isRoot(); +} + +std::optional PosixFileSourceAccessor::maybeLstat(const CanonPath & path) +{ + if (!path.isRoot()) + return std::nullopt; + return sourceAccessorStatFromPosixStat(st); +} + +SourceAccessor::DirEntries PosixFileSourceAccessor::readDirectory(const CanonPath & path) +{ + throw NotADirectory("reading directory '%1%': %2%", showPath(path), "Not a directory"); +} + +std::string PosixFileSourceAccessor::readLink(const CanonPath & path) +{ + if (!path.isRoot()) + throw NotADirectory("reading symlink '%1%': %2%", showPath(path), "Not a directory"); + throw NotASymlink("path '%1%' is not a symlink", showPath(path)); +} + +} // namespace + PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) - : root(std::move(argRoot)) - , trackLastModified(trackLastModified) + : PosixSourceAccessorBase(trackLastModified) + , root(std::move(argRoot)) { assert(root.empty() || root.is_absolute()); displayPrefix = root.string(); @@ -106,27 +224,8 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP if (!st) return std::nullopt; - /* The contract is that trackLastModified implies that the caller uses the accessor - from a single thread. Thus this is not a CAS loop. */ - if (trackLastModified) - mtime = std::max(mtime, st->st_mtime); - - return Stat{ - .type = S_ISREG(st->st_mode) ? tRegular - : S_ISDIR(st->st_mode) ? tDirectory - : S_ISLNK(st->st_mode) ? tSymlink - : S_ISCHR(st->st_mode) ? tChar - : S_ISBLK(st->st_mode) ? tBlock - : -#ifdef S_ISSOCK - S_ISSOCK(st->st_mode) ? tSocket - : -#endif - S_ISFIFO(st->st_mode) ? tFifo - : tUnknown, - .fileSize = S_ISREG(st->st_mode) ? std::optional(st->st_size) : std::nullopt, - .isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR, - }; + PosixSourceAccessorBase::maybeUpdateMtime(st->st_mtime); + return sourceAccessorStatFromPosixStat(*st); } SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) @@ -211,6 +310,40 @@ ref getFSSourceAccessor() ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified) { +#ifndef _WIN32 + /* Die if the path ends in a / or /. That is a footgun that matters + for symlink resolution. TODO: Strip them or make this a proper error if + this becomes an issue. Defense-in-depth. */ + assert(!root.native().ends_with(OS_STR("/."))); + assert(!root.native().ends_with(OS_STR("/"))); + + AutoCloseFD fd = openFileReadonly(root, FinalSymlink::DontFollow); + + if (!fd) { + if (errno == ELOOP) { + /* This branch is taken either if the final component is a symlink + or we hit a symlink loop. If that's the latter this will also + throw. This can be done with O_PATH descriptor for the symlink + itself, but it's not portable. */ + auto linkTarget = readLink(root); + auto res = make_ref(); + /* Create an in-memory accessor with the symlink at the root. */ + MemorySink sink{*res}; + sink.createSymlink(CanonPath::root, os_string_to_string(linkTarget)); + return res; + } + + throw NativeSysError("opening file %1%", PathFmt(root)); + } + + auto st = nix::fstat(fd.get()); + if (S_ISREG(st.st_mode)) + return make_ref(std::move(fd), std::move(root), trackLastModified, st); + + /* TODO: Use the file descriptor for fd-relative operations on the directory. */ +#endif + return make_ref(std::move(root), trackLastModified); } + } // namespace nix From b919a12f7bd8a9c8a588a0c68b9c44a46b006c2d Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Mon, 9 Mar 2026 09:33:54 -0700 Subject: [PATCH 061/555] libstore: Replace strcpy with std::ranges::copy for loopback interface setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both loopback interface initialization sites used strcpy() to copy the interface name "lo" into ifr.ifr_name. While safe in practice (the literal is 3 bytes, the buffer is IFNAMSIZ/16 bytes), strcpy is flagged by static analysis tools (semgrep unsafe-strcpy, CWE-120) and is a banned function in several secure coding standards. Replace with std::ranges::copy() and a string_view literal, which is idiomatic C++ and avoids C string functions entirely. Also add zero-initialization (= {}) to the ifreq struct in linux-derivation-builder.cc, which was previously uninitialized. The test-support copy already had this. The test-support path is covered by the HttpsBinaryCacheStore unit test suite, which exercises setupNetworkTests() -> enterNetworkNamespace(). The production path in linux-derivation-builder.cc runs inside a Linux sandbox namespace during derivation builds, so it is not reachable from unit tests — it is covered by the functional test suite (tests/functional/linux-sandbox.sh). Co-Authored-By: Claude Opus 4.6 --- src/libstore-test-support/libstore-network.cc | 5 ++++- src/libstore/unix/build/linux-derivation-builder.cc | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libstore-test-support/libstore-network.cc b/src/libstore-test-support/libstore-network.cc index 8aa047bdd609..800dc17ed9f5 100644 --- a/src/libstore-test-support/libstore-network.cc +++ b/src/libstore-test-support/libstore-network.cc @@ -5,6 +5,8 @@ #ifdef __linux__ # include "nix/util/file-system.hh" # include "nix/util/linux-namespaces.hh" +# include +# include # include # include # include @@ -34,8 +36,9 @@ static void enterNetworkNamespace() if (!fd) throw SysError("cannot open IP socket for loopback interface"); + using namespace std::string_view_literals; struct ::ifreq ifr = {}; - strcpy(ifr.ifr_name, "lo"); + std::ranges::copy("lo"sv, ifr.ifr_name); ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; if (::ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) throw SysError("cannot set loopback interface flags"); diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 0ffcf0ccb657..9e3f707aee86 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -9,6 +9,8 @@ # include "nix/util/serialise.hh" # include "linux/fchmodat2-compat.hh" +# include +# include # include # include # include @@ -472,8 +474,9 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu if (!fd) throw SysError("cannot open IP socket"); - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); + using namespace std::string_view_literals; + struct ifreq ifr = {}; + std::ranges::copy("lo"sv, ifr.ifr_name); ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) throw SysError("cannot set loopback interface flags"); From bfeb7b874b1f93f3b1db9343732c574adf4abab7 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 9 Mar 2026 14:27:38 -0400 Subject: [PATCH 062/555] local-overlay-store: require upper-layer via isOverridden The setting framework lacks required-setting support, so upper-layer used an empty-string sentinel. This commit replaces it with an absolute dummy default and check `isOverridden()` at store construction time. --- src/libstore-tests/local-overlay-store.cc | 18 ++++++++++++++++++ .../include/nix/store/local-overlay-store.hh | 2 +- src/libstore/local-overlay-store.cc | 3 +++ src/libutil/include/nix/util/configuration.hh | 4 ++-- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/libstore-tests/local-overlay-store.cc b/src/libstore-tests/local-overlay-store.cc index 4c12f6256aba..e7b392b1b083 100644 --- a/src/libstore-tests/local-overlay-store.cc +++ b/src/libstore-tests/local-overlay-store.cc @@ -36,4 +36,22 @@ TEST(LocalOverlayStore, constructConfig_rootPath) EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } +TEST(LocalOverlayStore, upperLayer_notOverridden) +{ + LocalOverlayStoreConfig config{"", {}}; + EXPECT_FALSE(config.upperLayer.isOverridden()); +} + +TEST(LocalOverlayStore, upperLayer_overridden) +{ + LocalOverlayStoreConfig config{ + "", + { + {"upper-layer", "/some/upper"}, + }, + }; + EXPECT_TRUE(config.upperLayer.isOverridden()); + EXPECT_EQ(config.upperLayer.get(), std::filesystem::path{"/some/upper"}); +} + } // namespace nix diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index 5d8ec1b4571a..d98602ad8319 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -33,7 +33,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig const Setting upperLayer{ (StoreConfig *) this, - "", + "/upper-layer-must-be-set", "upper-layer", R"( Directory containing the OverlayFS upper layer for this store's store dir. diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 705e9ddf71db..6025affddf07 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -46,6 +46,9 @@ LocalOverlayStore::LocalOverlayStore(ref config) , config{config} , lowerStore(openStore(config->lowerStoreUri.get()).dynamic_pointer_cast()) { + if (!config->upperLayer.isOverridden()) + throw Error("overlay store at %s requires the 'upper-layer' setting", PathFmt(config->realStoreDir.get())); + if (config->checkMount.get()) { std::smatch match; std::string mountInfo; diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 446b68f3d33a..22ebbb3e5519 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -189,6 +189,8 @@ public: std::optional experimentalFeature; + bool isOverridden() const; + protected: AbstractSetting( @@ -214,8 +216,6 @@ protected: virtual std::map toJSONObject() const; virtual void convertToArg(Args & args, const std::string & category); - - bool isOverridden() const; }; /** From 0d5d424ff8195663cc23e819e1e6804f3528bf93 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 9 Mar 2026 01:45:33 -0400 Subject: [PATCH 063/555] libstore: use `std::optional` for path settings that can be unset `sshKey`, `secretKeyFile`, `remountHook`, and `upperLayer` used empty strings denoting a setting that was "not configured". This is unsound for a type called `AbsolutePath` since an empty path is not absolute. This commit changes them to `Setting>`, which makes the "unset" state explicit and will allow the `AbsolutePath` constructors to enforce the absolute invariant unconditionally. `upperLayer` is effectively required, so the `LocalOverlayStore` constructor now errors immediately if it is unset. --- src/libstore-tests/machines.cc | 4 ++-- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/include/nix/store/binary-cache-store.hh | 4 ++-- src/libstore/include/nix/store/common-ssh-store-config.hh | 4 ++-- src/libstore/include/nix/store/local-overlay-store.hh | 4 ++-- src/libstore/include/nix/store/machines.hh | 2 +- src/libstore/include/nix/store/ssh.hh | 4 ++-- src/libstore/local-overlay-store.cc | 6 +++--- src/libstore/machines.cc | 6 +++--- src/libstore/ssh.cc | 6 +++--- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index c4cbf47e4e22..4246a876ca87 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -27,7 +27,7 @@ TEST(machines, getMachinesUriOnly) ASSERT_THAT(actual, SizeIs(1)); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); - EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::filesystem::path{}))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::nullopt))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); @@ -49,7 +49,7 @@ TEST(machines, getMachinesDefaults) ASSERT_THAT(actual, SizeIs(1)); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); - EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::filesystem::path{}))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::nullopt))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 34a32096201f..73fc6981d5be 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -27,8 +27,8 @@ namespace nix { BinaryCacheStore::BinaryCacheStore(Config & config) : config{config} { - if (!config.secretKeyFile.get().empty()) - signers.push_back(std::make_unique(SecretKey{readFile(config.secretKeyFile.get())})); + if (auto & skf = config.secretKeyFile.get()) + signers.push_back(std::make_unique(SecretKey{readFile(*skf)})); if (config.secretKeyFiles != "") { std::stringstream ss(config.secretKeyFiles); diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index bc499130b087..4728dbd5e0c6 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -39,8 +39,8 @@ struct BinaryCacheStoreConfig : virtual StoreConfig fetch debug info on demand )"}; - Setting secretKeyFile{ - this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; + Setting> secretKeyFile{ + this, std::nullopt, "secret-key", "Path to the secret key used to sign the binary cache."}; Setting secretKeyFiles{ this, "", "secret-keys", "List of comma-separated paths to the secret keys used to sign the binary cache."}; diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index 42b3415b777a..b9c21fd8ef68 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -14,8 +14,8 @@ struct CommonSSHStoreConfig : virtual StoreConfig CommonSSHStoreConfig(const ParsedURL::Authority & authority, const Params & params); - Setting sshKey{ - this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; + Setting> sshKey{ + this, std::nullopt, "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; Setting sshPublicHostKey{ this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index d98602ad8319..dae7925d29f4 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -53,9 +53,9 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig default, but can be disabled if needed. )"}; - const Setting remountHook{ + const Setting> remountHook{ (StoreConfig *) this, - "", + std::nullopt, "remount-hook", R"( Script or other executable to run when overlay filesystem needs remounting. diff --git a/src/libstore/include/nix/store/machines.hh b/src/libstore/include/nix/store/machines.hh index 6b3484699491..a3e9353c80fa 100644 --- a/src/libstore/include/nix/store/machines.hh +++ b/src/libstore/include/nix/store/machines.hh @@ -17,7 +17,7 @@ struct Machine const StoreReference storeUri; const StringSet systemTypes; - const std::filesystem::path sshKey; + const std::optional sshKey; const unsigned int maxJobs; const float speedFactor; const StringSet supportedFeatures; diff --git a/src/libstore/include/nix/store/ssh.hh b/src/libstore/include/nix/store/ssh.hh index c64e9b1cca93..e090dba2317f 100644 --- a/src/libstore/include/nix/store/ssh.hh +++ b/src/libstore/include/nix/store/ssh.hh @@ -18,7 +18,7 @@ private: ParsedURL::Authority authority; std::string hostnameAndUser; bool fakeSSH; - const std::filesystem::path keyFile; + const std::optional keyFile; /** * Raw bytes, not Base64 encoding. */ @@ -50,7 +50,7 @@ public: SSHMaster( const ParsedURL::Authority & authority, - std::filesystem::path keyFile, + std::optional keyFile, std::string_view sshPublicHostKey, bool useMaster, bool compress, diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 6025affddf07..89c5a42d8ab7 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -283,10 +283,10 @@ void LocalOverlayStore::remountIfNecessary() if (!_remountRequired) return; - if (config->remountHook.get().empty()) { - warn("%s needs remounting, set remount-hook to do this automatically", PathFmt(config->realStoreDir.get())); + if (auto & hook = config->remountHook.get()) { + runProgram(*hook, false, {config->realStoreDir.get().native()}); } else { - runProgram(config->remountHook.get(), false, {config->realStoreDir.get().native()}); + warn("%s needs remounting, set remount-hook to do this automatically", PathFmt(config->realStoreDir.get())); } _remountRequired = false; diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 3b11ec902525..00a4d4d1fe29 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -69,8 +69,8 @@ StoreReference Machine::completeStoreReference() const } if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { - if (!sshKey.empty()) - storeUri.params["ssh-key"] = sshKey.string(); + if (sshKey) + storeUri.params["ssh-key"] = sshKey->string(); if (sshPublicHostKey != "") storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey; } @@ -179,7 +179,7 @@ static Machine parseBuilderLine(const StringSet & defaultSystems, const std::str // `systemTypes` isSet(1) ? tokenizeString(tokens[1], ",") : defaultSystems, // `sshKey` - isSet(2) ? tokens[2] : "", + isSet(2) && !tokens[2].empty() ? std::make_optional(tokens[2]) : std::nullopt, // `maxJobs` isSet(3) ? parseUnsignedIntField(3) : 1U, // `speedFactor` diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 849fcff2fdfc..0240f9cca322 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -66,7 +66,7 @@ OsStrings getNixSshOpts() SSHMaster::SSHMaster( const ParsedURL::Authority & authority, - std::filesystem::path keyFile, + std::optional keyFile, std::string_view sshPublicHostKey, bool useMaster, bool compress, @@ -95,8 +95,8 @@ void SSHMaster::addCommonSSHOpts(OsStrings & args) auto sshArgs = getNixSshOpts(); args.insert(args.end(), sshArgs.begin(), sshArgs.end()); - if (!keyFile.empty()) - args.insert(args.end(), {OS_STR("-i"), keyFile.native()}); + if (keyFile) + args.insert(args.end(), {OS_STR("-i"), keyFile->native()}); if (!sshPublicHostKey.empty()) { std::filesystem::path fileName = tmpDir->path() / "host-key"; writeFile(fileName, authority.host + " " + sshPublicHostKey + "\n"); From 53197b9cca9a0e6bf4de1172ec1eed56de56631b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 9 Mar 2026 21:56:45 +0300 Subject: [PATCH 064/555] libexpr: Get rid of std::filesystem::path in parser.y This has the exact same behaviour on unix, while fixing path resolution bugs on windows. I've checked nix up to 2.3 and all versions got the case of literal being `path/to/symlink/../otherfile` wrong, because they never resolved any symlinks at that point and always used absPath there. --- src/libexpr/parser.y | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index e5e4241ea7b0..77e63ed6299f 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -426,7 +426,7 @@ path_start /* Absolute paths are always interpreted relative to the root filesystem accessor, rather than the accessor of the current Nix expression. */ - auto path = canonPath(literal).string(); + auto path = CanonPath(literal).abs(); /* add back in the trailing '/' to the first segment */ if (literal.size() > 1 && literal.back() == '/') path += '/'; @@ -442,8 +442,7 @@ path_start return std::nullopt; }); - auto basePath = std::filesystem::path(state->basePath.path.abs()); - auto path = absPath(literal, &basePath).string(); + auto path = CanonPath(literal, state->basePath.path).abs(); /* add back in the trailing '/' to the first segment */ if (literal.size() > 1 && literal.back() == '/') path += '/'; From 73151045c2308505060155d14591e1a59a7cbcc8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 9 Mar 2026 22:05:28 +0300 Subject: [PATCH 065/555] printVersion: Fix quoting of the output (once again) --- src/libmain/shared.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index a65fc773aa44..3640efc07a00 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -330,12 +330,12 @@ void printVersion(const std::string & programName) std::cout << "System type: " << settings.thisSystem << "\n"; std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; - std::cout << "System configuration file: " << nixConfFile() << "\n"; + std::cout << "System configuration file: " << os_string_to_string(nixConfFile().native()) << "\n"; std::cout << "User configuration files: " << os_string_to_string(ExecutablePath{.directories = nixUserConfFiles()}.render()) << "\n"; std::cout << "Store directory: " << resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir << "\n"; - std::cout << "State directory: " << settings.nixStateDir << "\n"; + std::cout << "State directory: " << os_string_to_string(settings.nixStateDir.native()) << "\n"; } throw Exit(); } From a322fd0f80b4c2b74deaadb65fd25d464a39d1a5 Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Mon, 9 Mar 2026 12:14:38 -0700 Subject: [PATCH 066/555] libutil: Initialize Handler::arity with in-class default Handler() = default leaves size_t arity uninitialized. Currently unreachable since all callers use parameterized constructors, but this eliminates latent undefined behavior. Consistent with the existing pattern in the same file (shortName = 0, timesUsed = 0). Found by cppcheck (uninitMemberVar). Co-Authored-By: Claude Opus 4.6 --- src/libutil/include/nix/util/args.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/include/nix/util/args.hh b/src/libutil/include/nix/util/args.hh index b4337590a39f..2d4a81cafcd3 100644 --- a/src/libutil/include/nix/util/args.hh +++ b/src/libutil/include/nix/util/args.hh @@ -84,7 +84,7 @@ protected: struct Handler { std::function)> fun; - size_t arity; + size_t arity = 0; Handler() = default; From 1b356130787b44d5e6d443ff816f54ce2e702f03 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 9 Mar 2026 22:14:58 +0300 Subject: [PATCH 067/555] filetransfer: Fix Windows passing wide strings to curl_easy_setopt --- src/libstore/filetransfer.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 099f436ae65b..e6d4caa69fce 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -551,7 +551,7 @@ struct curlFileTransfer : public FileTransfer } if (auto & caFile = fileTransfer.settings.caFile.get()) - curl_easy_setopt(req, CURLOPT_CAINFO, caFile->c_str()); + curl_easy_setopt(req, CURLOPT_CAINFO, caFile->string().c_str()); #if !defined(_WIN32) curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback); @@ -569,7 +569,7 @@ struct curlFileTransfer : public FileTransfer /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ - curl_easy_setopt(req, CURLOPT_NETRC_FILE, fileTransfer.settings.netrcFile.get().c_str()); + curl_easy_setopt(req, CURLOPT_NETRC_FILE, fileTransfer.settings.netrcFile.get().string().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); if (writtenToSink) From 9f889e813fff4493f17d3afd3b3f53c0ef4991e5 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 9 Mar 2026 02:01:39 -0400 Subject: [PATCH 068/555] configuration: assert that `AbsolutePath` is absolute on construction The inherited constructors from `std::filesystem::path` allowed creating an `AbsolutePath` from any string without validation. Now that all settings using empty strings as "unset" have been converted to `std::optional`, the constructors can enforce the invariant that every `AbsolutePath` is actually absolute. --- src/libstore/builtins/fetchurl.cc | 7 ++++--- src/libstore/filetransfer.cc | 16 +++++++++++----- .../include/nix/store/filetransfer.hh | 2 +- src/libutil/include/nix/util/configuration.hh | 19 ++++++++++++++++++- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1ff66eb89fa4..767fc9b8f057 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -14,12 +14,13 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) this to be stored in a file. It would be nice if we could just pass a pointer to the data. */ if (ctx.netrcData != "") { - fileTransferSettings.netrcFile = "netrc"; + fileTransferSettings.netrcFile = ctx.tmpDirInSandbox / "netrc"; writeFile(fileTransferSettings.netrcFile.get(), ctx.netrcData, 0600); } - fileTransferSettings.caFile = "ca-certificates.crt"; - writeFile(*fileTransferSettings.caFile.get(), ctx.caFileData, 0600); + auto caFilePath = ctx.tmpDirInSandbox / "ca-certificates.crt"; + fileTransferSettings.caFile = std::optional{caFilePath}; + writeFile(caFilePath, ctx.caFileData, 0600); auto out = get(ctx.drv.outputs, "out"); if (!out) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 099f436ae65b..53f8a5019881 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -34,20 +34,26 @@ namespace nix { const unsigned int RETRY_TIME_MS_DEFAULT = 250; const unsigned int RETRY_TIME_MS_TOO_MANY_REQUESTS = 60000; -std::filesystem::path FileTransferSettings::getDefaultSSLCertFile() +std::optional FileTransferSettings::getDefaultSSLCertFile() { for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) if (pathAccessible(fn)) return fn; - return ""; + return std::nullopt; } FileTransferSettings::FileTransferSettings() { - auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); - if (sslOverride != "") - caFile = sslOverride; + std::optional sslOverride = + getEnvOs(OS_STR("NIX_SSL_CERT_FILE")) + .or_else([] { return getEnvOs(OS_STR("SSL_CERT_FILE")); }) + .and_then([](OsString s) -> std::optional { + return s.empty() ? std::nullopt : std::optional{std::move(s)}; + }) + .transform([](OsString s) { return AbsolutePath{std::filesystem::path{std::move(s)}}; }); + if (sslOverride) + caFile = *sslOverride; } FileTransferSettings fileTransferSettings; diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 340ad856ccc0..5df5ae59e6bb 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -24,7 +24,7 @@ const std::filesystem::path & nixConfDir(); struct FileTransferSettings : Config { private: - static std::filesystem::path getDefaultSSLCertFile(); + static std::optional getDefaultSSLCertFile(); public: FileTransferSettings(); diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 22ebbb3e5519..a8ba936d9a33 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -221,21 +221,38 @@ protected: /** * For `Setting`. `parse()` calls `canonPath`, * rejecting empty and relative paths. + * + * Constructors assert that the path is absolute. */ struct AbsolutePath : std::filesystem::path { - using path::path; using path::operator=; AbsolutePath(const std::filesystem::path & p) : path(p) { + assert(is_absolute()); } AbsolutePath(std::filesystem::path && p) : path(std::move(p)) { + assert(is_absolute()); + } + + AbsolutePath(const char * s) + : path(s) + { + assert(is_absolute()); + } + +#ifdef _WIN32 + AbsolutePath(const wchar_t * s) + : path(s) + { + assert(is_absolute()); } +#endif }; template<> From 34233db951e41159e36312f0b0b97868fcc1a484 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Thu, 19 Feb 2026 19:04:46 -0500 Subject: [PATCH 069/555] libstore: preserve mount flags when remounting store writable `makeStoreWritable` passed only `MS_REMOUNT | MS_BIND` to `mount()`, telling the kernel to drop every other flag. In a user namespace, flags like `nodev` and `nosuid` are locked by the kernel, so dropping them causes `EPERM`. This commit reads the active flags from `statvfs` and translates each `ST_*` constant to its `MS_*` counterpart individually, because the two sets are not always equal (`ST_RELATIME` is 4096, which collides with `MS_BIND`; the real `MS_RELATIME` is 1 << 21). Fixes #9705 --- src/libstore/local-store.cc | 24 +++++++++++- tests/nixos/default.nix | 2 + tests/nixos/store-remount.nix | 71 +++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 tests/nixos/store-remount.nix diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 08b9f5316910..2be86f2f6a99 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -621,7 +621,29 @@ void LocalStore::makeStoreWritable() throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (mount(0, config->realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + /* In a user namespace, mount flags like `nodev` and `nosuid` are + locked and dropping them causes `EPERM`, so here we translate each + `statvfs` flag to the corresponding `mount` flag individually. */ + unsigned long flags = MS_REMOUNT | MS_BIND; + if (stat.f_flag & ST_NODEV) + flags |= MS_NODEV; + if (stat.f_flag & ST_NOSUID) + flags |= MS_NOSUID; + if (stat.f_flag & ST_NOEXEC) + flags |= MS_NOEXEC; + if (stat.f_flag & ST_NOATIME) + flags |= MS_NOATIME; + if (stat.f_flag & ST_NODIRATIME) + flags |= MS_NODIRATIME; + if (stat.f_flag & ST_RELATIME) + flags |= MS_RELATIME; + if (stat.f_flag & ST_SYNCHRONOUS) + flags |= MS_SYNCHRONOUS; +# ifdef ST_NOSYMFOLLOW + if (stat.f_flag & ST_NOSYMFOLLOW) + flags |= MS_NOSYMFOLLOW; +# endif + if (mount(0, config->realStoreDir.get().c_str(), "none", flags, 0) == -1) throw SysError("remounting %s writable", PathFmt(config->realStoreDir.get())); } #endif diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index c30e872bc1d0..a2f6ba118416 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -205,4 +205,6 @@ in fetchersSubstitute = runNixOSTest ./fetchers-substitute.nix; chrootStore = runNixOSTest ./chroot-store.nix; + + storeRemount = runNixOSTest ./store-remount.nix; } diff --git a/tests/nixos/store-remount.nix b/tests/nixos/store-remount.nix new file mode 100644 index 000000000000..09f76dab797e --- /dev/null +++ b/tests/nixos/store-remount.nix @@ -0,0 +1,71 @@ +# Test that `makeStoreWritable` doesn't fail with EPERM when inherited +# mount flags like `nodev` are locked in a user namespace +# (https://github.com/NixOS/nix/issues/9705). + +{ lib, ... }: + +{ + name = "store-remount"; + + nodes.machine = + { pkgs, ... }: + { + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ + pkgs.nix + pkgs.util-linux + pkgs.strace + pkgs.gnugrep + pkgs.bash + ]; + nix.settings.substituters = lib.mkForce [ ]; + }; + + testScript = + { nodes }: + let + pkgs = nodes.machine.nixpkgs.pkgs; + nix = nodes.machine.nix.package; + in + '' + machine.wait_for_unit("multi-user.target") + + # Add nodev to the host store mount so it becomes locked in the + # user namespace below. + machine.succeed("mount -o remount,bind,rw,nodev /nix/store") + + # Set up a minimal container root (which needs its own db to avoid + # UID-mapping issues on the host db). + machine.succeed("mkdir -p /tmp/container/{nix/store,nix/var/nix/db,tmp,etc,proc,sys}") + machine.succeed("echo 'root:x:0:0:root:/root:/bin/sh' > /tmp/container/etc/passwd") + machine.succeed("echo 'root:x:0:' > /tmp/container/etc/group") + + # In a user namespace (`--private-users`), inherited flags like nodev + # are locked. We make the store read-only, then let `nix store info` + # trigger `makeStoreWritable`. Without the fix, the remount fails + # with EPERM because the old code implicitly dropped locked flags. + # + # We use strace to capture the mount() syscall and verify that + # MS_NODEV is present in the flags, since nix does + # unshare(CLONE_NEWNS) internally so the remount is not observable + # from outside the nix process. + (status, output) = machine.execute( + "systemd-nspawn --quiet --register=no --private-network " + "--private-users=pick " + "-D /tmp/container " + "--bind=/nix/store " + "--as-pid2 " + "${pkgs.bash}/bin/bash -c '" + " ${pkgs.util-linux}/bin/mount -o remount,bind,ro /nix/store && " + " ${pkgs.strace}/bin/strace -f -e trace=mount -o /strace.log " + " ${nix}/bin/nix store info --store local " + " --extra-experimental-features nix-command" + "'" + ) + + print(machine.succeed("cat /tmp/container/strace.log")) + + assert status == 0, f"nspawn failed (status {status}): {output}" + machine.succeed("${pkgs.gnugrep}/bin/grep -q MS_NODEV /tmp/container/strace.log") + ''; +} From 6911469734cc6d798e9407fba21bce87d15538e8 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Thu, 19 Feb 2026 19:04:46 -0500 Subject: [PATCH 070/555] cli: warn when private mount namespace setup fails The catch block silently swallowed errors from `saveMountNamespace()` and `unshare(CLONE_NEWNS)`. This commit logs a warning so the failure is visible. --- src/nix/main.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 2a790e7be934..297318ddcd7f 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -404,6 +404,7 @@ void mainWrapped(int argc, char ** argv) if (unshare(CLONE_NEWNS) == -1) throw SysError("setting up a private mount namespace"); } catch (Error & e) { + warn("failed to set up a private mount namespace: %s", e.msg()); } } #endif From f053019e48ac16c2e8de60bddc3d70db42e20106 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 9 Mar 2026 17:26:13 -0400 Subject: [PATCH 071/555] tests: rename signature fixture files for clarity `with-signature-structured` and `with-structured-signature` were redundant fixtures that only differed in `outPath` (with or without `.drv`). The unused `with-structured-signature` is removed, and `with-signature-structured` becomes `with-signature` since it is the canonical structured format. The old `with-signature` (unstructured string format) is renamed to `with-signature-unstructured`. Also fixes stale references to `with-structured-signature` in `meson.build` and the build-trace-entry documentation. --- .../source/protocols/json/build-trace-entry.md | 2 +- src/json-schema-checks/meson.build | 2 +- ...ured.json => with-signature-unstructured.json} | 5 +---- .../data/realisation/with-signature.json | 5 ++++- .../realisation/with-structured-signature.json | 15 --------------- src/libstore-tests/realisation.cc | 4 ++-- 6 files changed, 9 insertions(+), 24 deletions(-) rename src/libstore-tests/data/realisation/{with-signature-structured.json => with-signature-unstructured.json} (57%) delete mode 100644 src/libstore-tests/data/realisation/with-structured-signature.json diff --git a/doc/manual/source/protocols/json/build-trace-entry.md b/doc/manual/source/protocols/json/build-trace-entry.md index 2f8a4dbeeff6..d4a3a96edb64 100644 --- a/doc/manual/source/protocols/json/build-trace-entry.md +++ b/doc/manual/source/protocols/json/build-trace-entry.md @@ -11,7 +11,7 @@ ### Build trace entry with signature ```json -{{#include schema/build-trace-entry-v3/with-structured-signature.json}} +{{#include schema/build-trace-entry-v3/with-signature.json}} ``` + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory + +- [binary cache]{#gloss-binary-cache} + + A *binary cache* is a Nix store which uses a different format: its + metadata and signatures are kept in `.narinfo` files rather than in a + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). + - [build system]{#gloss-build-system} Generic term for software that facilitates the building of software by automating the invocation of compilers, linkers, and other tools. @@ -8,6 +43,26 @@ It has no knowledge of any particular programming language or toolchain. These details are specified in [derivation expressions](#gloss-derivation-expression). +- [closure]{#gloss-closure} + + The closure of a store path is the set of store paths that are + directly or indirectly “reachable” from that store path; that is, + it’s the closure of the path under the *references* relation. For + a package, the closure of its derivation is equivalent to the + build-time dependencies, while the closure of its [output path] is + equivalent to its runtime dependencies. For correct deployment it + is necessary to deploy whole closures, since otherwise at runtime + files could be missing. The command `nix-store --query --requisites ` prints out + closures of store paths. + + As an example, if the [store object] at path `P` contains a [reference] + to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` + references `R` then `R` is also in the closure of `P`. + + See [References](@docroot@/store/store-object.md#references) for details. + + [closure]: #gloss-closure + - [content address]{#gloss-content-address} A @@ -31,6 +86,20 @@ The industry term for storage and retrieval systems using [content addressing](#gloss-content-address). A Nix store also has [input addressing](#gloss-input-addressed-store-object), and metadata. +- [content-addressed store object]{#gloss-content-addressed-store-object} + + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. + This includes derivations, the outputs of [content-addressing derivations](#gloss-content-addressing-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + +- [content-addressing derivation]{#gloss-content-addressing-derivation} + + A derivation which has the + [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) + attribute set to `true`. + - [derivation]{#gloss-derivation} A derivation can be thought of as a [pure function](https://en.wikipedia.org/wiki/Pure_function) that produces new [store objects][store object] from existing store objects. @@ -43,20 +112,11 @@ [derivation]: #gloss-derivation -- [store derivation]{#gloss-store-derivation} - - A [derivation] represented as a [store object]. - - See [Store Derivation](@docroot@/store/derivation/index.md#store-derivation) for details. - - [store derivation]: #gloss-store-derivation - -- [directed acyclic graph]{#gloss-directed-acyclic-graph} +- [derivation expression]{#gloss-derivation-expression} - A [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAG) is graph whose edges are given a direction ("a to b" is not the same edge as "b to a"), and for which no possible path (created by joining together edges) forms a cycle. + A description of a [store derivation] using the [`derivation` primitive](./language/derivations.md) in the [Nix language]. - DAGs are very important to Nix. - In particular, the non-self-[references][reference] of [store object][store object] form a cycle. + [derivation expression]: #gloss-derivation-expression - [derivation path]{#gloss-derivation-path} @@ -68,79 +128,34 @@ [derivation path]: #gloss-derivation-path -- [derivation expression]{#gloss-derivation-expression} - - A description of a [store derivation] using the [`derivation` primitive](./language/derivations.md) in the [Nix language]. - - [derivation expression]: #gloss-derivation-expression - -- [instantiate]{#gloss-instantiate}, instantiation - - Translate a [derivation expression] into a [store derivation]. - - See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation. - - [instantiate]: #gloss-instantiate - -- [realise]{#gloss-realise}, realisation - - Ensure a [store path] is [valid][validity]. - - This can be achieved by: - - Fetching a pre-built [store object] from a [substituter] - - [Building](@docroot@/store/building.md) the corresponding [store derivation] - - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs - - See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - - See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). - - [realise]: #gloss-realise - -- [content-addressing derivation]{#gloss-content-addressing-derivation} - - A derivation which has the - [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) - attribute set to `true`. - -- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - - A [store derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. +- [deriver]{#gloss-deriver} -- [store]{#gloss-store} + The [store derivation] that produced an [output path]. - A collection of [store objects][store object], with operations to manipulate that collection. - See [Nix Store](./store/index.md) for details. + The deriver for an output path can be queried with the `--deriver` option to + [`nix-store --query`](@docroot@/command-ref/nix-store/query.md). - There are many types of stores, see [Store Types](./store/types/index.md) for details. +- [deriving path]{#gloss-deriving-path} - [store]: #gloss-store + Deriving paths are a way to refer to [store objects][store object] that might not yet be [realised][realise]. -- [Nix instance]{#gloss-nix-instance} - - 1. An installation of Nix, which includes the presence of a [store], and the Nix package manager which operates on that store. - A local Nix installation and a [remote builder](@docroot@/advanced-topics/distributed-builds.md) are two examples of Nix instances. - 2. A running Nix process, such as the `nix` command. + See [Deriving Path](./store/derivation/index.md#deriving-path) for details. -- [binary cache]{#gloss-binary-cache} + Not to be confused with [derivation path]. - A *binary cache* is a Nix store which uses a different format: its - metadata and signatures are kept in `.narinfo` files rather than in a - [Nix database]. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). +- [directed acyclic graph]{#gloss-directed-acyclic-graph} -- [store path]{#gloss-store-path} + A [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAG) is graph whose edges are given a direction ("a to b" is not the same edge as "b to a"), and for which no possible path (created by joining together edges) forms a cycle. - The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. + DAGs are very important to Nix. + In particular, the non-self-[references][reference] of [store object][store object] form a cycle. - > **Example** - > - > `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` +- [experimental feature]{#gloss-experimental-feature} - See [Store Path](@docroot@/store/store-path.md) for details. + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - [store path]: #gloss-store-path + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). - [file system object]{#gloss-file-system-object} @@ -150,21 +165,19 @@ [file system object]: #gloss-file-system-object -- [store object]{#gloss-store-object} - - Part of the contents of a [store]. - - A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. - It can be referred to by a [store path]. - - See [Store Object](@docroot@/store/store-object.md) for details. +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - [store object]: #gloss-store-object + A [store derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [IFD]{#gloss-ifd} [Import From Derivation](./language/import-from-derivation.md) +- [impure derivation]{#gloss-impure-derivation} + + [An experimental feature](@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. + - [input-addressed store object]{#gloss-input-addressed-store-object} A store object produced by building a @@ -174,42 +187,28 @@ See [input-addressing derivation outputs](store/derivation/outputs/input-address.md) for details. -- [content-addressed store object]{#gloss-content-addressed-store-object} - - A [store object] which is [content-addressed](#gloss-content-address), - i.e. whose [store path] is determined by its contents. - This includes derivations, the outputs of [content-addressing derivations](#gloss-content-addressing-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). - - See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. - -- [substitute]{#gloss-substitute} +- [installable]{#gloss-installable} - A substitute is a command invocation stored in the [Nix database] that - describes how to build a store object, bypassing the normal build - mechanism (i.e., derivations). Typically, the substitute builds the - store object by downloading a pre-built version of the store object - from some server. + Something that can be realised in the Nix store. -- [substituter]{#gloss-substituter} + See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. - An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. - Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. +- [instantiate]{#gloss-instantiate}, instantiation - See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. + Translate a [derivation expression] into a [store derivation]. - [substituter]: #gloss-substituter + See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation. -- [purity]{#gloss-purity} + [instantiate]: #gloss-instantiate - The assumption that equal Nix derivations when run always produce - the same output. This cannot be guaranteed in general (e.g., a - builder can rely on external inputs such as the network or the - system time) but the Nix model assumes it. +- [Nix Archive (NAR)]{#gloss-nar} -- [impure derivation]{#gloss-impure-derivation} + A *N*ix *AR*chive. This is a serialisation of a path in the Nix + store. It can contain regular files, directories and symbolic + links. NARs are generated and unpacked using `nix-store --dump` + and `nix-store --restore`. - [An experimental feature](@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, - so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. - [Nix database]{#gloss-nix-database} @@ -235,145 +234,136 @@ > > Building and deploying software using Nix entails writing Nix expressions to describe [packages][package] and compositions thereof. -- [reference]{#gloss-reference} - - An edge from one [store object] to another. +- [Nix instance]{#gloss-nix-instance} + + 1. An installation of Nix, which includes the presence of a [store], and the Nix package manager which operates on that store. + A local Nix installation and a [remote builder](@docroot@/advanced-topics/distributed-builds.md) are two examples of Nix instances. + 2. A running Nix process, such as the `nix` command. - See [References](@docroot@/store/store-object.md#references) for details. +- [output]{#gloss-output} - [reference]: #gloss-reference + A [store object] produced by a [store derivation]. + See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. - See [References](@docroot@/store/store-object.md#references) for details. + [output]: #gloss-output -- [reachable]{#gloss-reachable} +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. - A store path `Q` is reachable from another store path `P` if `Q` - is in the *closure* of the *references* relation. +- [output path]{#gloss-output-path} - See [References](@docroot@/store/store-object.md#references) for details. + The [store path] to the [output] of a [store derivation]. -- [closure]{#gloss-closure} + [output path]: #gloss-output-path - The closure of a store path is the set of store paths that are - directly or indirectly “reachable” from that store path; that is, - it’s the closure of the path under the *references* relation. For - a package, the closure of its derivation is equivalent to the - build-time dependencies, while the closure of its [output path] is - equivalent to its runtime dependencies. For correct deployment it - is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command `nix-store --query --requisites ` prints out - closures of store paths. +- [package]{#package} - As an example, if the [store object] at path `P` contains a [reference] - to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` - references `R` then `R` is also in the closure of `P`. + A software package; files that belong together for a particular purpose, and metadata. - See [References](@docroot@/store/store-object.md#references) for details. + Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. - [closure]: #gloss-closure + The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#type-attrs) containing: + - attributes that refer to the files of a package, typically in the form of [derivation outputs](#gloss-output), + - attributes with metadata, such as information about how the package is supposed to be used. -- [requisite]{#gloss-requisite} + The exact shape of these attribute sets is up to convention. - A store object [reachable] by a path (chain of references) from a given [store object]. - The [closure] is the set of requisites. + [package]: #package - See [References](@docroot@/store/store-object.md#references) for details. +- [profile]{#gloss-profile} -- [referrer]{#gloss-referrer} + A symlink to the current *user environment* of a user, e.g., + `/nix/var/nix/profiles/default`. - A reversed edge from one [store object] to another. +- [purity]{#gloss-purity} -- [output]{#gloss-output} + The assumption that equal Nix derivations when run always produce + the same output. This cannot be guaranteed in general (e.g., a + builder can rely on external inputs such as the network or the + system time) but the Nix model assumes it. - A [store object] produced by a [store derivation]. - See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. +- [reachable]{#gloss-reachable} - [output]: #gloss-output + A store path `Q` is reachable from another store path `P` if `Q` + is in the *closure* of the *references* relation. -- [output path]{#gloss-output-path} + See [References](@docroot@/store/store-object.md#references) for details. - The [store path] to the [output] of a [store derivation]. +- [realise]{#gloss-realise}, realisation - [output path]: #gloss-output-path + Ensure a [store path] is [valid][validity]. -- [output closure]{#gloss-output-closure}\ - The [closure] of an [output path]. It only contains what is [reachable] from the output. + This can be achieved by: + - Fetching a pre-built [store object] from a [substituter] + - [Building](@docroot@/store/building.md) the corresponding [store derivation] + - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs -- [deriving path]{#gloss-deriving-path} + See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - Deriving paths are a way to refer to [store objects][store object] that might not yet be [realised][realise]. + See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). - See [Deriving Path](./store/derivation/index.md#deriving-path) for details. + [realise]: #gloss-realise - Not to be confused with [derivation path]. +- [reference]{#gloss-reference} -- [deriver]{#gloss-deriver} + An edge from one [store object] to another. - The [store derivation] that produced an [output path]. + See [References](@docroot@/store/store-object.md#references) for details. - The deriver for an output path can be queried with the `--deriver` option to - [`nix-store --query`](@docroot@/command-ref/nix-store/query.md). + [reference]: #gloss-reference -- [validity]{#gloss-validity} + See [References](@docroot@/store/store-object.md#references) for details. - A store path is valid if all [store object]s in its [closure] can be read from the [store]. +- [referrer]{#gloss-referrer} - For a [local store], this means: - - The store path leads to an existing [store object] in that [store]. - - The store path is listed in the [Nix database] as being valid. - - All paths in the store path's [closure] are valid. + A reversed edge from one [store object] to another. - [validity]: #gloss-validity - [local store]: @docroot@/store/types/local-store.md +- [requisite]{#gloss-requisite} -- [user environment]{#gloss-user-env} + A store object [reachable] by a path (chain of references) from a given [store object]. + The [closure] is the set of requisites. - An automatically generated store object that consists of a set of - symlinks to “active” applications, i.e., other store paths. These - are generated automatically by - [`nix-env`](./command-ref/nix-env.md). See *profiles*. + See [References](@docroot@/store/store-object.md#references) for details. -- [profile]{#gloss-profile} +- [store]{#gloss-store} - A symlink to the current *user environment* of a user, e.g., - `/nix/var/nix/profiles/default`. + A collection of [store objects][store object], with operations to manipulate that collection. + See [Nix Store](./store/index.md) for details. -- [installable]{#gloss-installable} + There are many types of stores, see [Store Types](./store/types/index.md) for details. - Something that can be realised in the Nix store. + [store]: #gloss-store - See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. +- [store derivation]{#gloss-store-derivation} -- [Nix Archive (NAR)]{#gloss-nar} + A [derivation] represented as a [store object]. - A *N*ix *AR*chive. This is a serialisation of a path in the Nix - store. It can contain regular files, directories and symbolic - links. NARs are generated and unpacked using `nix-store --dump` - and `nix-store --restore`. + See [Store Derivation](@docroot@/store/derivation/index.md#store-derivation) for details. - See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + [store derivation]: #gloss-store-derivation -- [`∅`]{#gloss-empty-set} +- [store object]{#gloss-store-object} - The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + Part of the contents of a [store]. -- [`ε`]{#gloss-epsilon} + A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. + It can be referred to by a [store path]. - The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. + See [Store Object](@docroot@/store/store-object.md) for details. -- [package]{#package} + [store object]: #gloss-store-object - A software package; files that belong together for a particular purpose, and metadata. +- [store path]{#gloss-store-path} - Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. + The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. - The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#type-attrs) containing: - - attributes that refer to the files of a package, typically in the form of [derivation outputs](#gloss-output), - - attributes with metadata, such as information about how the package is supposed to be used. + > **Example** + > + > `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` - The exact shape of these attribute sets is up to convention. + See [Store Path](@docroot@/store/store-path.md) for details. - [package]: #package + [store path]: #gloss-store-path - [string interpolation]{#gloss-string-interpolation} @@ -385,31 +375,40 @@ [path]: ./language/types.md#type-path [attribute name]: ./language/types.md#type-attrs -- [base directory]{#gloss-base-directory} +- [substitute]{#gloss-substitute} - The location from which relative paths are resolved. + A substitute is a command invocation stored in the [Nix database] that + describes how to build a store object, bypassing the normal build + mechanism (i.e., derivations). Typically, the substitute builds the + store object by downloading a pre-built version of the store object + from some server. - - For expressions in a file, the base directory is the directory containing that file. - This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). - +- [substituter]{#gloss-substituter} - - - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. - [base directory]: #gloss-base-directory + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. -- [experimental feature]{#gloss-experimental-feature} + [substituter]: #gloss-substituter - Not yet stabilized functionality guarded by named experimental feature flags. - These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. +- [user environment]{#gloss-user-env} - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). + An automatically generated store object that consists of a set of + symlinks to “active” applications, i.e., other store paths. These + are generated automatically by + [`nix-env`](./command-ref/nix-env.md). See *profiles*. + +- [validity]{#gloss-validity} + A store path is valid if all [store object]s in its [closure] can be read from the [store]. + + For a [local store], this means: + - The store path leads to an existing [store object] in that [store]. + - The store path is listed in the [Nix database] as being valid. + - All paths in the store path's [closure] are valid. + + [validity]: #gloss-validity + [local store]: @docroot@/store/types/local-store.md [Nix language]: ./language/index.md From 04ef6379cafd8f900f08c2f12571520b0df342bf Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 28 Mar 2026 16:25:35 +0300 Subject: [PATCH 179/555] libstore: Use new Worker::Waker mechanism for the substitution thread Previously we would create a new pipe for each substitution goal, but now we can reuse the new Waker coroutine resumption mechanism that writes to a single wakeup pipe and a thread-safe queue of goals to resume. Also documents and fixes some lifetimes issues in the thread captures. I haven't seen this fail, but looking at the code, I'm pretty sure that the thread can still be running when the Worker dies, depending on when the SIGUSR1 signal arrives and when the thread unwinds its stack. So use the same weak_ptr approach to optionally share the ownership of all the members (like the store, substituter and the Waker) that are needed by the substitution thread. --- src/libstore/build/goal.cc | 7 ++ src/libstore/build/substitution-goal.cc | 65 +++++++++---------- src/libstore/build/worker.cc | 11 ---- src/libstore/include/nix/store/build/goal.hh | 12 ++++ .../nix/store/build/substitution-goal.hh | 6 -- .../include/nix/store/build/worker.hh | 17 +---- 6 files changed, 53 insertions(+), 65 deletions(-) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index ef3501b00916..ce6b3e58833d 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -308,6 +308,13 @@ Goal::Co Goal::waitForAWhile() co_return Return{}; } +Goal::Co Goal::waitUntilWoken() +{ + worker.waitForCompletion(shared_from_this()); + co_await Suspend{}; + co_return Return{}; +} + Goal::Co Goal::waitForBuildSlot() { worker.waitForBuildSlot(shared_from_this()); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 91ca7e1f21ea..2c1573b82ca9 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -207,53 +207,54 @@ Goal::Co PathSubstitutionGoal::tryToRun( auto maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); - outPipe = worker.makeMuxablePipe(); - auto promise = std::promise(); - - thr = std::thread([this, &promise, &subPath, &sub]() { + auto future = promise.get_future(); + + /* Be careful with ownership. cleanup() doesn't signal the worker thread + to cleanly shutdown, so the worker can die while the thread is still + running. That's why we use weak_ptr for everything that is owned by the + Worker. */ + thr = std::thread([weakGoal = weak_from_this(), + promise = std::move(promise), + subPath, + storePath = storePath, + repair = repair, + sub, + maybeWaker = worker.getCrossThreadWaker(), + maybeWorkerStore = worker.store.weak_from_this()]() mutable { try { ReceiveInterrupts receiveInterrupts; - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide.close(); }); + /* The Worker might have died while we were starting up. */ + auto workerStore = maybeWorkerStore.lock(); + if (!workerStore) + return; Activity act( *logger, actSubstitute, - Logger::Fields{worker.store.printStorePath(storePath), sub->config.getHumanReadableURI()}); + Logger::Fields{workerStore->printStorePath(storePath), sub->config.getHumanReadableURI()}); PushActivity pact(act.id); - copyStorePath(*sub, worker.store, subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs); + copyStorePath(*sub, *workerStore, subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { promise.set_exception(std::current_exception()); } + + /* The Worker might have already died (and the waker with it) by the + time we finished. N.B. if enqueueing to the waker throws, we better + std::terminate, since something has gone very wrong. This intentionally + lets the thread crash on exceptions for that reason. */ + if (auto waker = maybeWaker.lock()) + waker->enqueue(weakGoal); }); - worker.childStarted( - shared_from_this(), - { -#ifndef _WIN32 - outPipe.readSide.get() -#else - &outPipe -#endif - }, - true, - false); - - while (true) { - auto event = co_await WaitForChildEvent{}; - if (std::get_if(&event)) { - // Substitution doesn't process child output - } else if (std::get_if(&event)) { - break; - } else if (std::get_if(&event)) { - unreachable(); // Substitution doesn't use timeouts - } - } + /* Use up the substitution slot. */ + worker.childStarted(shared_from_this(), /*channels=*/{}, /*inBuildSlot=*/true, /*respectTimeouts=*/false); + /* Suspend until the thread finishes. */ + co_await waitUntilWoken(); trace("substitute finished"); @@ -261,7 +262,7 @@ Goal::Co PathSubstitutionGoal::tryToRun( worker.childTerminated(this); try { - promise.get_future().get(); + future.get(); } catch (std::exception & e) { /* Cause the parent build to fail unless --fallback is given, or the substitute has disappeared. The latter case behaves @@ -313,8 +314,6 @@ void PathSubstitutionGoal::cleanup() thr.join(); worker.childTerminated(this, JobCategory::Substitution); } - - outPipe.close(); } catch (...) { ignoreExceptionInDestructor(); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 9d2664305a3f..20131aaf3f7e 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -607,17 +607,6 @@ void Worker::markContentsGood(const StorePath & path) pathContentsGoodCache.insert_or_assign(path, true); } -MuxablePipe Worker::makeMuxablePipe() -{ - MuxablePipe pipe; -#ifndef _WIN32 - pipe.create(); -#else - pipe.createAsyncPipe(ioport.get()); -#endif - return pipe; -} - GoalPtr upcast_goal(std::shared_ptr subGoal) { return subGoal; diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 2efcd27a5be6..3e32f4fecb61 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -609,7 +609,19 @@ public: protected: Co await(Goals waitees); + /** + * Awaiting on the resulting coroutine yields the goal for several seconds. + * Used for retrying goals blocked on acquiring lockfiles. + */ Co waitForAWhile(); + + /** + * Awaiting on the resulting coroutine yields the goal until it is + * explicitly woken up via Worker::wakeUp. Wakeup can be queued from another + * thread via Worker::Waker. + */ + Co waitUntilWoken(); + Co waitForBuildSlot(); Co yield(); }; diff --git a/src/libstore/include/nix/store/build/substitution-goal.hh b/src/libstore/include/nix/store/build/substitution-goal.hh index 7ce28d1deff4..27585e348d98 100644 --- a/src/libstore/include/nix/store/build/substitution-goal.hh +++ b/src/libstore/include/nix/store/build/substitution-goal.hh @@ -4,7 +4,6 @@ #include "nix/store/build/worker.hh" #include "nix/store/store-api.hh" #include "nix/store/build/goal.hh" -#include "nix/util/muxable-pipe.hh" #include #include #include @@ -23,11 +22,6 @@ struct PathSubstitutionGoal : public Goal */ RepairFlag repair; - /** - * Pipe for the substituter's standard output. - */ - MuxablePipe outPipe; - /** * The substituter thread. */ diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 6f58d4021b11..2f5272cef5d7 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -150,8 +150,8 @@ private: { #ifndef _WIN32 /** - * Wakeup pipe polled alongside all other goal FDs. Gets written to by wakeUpCrossThread. - * Not needed on Windows. + * Wakeup pipe polled alongside all other goal FDs. Gets written to by + * enqueue(). Not needed on Windows. */ unix::SelfPipe wakeupPipe; #else @@ -173,13 +173,6 @@ private: #endif } -#ifdef _WIN32 - Waker(Descriptor ioport) - : ioport(ioport) - { - } -#endif - public: void enqueue(WeakGoalPtr goal); }; @@ -403,12 +396,6 @@ public: act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } - - /** - * Create a MuxablePipe that the worker can poll. Primary exists to - * deduplicate WIN32 ifdefs. - */ - MuxablePipe makeMuxablePipe(); }; } // namespace nix From 05ff8f2cf9487d5c5cbf038d302d299ec83edc80 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sun, 29 Mar 2026 01:07:24 +0000 Subject: [PATCH 180/555] fix(libstore/aws-creds): use S3 URL region as STS WebIdentity fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, GitHub Actions OIDC), the aws-c-auth provider needs a region to select an STS endpoint. It looks in AWS_REGION, AWS_DEFAULT_REGION, then the profile config. In IRSA setups that export AWS_WEB_IDENTITY_TOKEN_FILE and AWS_ROLE_ARN but no region — and additionally set AWS_CONFIG_FILE=/dev/null to suppress SSO/profile interference — all three sources are absent. The provider returns nullptr, the credential chain silently skips it, and the user sees a misleading 'IMDS provider' failure. Thread the ?region= parameter from the parsed S3 URL through to the STS WebIdentity provider as a last-resort fallback. aws-c-auth gives options.region precedence over env vars, so we only set it when neither AWS_REGION nor AWS_DEFAULT_REGION is present, preserving existing behaviour for users who do set a region. The credential-provider cache key changes from profile to (profile, region) since the region is baked into the provider at construction time. --- .../aws-sts-webidentity-region-fallback.md | 11 ++++ src/libstore/aws-creds.cc | 56 +++++++++++++++---- 2 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 doc/manual/rl-next/aws-sts-webidentity-region-fallback.md diff --git a/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md b/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md new file mode 100644 index 000000000000..5034c65438cb --- /dev/null +++ b/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md @@ -0,0 +1,11 @@ +--- +synopsis: S3 substituters fall back to the URL's region for STS WebIdentity auth +prs: [15594] +--- + +When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, +GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL +as a fallback for the STS endpoint region if neither `AWS_REGION` nor +`AWS_DEFAULT_REGION` is set. Previously, IRSA setups that exported +`AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail +with a misleading "IMDS provider" error. diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index b471962897bb..f755717eb531 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -4,6 +4,7 @@ # include # include "nix/store/s3-url.hh" +# include "nix/util/environment-variables.hh" # include "nix/util/logging.hh" # include @@ -170,15 +171,33 @@ static std::shared_ptr createSSOProvider( return createWrappedProvider(aws_credentials_provider_new_sso(allocator, &options), allocator); } +/** + * Check whether the AWS SDK can resolve a region from the standard + * environment variables. Mirrors aws_credentials_provider_resolve_region_from_env. + */ +static bool awsRegionSetInEnv() +{ + return getEnvNonEmpty("AWS_REGION") || getEnvNonEmpty("AWS_DEFAULT_REGION"); +} + /** * Create an STS WebIdentity credentials provider using the C library directly. * This reads AWS_WEB_IDENTITY_TOKEN_FILE, AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, * and AWS_REGION from the environment (falling back to the profile config). * Used by EKS IRSA, GitHub Actions OIDC, and other sts:AssumeRoleWithWebIdentity flows. * Returns nullptr if the required parameters can't be resolved. + * + * @param fallbackRegion Region to use when neither AWS_REGION nor + * AWS_DEFAULT_REGION is set — typically the ?region= from the S3 URL. + * Note: this also overrides any region from the profile config, since + * aws-c-auth gives options.region precedence over all implicit sources. + * Without this fallback the provider fails entirely in IRSA setups where + * the pod environment sets the token and role ARN but not the region + * (observed with AWS_CONFIG_FILE=/dev/null). */ static std::shared_ptr createSTSWebIdentityProvider( const std::string & profileName, + const std::string & fallbackRegion, Aws::Crt::Io::ClientBootstrap * bootstrap, Aws::Crt::Io::TlsContext * tlsContext, Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator()) @@ -192,6 +211,12 @@ static std::shared_ptr createSTSWebIdentit options.profile_name_override = aws_byte_cursor_from_c_str(profileName.c_str()); } + // aws-c-auth gives options.region precedence over env vars, so only set it + // when env vars are absent — otherwise we'd mask a user-supplied AWS_REGION. + if (!fallbackRegion.empty() && !awsRegionSetInEnv()) { + options.region = aws_byte_cursor_from_c_str(fallbackRegion.c_str()); + } + return createWrappedProvider(aws_credentials_provider_new_sts_web_identity(allocator, &options), allocator); } @@ -288,32 +313,38 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider } } - AwsCredentials getCredentialsRaw(const std::string & profile); + AwsCredentials getCredentialsRaw(const std::string & profile, const std::string & region); AwsCredentials getCredentials(const ParsedS3URL & url) override { auto profile = url.profile.value_or(""); + auto region = url.region.value_or(""); try { - return getCredentialsRaw(profile); + return getCredentialsRaw(profile, region); } catch (AwsAuthError & e) { warn("AWS authentication failed for S3 request %s: %s", url.toHttpsUrl(), e.message()); - credentialProviderCache.erase(profile); + credentialProviderCache.erase({profile, region}); throw; } } - std::shared_ptr createProviderForProfile(const std::string & profile); + std::shared_ptr + createProviderForProfile(const std::string & profile, const std::string & region); private: Aws::Crt::ApiHandle apiHandle; std::shared_ptr tlsContext; Aws::Crt::Io::ClientBootstrap * bootstrap; - boost::concurrent_flat_map> - credentialProviderCache; + // Keyed by (profile, region). Region is part of the key because it is baked + // into the STS WebIdentity provider at construction time; two S3 URLs with + // different ?region= parameters need distinct chains. + boost:: + concurrent_flat_map, std::shared_ptr> + credentialProviderCache; }; std::shared_ptr -AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) +AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile, const std::string & region) { // profileDisplayName is only used for debug logging - SDK uses its default profile // when ProfileNameOverride is not set @@ -369,7 +400,7 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) bool ecsAdded = false; if (tlsContext) { addProviderToChain("STS WebIdentity", [&]() { - return createSTSWebIdentityProvider(profile, bootstrap, tlsContext.get(), allocator); + return createSTSWebIdentityProvider(profile, region, bootstrap, tlsContext.get(), allocator); }); ecsAdded = addProviderToChain("ECS", [&]() { return createECSProvider(bootstrap, tlsContext.get(), allocator); }); @@ -395,18 +426,19 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChain(chainConfig, allocator); } -AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile) +AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile, const std::string & region) { std::shared_ptr provider; + auto key = std::make_pair(profile, region); credentialProviderCache.try_emplace_and_cvisit( - profile, + key, nullptr, - [&](auto & kv) { provider = kv.second = createProviderForProfile(profile); }, + [&](auto & kv) { provider = kv.second = createProviderForProfile(profile, region); }, [&](const auto & kv) { provider = kv.second; }); if (!provider) { - credentialProviderCache.erase_if(profile, [](const auto & kv) { + credentialProviderCache.erase_if(key, [](const auto & kv) { [[maybe_unused]] auto [_, provider] = kv; return !provider; }); From aa96fd767aa83b14f89fce4f9251d02245fb2b93 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 29 Mar 2026 22:14:43 +0300 Subject: [PATCH 181/555] libexpr-test-support: Remove duplicate definitions of rc::Arbitrary for SingleDerivedPath This redefinition is strictly speaking UB. --- .../include/nix/expr/tests/value/context.hh | 22 +++++-------------- .../include/nix/store/tests/derived-path.hh | 21 +++++++++--------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh index 68a0b8dea7d7..b8f4b4f67acf 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh @@ -4,32 +4,20 @@ #include #include "nix/expr/value/context.hh" +#include "nix/store/tests/derived-path.hh" // IWYU pragma: keep namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); -}; - -template<> -struct Arbitrary -{ - static Gen arbitrary(); -}; - -template<> -struct Arbitrary -{ - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/derived-path.hh b/src/libstore-test-support/include/nix/store/tests/derived-path.hh index b3b43474a914..7d93af08fd1f 100644 --- a/src/libstore-test-support/include/nix/store/tests/derived-path.hh +++ b/src/libstore-test-support/include/nix/store/tests/derived-path.hh @@ -9,36 +9,35 @@ #include "nix/store/tests/outputs-spec.hh" namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc From 94159358f068ab47f1a2c5d3441501df1102a329 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 29 Mar 2026 23:27:44 +0300 Subject: [PATCH 182/555] Add missing pragma once to headersA Needed for unity builds. --- src/libcmd/include/nix/cmd/misc-store-flags.hh | 3 +++ src/libstore/include/nix/store/local-binary-cache-store.hh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/libcmd/include/nix/cmd/misc-store-flags.hh b/src/libcmd/include/nix/cmd/misc-store-flags.hh index 27e139076802..5c38a2593427 100644 --- a/src/libcmd/include/nix/cmd/misc-store-flags.hh +++ b/src/libcmd/include/nix/cmd/misc-store-flags.hh @@ -1,3 +1,6 @@ +#pragma once +/// @file + #include "nix/util/args.hh" #include "nix/store/content-address.hh" diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index ff28b4b7341a..69a4bac1c8a9 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -1,3 +1,6 @@ +#pragma once +/// @file + #include "nix/store/binary-cache-store.hh" namespace nix { From 39827d46c1f3070c6b8afc15e0d634eae8f01f8a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 29 Mar 2026 23:33:38 +0300 Subject: [PATCH 183/555] Prep for unity builds For unity builds we have several issues. We must make sure that we don't do using namespace nix; in any translation unit in libexpr-tests and libstore-tests, because we have an ambiguity between ::Store and nix::Store (same for StorePath). We also need to disambiguate between nix::testing and ::testing namespaces for name lookup. Settings registration RAII objects also need to be called differently. --- src/libexpr-tests/dynamic-attrs-bench.cc | 4 +++- src/libexpr-tests/get-drvs-bench.cc | 5 +++-- src/libexpr-tests/main.cc | 6 ++---- src/libexpr-tests/primops.cc | 12 ++++++------ src/libstore-tests/main.cc | 4 +--- .../include/nix/store/build/derivation-builder.hh | 3 +++ src/libutil-tests/file-content-address.cc | 4 ++-- src/libutil-tests/suggestions.cc | 4 ++-- src/libutil/include/nix/util/compression-settings.hh | 5 +++++ src/nix/nix-env/nix-env.cc | 2 +- src/nix/unix/daemon.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 12 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/libexpr-tests/dynamic-attrs-bench.cc b/src/libexpr-tests/dynamic-attrs-bench.cc index 1b1c199bdff7..ea0736564f59 100644 --- a/src/libexpr-tests/dynamic-attrs-bench.cc +++ b/src/libexpr-tests/dynamic-attrs-bench.cc @@ -5,7 +5,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/store/store-open.hh" -using namespace nix; +namespace nix { static std::string mkDynamicAttrsExpr(size_t attrCount) { @@ -54,3 +54,5 @@ static void BM_EvalDynamicAttrs(benchmark::State & state) } BENCHMARK(BM_EvalDynamicAttrs)->Arg(100)->Arg(500)->Arg(2'000); + +} // namespace nix diff --git a/src/libexpr-tests/get-drvs-bench.cc b/src/libexpr-tests/get-drvs-bench.cc index a5cd59154f22..9241e480480b 100644 --- a/src/libexpr-tests/get-drvs-bench.cc +++ b/src/libexpr-tests/get-drvs-bench.cc @@ -6,8 +6,7 @@ #include "nix/store/store-open.hh" #include "nix/util/fmt.hh" -using namespace nix; - +namespace nix { namespace { struct GetDerivationsEnv @@ -64,3 +63,5 @@ static void BM_GetDerivationsAttrScan(benchmark::State & state) } BENCHMARK(BM_GetDerivationsAttrScan)->Arg(1'000)->Arg(5'000)->Arg(10'000); + +} // namespace nix diff --git a/src/libexpr-tests/main.cc b/src/libexpr-tests/main.cc index 88a9d6684d5b..365fa482de2d 100644 --- a/src/libexpr-tests/main.cc +++ b/src/libexpr-tests/main.cc @@ -3,16 +3,14 @@ #include "nix/store/tests/test-main.hh" #include "nix/util/config-global.hh" -using namespace nix; - int main(int argc, char ** argv) { - auto res = testMainForBuidingPre(argc, argv); + auto res = nix::testMainForBuidingPre(argc, argv); if (res) return res; // For pipe operator tests in trivial.cc - experimentalFeatureSettings.set("experimental-features", "pipe-operators"); + nix::experimentalFeatureSettings.set("experimental-features", "pipe-operators"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/src/libexpr-tests/primops.cc b/src/libexpr-tests/primops.cc index ac5e893bf7d3..fc4f6c0702aa 100644 --- a/src/libexpr-tests/primops.cc +++ b/src/libexpr-tests/primops.cc @@ -637,7 +637,7 @@ TEST_F(PrimOpTest, toStringLambdaThrows) } class ToStringPrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(ToStringPrimOpTest, toString) @@ -651,7 +651,7 @@ TEST_P(ToStringPrimOpTest, toString) INSTANTIATE_TEST_SUITE_P( toString, ToStringPrimOpTest, - testing::Values( + ::testing::Values( CASE(R"("foo")", "foo"), CASE(R"(1)", "1"), CASE(R"([1 2 3])", "1 2 3"), @@ -799,7 +799,7 @@ TEST_F(PrimOpTest, splitVersion) } class CompareVersionsPrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(CompareVersionsPrimOpTest, compareVersions) @@ -813,7 +813,7 @@ TEST_P(CompareVersionsPrimOpTest, compareVersions) INSTANTIATE_TEST_SUITE_P( compareVersions, CompareVersionsPrimOpTest, - testing::Values( + ::testing::Values( // The first two are weird cases. Intuition tells they should // be the same but they aren't. CASE(1.0, 1.0.0, -1), @@ -835,7 +835,7 @@ INSTANTIATE_TEST_SUITE_P( class ParseDrvNamePrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(ParseDrvNamePrimOpTest, parseDrvName) @@ -857,7 +857,7 @@ TEST_P(ParseDrvNamePrimOpTest, parseDrvName) INSTANTIATE_TEST_SUITE_P( parseDrvName, ParseDrvNamePrimOpTest, - testing::Values( + ::testing::Values( std::make_tuple("nix-0.12pre12876", "nix", "0.12pre12876"), std::make_tuple("a-b-c-1234pre5+git", "a-b-c", "1234pre5+git"))); diff --git a/src/libstore-tests/main.cc b/src/libstore-tests/main.cc index c45e3a7f384a..93d93e4aa0ce 100644 --- a/src/libstore-tests/main.cc +++ b/src/libstore-tests/main.cc @@ -3,11 +3,9 @@ #include "nix/store/tests/test-main.hh" #include "nix/store/tests/libstore-network.hh" -using namespace nix; - int main(int argc, char ** argv) { - auto res = testMainForBuidingPre(argc, argv); + auto res = nix::testMainForBuidingPre(argc, argv); if (res) return res; diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index f521c0402a2f..bff249f78154 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -48,6 +48,9 @@ struct ChrootPath bool optional = false; }; +void to_json(nlohmann::json & j, const ChrootPath & cp); +void from_json(const nlohmann::json & j, ChrootPath & cp); + typedef std::map PathsInChroot; // maps target path to source path /** diff --git a/src/libutil-tests/file-content-address.cc b/src/libutil-tests/file-content-address.cc index a6b10d4f62fc..8d6ca3ce87ee 100644 --- a/src/libutil-tests/file-content-address.cc +++ b/src/libutil-tests/file-content-address.cc @@ -33,7 +33,7 @@ TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException) { EXPECT_THAT( []() { parseFileSerialisationMethod("narwhal"); }, - testing::ThrowsMessage(testing::HasSubstr("narwhal"))); + ::testing::ThrowsMessage(::testing::HasSubstr("narwhal"))); } /* ---------------------------------------------------------------------------- @@ -66,7 +66,7 @@ TEST(FileIngestionMethod, testParseFileIngestionMethodOptException) { EXPECT_THAT( []() { parseFileIngestionMethod("narwhal"); }, - testing::ThrowsMessage(testing::HasSubstr("narwhal"))); + ::testing::ThrowsMessage(::testing::HasSubstr("narwhal"))); } } // namespace nix diff --git a/src/libutil-tests/suggestions.cc b/src/libutil-tests/suggestions.cc index a23e5d3f43b4..a94ec30ee692 100644 --- a/src/libutil-tests/suggestions.cc +++ b/src/libutil-tests/suggestions.cc @@ -9,7 +9,7 @@ struct LevenshteinDistanceParam int distance; }; -class LevenshteinDistanceTest : public testing::TestWithParam +class LevenshteinDistanceTest : public ::testing::TestWithParam {}; TEST_P(LevenshteinDistanceTest, CorrectlyComputed) @@ -23,7 +23,7 @@ TEST_P(LevenshteinDistanceTest, CorrectlyComputed) INSTANTIATE_TEST_SUITE_P( LevenshteinDistance, LevenshteinDistanceTest, - testing::Values( + ::testing::Values( LevenshteinDistanceParam{"foo", "foo", 0}, LevenshteinDistanceParam{"foo", "", 3}, LevenshteinDistanceParam{"", "", 0}, diff --git a/src/libutil/include/nix/util/compression-settings.hh b/src/libutil/include/nix/util/compression-settings.hh index f9d42c298c6d..756d636f13e4 100644 --- a/src/libutil/include/nix/util/compression-settings.hh +++ b/src/libutil/include/nix/util/compression-settings.hh @@ -18,4 +18,9 @@ std::optional BaseSetting>::pars template<> std::string BaseSetting>::to_string() const; +/* Same as with all settings - empty string means std::nullopt. */ +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index 63d3eb137b35..b93a51ac533b 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -67,7 +67,7 @@ struct EnvSettings : Config EnvSettings envSettings; -static GlobalConfig::Register rSettings(&envSettings); +static GlobalConfig::Register rEnvSettings(&envSettings); typedef enum { srcNixExprDrvs, srcNixExprs, srcStorePaths, srcProfile, srcAttrPath, srcUnknown } InstallSourceType; diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index c9db03a91200..a8cfb089fc28 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -100,7 +100,7 @@ struct AuthorizationSettings : Config AuthorizationSettings authorizationSettings; -static GlobalConfig::Register rSettings(&authorizationSettings); +static GlobalConfig::Register rAuthorizationSettings(&authorizationSettings); #ifndef __linux__ # define SPLICE_F_MOVE 0 diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 513d76e9b7ef..17b410b32549 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -43,7 +43,7 @@ struct UpgradeSettings : Config UpgradeSettings upgradeSettings; -static GlobalConfig::Register rSettings(&upgradeSettings); +static GlobalConfig::Register rUpgradeSettings(&upgradeSettings); struct CmdUpgradeNix : MixDryRun, StoreCommand { From 7ed1fa9394666fa2ce82041102f52d69184be91d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 29 Mar 2026 23:34:34 +0300 Subject: [PATCH 184/555] Move lexer-helpers.cc into a separate library for unity builds Flex and bison generated code is very tightly coupled with some include cycles that simply must be broken by compiling in separate translation units. --- src/libexpr/meson.build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 893718cb82b7..510b6d696e3b 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -168,7 +168,6 @@ sources = files( 'function-trace.cc', 'get-drvs.cc', 'json-to-value.cc', - 'lexer-helpers.cc', 'nixexpr.cc', 'paths.cc', 'primops.cc', @@ -219,6 +218,7 @@ parser_library = static_library( 'nixexpr-parser', parser_tab, lexer_tab, + 'lexer-helpers.cc', cpp_args : parser_library_cpp_args, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, @@ -229,6 +229,8 @@ parser_library = static_library( override_options : [ 'b_ndebug=@0@'.format(not get_option('debug')), 'b_lto=@0@'.format(get_option('b_lto') and cxx.get_id() != 'gcc'), + # Unity builds mess up the recursive header dependency between flex and bison generated code. + 'unity=off', ], ) From 3f4b66013a714bec3add1cb54709f9faf7193108 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 29 Mar 2026 23:46:16 +0300 Subject: [PATCH 185/555] Make unity builds work --- nix-meson-build-support/common/meson.build | 1 + src/libexpr-tests/regex-cache-bench.cc | 4 +- src/libfetchers-tests/git-utils.cc | 4 +- src/libfetchers-tests/git.cc | 4 +- .../include/nix/store/tests/outputs-spec.hh | 5 +- .../include/nix/store/tests/path.hh | 9 +-- .../include/nix/store/tests/protocol.hh | 6 ++ src/libstore-test-support/outputs-spec.cc | 7 +- src/libstore-test-support/path.cc | 12 +-- src/libstore-tests/derivation-parser-bench.cc | 4 +- .../derivation/external-formats.cc | 73 +++++++++---------- src/libstore-tests/machines.cc | 14 ++-- src/libstore-tests/nar-info.cc | 12 +++ src/libstore-tests/path-info.cc | 12 +++ src/libstore-tests/ref-scan-bench.cc | 4 +- .../register-valid-paths-bench.cc | 4 +- src/libstore-tests/serve-protocol.cc | 14 +--- src/libstore-tests/worker-protocol.cc | 20 ++--- src/libstore-tests/write-derivation.cc | 21 +++--- .../build/derivation-trampoline-goal.cc | 19 +++-- .../include/nix/store/local-settings.hh | 6 ++ .../include/nix/store/store-reference.hh | 19 +++++ src/libutil/compression-settings.cc | 5 -- .../include/nix/util/compression-settings.hh | 1 - src/nix/nix-store/graphml.cc | 10 +-- 25 files changed, 165 insertions(+), 125 deletions(-) diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index d0dde9134847..edb40635f298 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -36,6 +36,7 @@ do_pch = cxx.get_id() == 'clang' if cxx.get_id() == 'gcc' add_project_arguments( '-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings. + '-Wno-subobject-linkage', # GCC doesn't like unity builds. language : 'cpp', ) endif diff --git a/src/libexpr-tests/regex-cache-bench.cc b/src/libexpr-tests/regex-cache-bench.cc index 2eb17b212ab0..c0c61e58e993 100644 --- a/src/libexpr-tests/regex-cache-bench.cc +++ b/src/libexpr-tests/regex-cache-bench.cc @@ -5,7 +5,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/store/store-open.hh" -using namespace nix; +namespace nix { static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state) { @@ -44,3 +44,5 @@ static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state) } BENCHMARK(BM_EvalManyBuiltinsMatchSameRegex); + +} // namespace nix diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index 0b21fd0c67d5..a80ff590f932 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -15,7 +15,7 @@ #include #include -namespace nix { +namespace nix::fetchers { class GitUtilsTest : public ::testing::Test { @@ -234,4 +234,4 @@ TEST(GitUtils, isLegalRefName) ASSERT_FALSE(isLegalRefName("")); } -} // namespace nix +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/git.cc b/src/libfetchers-tests/git.cc index 9fb9a2ce1cd6..e970c3b8bdc4 100644 --- a/src/libfetchers-tests/git.cc +++ b/src/libfetchers-tests/git.cc @@ -98,7 +98,7 @@ static void commitAll(git_repository * repo, const char * msg) } // namespace -using namespace nix; +namespace nix::fetchers { class GitTest : public ::testing::Test { @@ -201,3 +201,5 @@ TEST_F(GitTest, submodulePeriodSupport) ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n"); } + +} // namespace nix::fetchers diff --git a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh index 5bbcc734086a..a30f83770257 100644 --- a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh +++ b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh @@ -9,12 +9,11 @@ #include "nix/store/tests/path.hh" namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/path.hh b/src/libstore-test-support/include/nix/store/tests/path.hh index ff80b1299a04..e2ecd23ca5f0 100644 --- a/src/libstore-test-support/include/nix/store/tests/path.hh +++ b/src/libstore-test-support/include/nix/store/tests/path.hh @@ -18,18 +18,17 @@ void showValue(const StorePath & p, std::ostream & os); } // namespace nix namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/protocol.hh b/src/libstore-test-support/include/nix/store/tests/protocol.hh index 563c8cfb6c4e..543facead0ec 100644 --- a/src/libstore-test-support/include/nix/store/tests/protocol.hh +++ b/src/libstore-test-support/include/nix/store/tests/protocol.hh @@ -122,4 +122,10 @@ public: VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) \ VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) +/// Has to be a `BufferedSink` for handshake. +struct NullBufferedSink : BufferedSink +{ + void writeUnbuffered(std::string_view data) override {} +}; + } // namespace nix diff --git a/src/libstore-test-support/outputs-spec.cc b/src/libstore-test-support/outputs-spec.cc index d5128a8bd91a..947f02eb539f 100644 --- a/src/libstore-test-support/outputs-spec.cc +++ b/src/libstore-test-support/outputs-spec.cc @@ -3,10 +3,13 @@ #include namespace rc { -using namespace nix; -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using nix::OutputsSpec; + using nix::StorePathName; + using nix::StringSet; + return gen::mapcat( gen::inRange(0, std::variant_size_v), [](uint8_t n) -> Gen { switch (n) { diff --git a/src/libstore-test-support/path.cc b/src/libstore-test-support/path.cc index 1459203104c6..1e5f74a28541 100644 --- a/src/libstore-test-support/path.cc +++ b/src/libstore-test-support/path.cc @@ -20,7 +20,6 @@ void showValue(const StorePath & p, std::ostream & os) } // namespace nix namespace rc { -using namespace nix; Gen storePathChar() { @@ -52,18 +51,19 @@ Gen storePathChar() gen::inRange(0, 10 + 2 * 26 + 6)); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { - return gen::construct( + return gen::construct( gen::suchThat(gen::container(storePathChar()), [](const std::string & s) { return !(s == "" || s == "." || s == ".." || s.starts_with(".-") || s.starts_with("..-")); })); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { - return gen::construct( - gen::arbitrary(), gen::apply([](StorePathName n) { return n.name; }, gen::arbitrary())); + return gen::construct( + gen::arbitrary(), + gen::apply([](nix::StorePathName n) { return n.name; }, gen::arbitrary())); } } // namespace rc diff --git a/src/libstore-tests/derivation-parser-bench.cc b/src/libstore-tests/derivation-parser-bench.cc index f0aa721cb7a9..93fa64f6191e 100644 --- a/src/libstore-tests/derivation-parser-bench.cc +++ b/src/libstore-tests/derivation-parser-bench.cc @@ -7,7 +7,7 @@ #include #include -using namespace nix; +namespace nix { // Benchmark parsing real derivation files static void BM_ParseRealDerivationFile(benchmark::State & state, const std::string & filename) @@ -54,3 +54,5 @@ BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, hello, (getUnitTestData() / "deriv BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string()); BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string()); BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string()); + +} // namespace nix diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index a9be99f996ed..6fee675e9849 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -177,44 +177,41 @@ struct DerivationJsonAtermTest : DerivationTest, MAKE_TEST_P(DerivationJsonAtermTest); -Derivation makeSimpleDrv() -{ - Derivation drv; - drv.name = "simple-derivation"; - drv.inputSrcs = { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { - .map = - { - { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - { - .value = - { - "cat", - "dog", - }, - }, - }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = StringPairs{ - { - "BIG_BAD", - "WOLF", - }, - }; - return drv; -} - -INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values(makeSimpleDrv())); +INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values([]() { + Derivation drv; + drv.name = "simple-derivation"; + drv.inputSrcs = { + StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = + { + { + StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + .value = + { + "cat", + "dog", + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = StringPairs{ + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; + }())); struct DynDerivationJsonAtermTest : DynDerivationTest, JsonCharacterizationTest, diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index 4246a876ca87..e412376323de 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -7,13 +7,13 @@ #include #include -using testing::Contains; -using testing::ElementsAre; -using testing::Eq; -using testing::Field; -using testing::SizeIs; +using ::testing::Contains; +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::Field; +using ::testing::SizeIs; -using namespace nix; +namespace nix { TEST(machines, getMachinesWithEmptyBuilders) { @@ -210,3 +210,5 @@ TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) {}, "@" + std::filesystem::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()), FormatError); } + +} // namespace nix diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index 9c8a93e16b77..21a1d63a7e94 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -169,4 +169,16 @@ JSON_TEST_V2(impure, true) JSON_TEST_V3(pure, false) JSON_TEST_V3(impure, true) +#undef JSON_TEST_V1 +#undef JSON_READ_TEST_V1 +#undef JSON_WRITE_TEST_V1 + +#undef JSON_TEST_V2 +#undef JSON_READ_TEST_V2 +#undef JSON_WRITE_TEST_V2 + +#undef JSON_TEST_V3 +#undef JSON_READ_TEST_V3 +#undef JSON_WRITE_TEST_V3 + } // namespace nix diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index 9141d06cd671..7734ee175561 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -182,6 +182,18 @@ JSON_TEST_V3(empty_impure, makeEmpty(), true) JSON_TEST_V3(pure, makeFull(*store, false), false) JSON_TEST_V3(impure, makeFull(*store, true), true) +#undef JSON_TEST_V1 +#undef JSON_READ_TEST_V1 +#undef JSON_WRITE_TEST_V1 + +#undef JSON_TEST_V2 +#undef JSON_READ_TEST_V2 +#undef JSON_WRITE_TEST_V2 + +#undef JSON_TEST_V3 +#undef JSON_READ_TEST_V3 +#undef JSON_WRITE_TEST_V3 + TEST_F(PathInfoTestV2, PathInfo_full_shortRefs) { ValidPathInfo it = makeFullKeyed(*store, true); diff --git a/src/libstore-tests/ref-scan-bench.cc b/src/libstore-tests/ref-scan-bench.cc index ff0aa181503a..05264fd29061 100644 --- a/src/libstore-tests/ref-scan-bench.cc +++ b/src/libstore-tests/ref-scan-bench.cc @@ -6,7 +6,7 @@ #include -using namespace nix; +namespace nix { template static void randomReference(std::mt19937 & urng, OIt outIter) @@ -91,3 +91,5 @@ static void BM_RefScanSinkRandom(benchmark::State & state) } BENCHMARK(BM_RefScanSinkRandom)->Arg(10'000)->Arg(100'000)->Arg(1'000'000)->Arg(5'000'000)->Arg(10'000'000); + +} // namespace nix diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 1d795818369e..400a7055569e 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -12,7 +12,7 @@ # include # include -using namespace nix; +namespace nix { static void BM_RegisterValidPathsDerivations(benchmark::State & state) { @@ -76,4 +76,6 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) BENCHMARK(BM_RegisterValidPathsDerivations)->Arg(10); +} // namespace nix + #endif diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 3eeecd70f4c8..6418cb55dc3d 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -17,8 +17,6 @@ namespace nix { const char serveProtoDir[] = "serve-protocol"; -static constexpr std::string_view defaultStoreDir = "/nix/store"; - struct ServeProtoTest : VersionedProtoTest { /** @@ -326,12 +324,12 @@ VERSIONED_CHARACTERIZATION_TEST( }), (std::tuple{ ({ - UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy}; + UnkeyedValidPathInfo info{"/nix/store", Hash::dummy}; info.narSize = 34878; info; }), ({ - UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy}; + UnkeyedValidPathInfo info{"/nix/store", Hash::dummy}; info.deriver = StorePath{ "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", }; @@ -356,7 +354,7 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple{ ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.deriver = StorePath{ @@ -547,12 +545,6 @@ TEST_F(ServeProtoTest, handshake_log) }); } -/// Has to be a `BufferedSink` for handshake. -struct NullBufferedSink : BufferedSink -{ - void writeUnbuffered(std::string_view data) override {} -}; - TEST_F(ServeProtoTest, handshake_client_replay) { CharacterizationTest::readTest("handshake-to-client.bin", [&](std::string toClientLog) { diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index 044fbcc92028..ed288a7443ab 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -91,8 +91,6 @@ TEST(WorkerProtoVersion, partialOrderingEmptyFeatures) const char workerProtoDir[] = "worker-protocol"; -static constexpr std::string_view defaultStoreDir = "/nix/store"; - struct WorkerProtoTest : VersionedProtoTest { /** @@ -597,7 +595,7 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple{ ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.registrationTime = 23423; @@ -606,7 +604,7 @@ VERSIONED_CHARACTERIZATION_TEST( }), ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.deriver = StorePath{ @@ -641,7 +639,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -655,7 +653,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -696,7 +694,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -711,7 +709,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -979,12 +977,6 @@ TEST_F(WorkerProtoTest, handshake_features) })); } -/// Has to be a `BufferedSink` for handshake. -struct NullBufferedSink : BufferedSink -{ - void writeUnbuffered(std::string_view data) override {} -}; - TEST_F(WorkerProtoTest, handshake_client_replay) { CharacterizationTest::readTest("handshake-to-client.bin", [&](std::string toClientLog) { diff --git a/src/libstore-tests/write-derivation.cc b/src/libstore-tests/write-derivation.cc index c68753823d19..43060cb697b8 100644 --- a/src/libstore-tests/write-derivation.cc +++ b/src/libstore-tests/write-derivation.cc @@ -27,22 +27,19 @@ class WriteDerivationTest : public LibStoreTest ref config; }; -static Derivation makeSimpleDrv() -{ - Derivation drv; - drv.name = "simple-derivation"; - drv.platform = "system"; - drv.builder = "foo"; - drv.args = {"bar", "baz"}; - drv.env = StringPairs{{"BIG_BAD", "WOLF"}}; - return drv; -} - } // namespace TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce) { - auto drv = makeSimpleDrv(); + auto drv = []() { + Derivation drv; + drv.name = "simple-derivation"; + drv.platform = "system"; + drv.builder = "foo"; + drv.args = {"bar", "baz"}; + drv.env = StringPairs{{"BIG_BAD", "WOLF"}}; + return drv; + }(); auto path1 = store->writeDerivation(drv, NoRepair); config->readOnly = true; diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index 58a43c043a2a..4c6553a52182 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -46,18 +46,17 @@ void DerivationTrampolineGoal::commonInit() DerivationTrampolineGoal::~DerivationTrampolineGoal() {} -static StorePath pathPartOfReq(const SingleDerivedPath & req) -{ - return std::visit( - overloaded{ - [&](const SingleDerivedPath::Opaque & bo) { return bo.path; }, - [&](const SingleDerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); }, - }, - req.raw()); -} - std::string DerivationTrampolineGoal::key() { + auto pathPartOfReq = [](this const auto & self, const SingleDerivedPath & req) -> StorePath { + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) { return bo.path; }, + [&](const SingleDerivedPath::Built & bfd) { return self(*bfd.drvPath); }, + }, + req.raw()); + }; + return "da$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + DerivedPath::Built{ .drvPath = drvReq, .outputs = wantedOutputs, diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index cac6dc1b0c49..962e831a3ed2 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -721,4 +721,10 @@ public: const ExternalBuilder * findExternalDerivationBuilderIfSupported(const Derivation & drv); }; +template<> +LocalSettings::ExternalBuilders BaseSetting::parse(const std::string & str) const; + +template<> +std::string BaseSetting::to_string() const; + } // namespace nix diff --git a/src/libstore/include/nix/store/store-reference.hh b/src/libstore/include/nix/store/store-reference.hh index 37ff2cdee3d5..49b41404ee04 100644 --- a/src/libstore/include/nix/store/store-reference.hh +++ b/src/libstore/include/nix/store/store-reference.hh @@ -4,6 +4,7 @@ #include #include "nix/util/types.hh" +#include "nix/util/configuration.hh" #include "nix/util/json-impls.hh" #include "nix/util/json-non-null.hh" @@ -128,6 +129,24 @@ template<> struct json_avoids_null : std::true_type {}; +template<> +StoreReference BaseSetting::parse(const std::string & str) const; + +template<> +std::string BaseSetting::to_string() const; + +template<> +std::vector BaseSetting>::parse(const std::string & str) const; + +template<> +std::string BaseSetting>::to_string() const; + +template<> +std::set BaseSetting>::parse(const std::string & str) const; + +template<> +std::string BaseSetting>::to_string() const; + } // namespace nix JSON_IMPL(StoreReference) diff --git a/src/libutil/compression-settings.cc b/src/libutil/compression-settings.cc index 6e30811ef1b2..14fdfd76b4d7 100644 --- a/src/libutil/compression-settings.cc +++ b/src/libutil/compression-settings.cc @@ -52,11 +52,6 @@ std::string BaseSetting>::to_string() const return ""; } -/* Same as with all settings - empty string means std::nullopt. */ -template<> -struct json_avoids_null : std::true_type -{}; - #define NIX_COMPRESSION_JSON(name, value) {CompressionAlgo::value, name}, NLOHMANN_JSON_SERIALIZE_ENUM(CompressionAlgo, {NIX_FOR_EACH_COMPRESSION_ALGO(NIX_COMPRESSION_JSON)}); #undef NIX_COMPRESSION_JSON diff --git a/src/libutil/include/nix/util/compression-settings.hh b/src/libutil/include/nix/util/compression-settings.hh index 756d636f13e4..942f3dd9c80e 100644 --- a/src/libutil/include/nix/util/compression-settings.hh +++ b/src/libutil/include/nix/util/compression-settings.hh @@ -18,7 +18,6 @@ std::optional BaseSetting>::pars template<> std::string BaseSetting>::to_string() const; -/* Same as with all settings - empty string means std::nullopt. */ template<> struct json_avoids_null : std::true_type {}; diff --git a/src/nix/nix-store/graphml.cc b/src/nix/nix-store/graphml.cc index 009db05d419c..3863247e1227 100644 --- a/src/nix/nix-store/graphml.cc +++ b/src/nix/nix-store/graphml.cc @@ -20,11 +20,6 @@ static std::string symbolicName(std::string_view p) return std::string(p.substr(0, p.find('-') + 1)); } -static std::string makeEdge(std::string_view src, std::string_view dst) -{ - return fmt(" \n", xmlQuote(src), xmlQuote(dst)); -} - static std::string makeNode(const ValidPathInfo & info) { return fmt( @@ -67,6 +62,11 @@ void printGraphML(ref store, StorePathSet && roots) for (auto & p : info->references) { if (p != path) { workList.insert(p); + + auto makeEdge = [](std::string_view src, std::string_view dst) -> std::string { + return fmt(" \n", xmlQuote(src), xmlQuote(dst)); + }; + cout << makeEdge(path.to_string(), p.to_string()); } } From 1e22d60be664469b7d2fe9bf9e0527bf9279c286 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 30 Mar 2026 00:01:09 +0300 Subject: [PATCH 186/555] Clean up BaseOption to_string/parse declarations with a macro Rename to NIX_ to avoid polluting the identifiers and change all of the declaration sites to make them much less verbose. --- src/libexpr/include/nix/expr/diagnose.hh | 6 +--- .../nix/expr/eval-profiler-settings.hh | 6 +--- src/libstore/include/nix/store/s3-url.hh | 6 +--- .../include/nix/store/store-reference.hh | 20 ++----------- .../include/nix/util/compression-settings.hh | 13 ++------- src/libutil/include/nix/util/config-impl.hh | 28 ++++++++----------- src/libutil/include/nix/util/configuration.hh | 6 ++++ 7 files changed, 25 insertions(+), 60 deletions(-) diff --git a/src/libexpr/include/nix/expr/diagnose.hh b/src/libexpr/include/nix/expr/diagnose.hh index 8dfe052134a1..68c8f6543f59 100644 --- a/src/libexpr/include/nix/expr/diagnose.hh +++ b/src/libexpr/include/nix/expr/diagnose.hh @@ -28,11 +28,7 @@ enum struct Diagnose { Fatal, }; -template<> -Diagnose BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(Diagnose) /** * Check a diagnostic setting and either do nothing, log a warning, or throw an error. diff --git a/src/libexpr/include/nix/expr/eval-profiler-settings.hh b/src/libexpr/include/nix/expr/eval-profiler-settings.hh index 32138e7f13f0..4fec151b8061 100644 --- a/src/libexpr/include/nix/expr/eval-profiler-settings.hh +++ b/src/libexpr/include/nix/expr/eval-profiler-settings.hh @@ -7,10 +7,6 @@ namespace nix { enum struct EvalProfilerMode { disabled, flamegraph }; -template<> -EvalProfilerMode BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(EvalProfilerMode) } // namespace nix diff --git a/src/libstore/include/nix/store/s3-url.hh b/src/libstore/include/nix/store/s3-url.hh index 533710745229..2bd2b55e6405 100644 --- a/src/libstore/include/nix/store/s3-url.hh +++ b/src/libstore/include/nix/store/s3-url.hh @@ -30,11 +30,7 @@ MakeError(InvalidS3AddressingStyle, Error); S3AddressingStyle parseS3AddressingStyle(std::string_view style); std::string_view showS3AddressingStyle(S3AddressingStyle style); -template<> -S3AddressingStyle BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(S3AddressingStyle) /** * Parsed S3 URL. diff --git a/src/libstore/include/nix/store/store-reference.hh b/src/libstore/include/nix/store/store-reference.hh index 49b41404ee04..5beeb0c070af 100644 --- a/src/libstore/include/nix/store/store-reference.hh +++ b/src/libstore/include/nix/store/store-reference.hh @@ -129,23 +129,9 @@ template<> struct json_avoids_null : std::true_type {}; -template<> -StoreReference BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; - -template<> -std::vector BaseSetting>::parse(const std::string & str) const; - -template<> -std::string BaseSetting>::to_string() const; - -template<> -std::set BaseSetting>::parse(const std::string & str) const; - -template<> -std::string BaseSetting>::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(StoreReference) +NIX_DECLARE_CONFIG_SERIALISER(std::vector) +NIX_DECLARE_CONFIG_SERIALISER(std::set) } // namespace nix diff --git a/src/libutil/include/nix/util/compression-settings.hh b/src/libutil/include/nix/util/compression-settings.hh index 942f3dd9c80e..9f0245d077d9 100644 --- a/src/libutil/include/nix/util/compression-settings.hh +++ b/src/libutil/include/nix/util/compression-settings.hh @@ -6,17 +6,8 @@ namespace nix { -template<> -CompressionAlgo BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; - -template<> -std::optional BaseSetting>::parse(const std::string & str) const; - -template<> -std::string BaseSetting>::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(CompressionAlgo) +NIX_DECLARE_CONFIG_SERIALISER(std::optional) template<> struct json_avoids_null : std::true_type diff --git a/src/libutil/include/nix/util/config-impl.hh b/src/libutil/include/nix/util/config-impl.hh index 88d82394f312..c719f8653007 100644 --- a/src/libutil/include/nix/util/config-impl.hh +++ b/src/libutil/include/nix/util/config-impl.hh @@ -122,23 +122,17 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) }); } -#define DECLARE_CONFIG_SERIALISER(TY) \ - template<> \ - TY BaseSetting::parse(const std::string & str) const; \ - template<> \ - std::string BaseSetting::to_string() const; - -DECLARE_CONFIG_SERIALISER(std::string) -DECLARE_CONFIG_SERIALISER(std::optional) -DECLARE_CONFIG_SERIALISER(bool) -DECLARE_CONFIG_SERIALISER(Strings) -DECLARE_CONFIG_SERIALISER(StringSet) -DECLARE_CONFIG_SERIALISER(StringMap) -DECLARE_CONFIG_SERIALISER(std::set) -DECLARE_CONFIG_SERIALISER(std::filesystem::path) -DECLARE_CONFIG_SERIALISER(AbsolutePath) -DECLARE_CONFIG_SERIALISER(std::set) -DECLARE_CONFIG_SERIALISER(std::optional) +NIX_DECLARE_CONFIG_SERIALISER(std::string) +NIX_DECLARE_CONFIG_SERIALISER(std::optional) +NIX_DECLARE_CONFIG_SERIALISER(bool) +NIX_DECLARE_CONFIG_SERIALISER(Strings) +NIX_DECLARE_CONFIG_SERIALISER(StringSet) +NIX_DECLARE_CONFIG_SERIALISER(StringMap) +NIX_DECLARE_CONFIG_SERIALISER(std::set) +NIX_DECLARE_CONFIG_SERIALISER(std::filesystem::path) +NIX_DECLARE_CONFIG_SERIALISER(AbsolutePath) +NIX_DECLARE_CONFIG_SERIALISER(std::set) +NIX_DECLARE_CONFIG_SERIALISER(std::optional) template T BaseSetting::parse(const std::string & str) const diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 595b9948c453..19d678601b1e 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -598,6 +598,12 @@ struct ExperimentalFeatureSettings : Config void require(const std::optional &) const; }; +#define NIX_DECLARE_CONFIG_SERIALISER(TY) \ + template<> \ + TY BaseSetting::parse(const std::string & str) const; \ + template<> \ + std::string BaseSetting::to_string() const; + // FIXME: don't use a global variable. extern ExperimentalFeatureSettings experimentalFeatureSettings; From a60d54c1a53eafd93756b2f1e24338fff02a5922 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 30 Mar 2026 01:13:54 +0300 Subject: [PATCH 187/555] packaging: Enable unity builds Best reviewed with git show -w --color-moved. --- flake.nix | 18 +- packaging/components.nix | 12 ++ packaging/dev-shell.nix | 451 ++++++++++++++++++++------------------- packaging/hydra.nix | 2 + 4 files changed, 254 insertions(+), 229 deletions(-) diff --git a/flake.nix b/flake.nix index dc3214d1e090..97db5eef9365 100644 --- a/flake.nix +++ b/flake.nix @@ -512,6 +512,16 @@ devShells = let makeShell = import ./packaging/dev-shell.nix { inherit lib devFlake; }; + makeShell' = + { pkgs }: + makeShell { + inherit pkgs; + nixComponents = pkgs.nixComponents2.overrideScope ( + finalScope: prevScope: { + withUnityBuild = false; + } + ); + }; prefixAttrs = prefix: lib.concatMapAttrs (k: v: { "${prefix}-${k}" = v; }); in forAllSystems ( @@ -519,7 +529,7 @@ prefixAttrs "native" ( forAllStdenvs ( stdenvName: - makeShell { + makeShell' { pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}; } ) @@ -528,7 +538,7 @@ prefixAttrs "static" ( forAllStdenvs ( stdenvName: - makeShell { + makeShell' { pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.pkgsStatic; } ) @@ -536,7 +546,7 @@ // prefixAttrs "llvm" ( forAllStdenvs ( stdenvName: - makeShell { + makeShell' { pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.pkgsLLVM; } ) @@ -544,7 +554,7 @@ // prefixAttrs "cross" ( forAllCrossSystems ( crossSystem: - makeShell { + makeShell' { pkgs = nixpkgsFor.${system}.cross.${crossSystem}; } ) diff --git a/packaging/components.nix b/packaging/components.nix index 09e767cf0891..112791c27d38 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -153,6 +153,13 @@ let ninja ] ++ prevAttrs.nativeBuildInputs or [ ]; + mesonFlags = + prevAttrs.mesonFlags or [ ] + ++ (lib.optionals scope.withUnityBuild [ + "-Dunity=on" + "-Dunity_size=8192" + "-Db_pch=false" + ]); mesonCheckFlags = prevAttrs.mesonCheckFlags or [ ] ++ [ "--print-errorlogs" ]; @@ -319,6 +326,11 @@ in */ withClangTidy = false; + /** + Whether to use [unity builds](https://mesonbuild.com/Unity-builds.html#unity-builds). + */ + withUnityBuild = true; + /** A user-provided extension function to apply to each component derivation. */ diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 72d517d876a8..b058bbe53db7 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -110,245 +110,246 @@ let in -{ pkgs }: - -# TODO: don't use nix-util for this? -pkgs.nixComponents2.nix-util.overrideAttrs ( - finalAttrs: prevAttrs: - - let - stdenv = pkgs.nixDependencies2.stdenv; - buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; - modular = devFlake.getSystem stdenv.buildPlatform.system; - transformFlag = - prefix: flag: - assert builtins.isString flag; - let - rest = builtins.substring 2 (builtins.stringLength flag) flag; - in - "-D${prefix}:${rest}"; - havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; - ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; - - availableComponents = lib.filterAttrs ( - k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v - ) allComponents; - - activeComponents = buildInputsClosureCond isInternal ( - lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents) - ); +{ pkgs, nixComponents }: - allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2; - internalDrvs = byDrvPath ( - # Drop the attr names (not present in buildInputs anyway) - lib.attrValues availableComponents - ++ lib.concatMap (c: lib.filter (v: !v.meta.broken) (lib.attrValues (c.tests or { }))) ( - lib.attrValues availableComponents - ) - ); +nixComponents.callPackage ( + { stdenv }: + (stdenv.mkDerivation ( + finalAttrs: - isInternal = - dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"}; + let + buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = + prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + + availableComponents = lib.filterAttrs ( + k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v + ) allComponents; + + activeComponents = buildInputsClosureCond isInternal ( + lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents) + ); - activeComponentNames = lib.listToAttrs ( - map (c: { - name = c.pname or c.name; - value = null; - }) activeComponents - ); + allComponents = lib.filterAttrs (k: v: lib.isDerivation v) nixComponents; + internalDrvs = byDrvPath ( + # Drop the attr names (not present in buildInputs anyway) + lib.attrValues availableComponents + ++ lib.concatMap (c: lib.filter (v: !v.meta.broken) (lib.attrValues (c.tests or { }))) ( + lib.attrValues availableComponents + ) + ); - isActiveComponent = name: activeComponentNames ? ${name}; + isInternal = + dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"}; - in - { - pname = "shell-for-nix"; + activeComponentNames = lib.listToAttrs ( + map (c: { + name = c.pname or c.name; + value = null; + }) activeComponents + ); - passthru = { - inherit activeComponents; + isActiveComponent = name: activeComponentNames ? ${name}; - # We use this attribute to store non-derivation values like functions and - # perhaps other things that are primarily for overriding and not the shell. - config = { - # Default getComponents - getComponents = - c: - builtins.removeAttrs c ( - lib.optionals (!havePerl) [ "nix-perl-bindings" ] - ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ] + in + { + pname = "shell-for-nix"; + + passthru = { + inherit activeComponents; + + # We use this attribute to store non-derivation values like functions and + # perhaps other things that are primarily for overriding and not the shell. + config = { + # Default getComponents + getComponents = + c: + builtins.removeAttrs c ( + lib.optionals (!havePerl) [ "nix-perl-bindings" ] + ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ] + ); + }; + + /** + Produce a devShell for a given set of nix components + + Example: + + ```nix + shell.withActiveComponents (c: { + inherit (c) nix-util; + }) + ``` + */ + withActiveComponents = + f2: + finalAttrs.finalPackage.overrideAttrs ( + finalAttrs: prevAttrs: { + passthru = prevAttrs.passthru // { + config = prevAttrs.passthru.config // { + getComponents = f2; + }; + }; + } ); - }; - /** - Produce a devShell for a given set of nix components - - Example: - - ```nix - shell.withActiveComponents (c: { - inherit (c) nix-util; - }) - ``` - */ - withActiveComponents = - f2: - finalAttrs.finalPackage.overrideAttrs ( - finalAttrs: prevAttrs: { - passthru = prevAttrs.passthru // { - config = prevAttrs.passthru.config // { - getComponents = f2; - }; - }; - } + small = finalAttrs.finalPackage.withActiveComponents ( + c: + lib.intersectAttrs (lib.genAttrs [ + "nix-cli" + "nix-util-tests" + "nix-store-tests" + "nix-expr-tests" + "nix-fetchers-tests" + "nix-flake-tests" + "nix-functional-tests" + "nix-perl-bindings" + ] (_: null)) c ); + }; - small = finalAttrs.finalPackage.withActiveComponents ( - c: - lib.intersectAttrs (lib.genAttrs [ - "nix-cli" - "nix-util-tests" - "nix-store-tests" - "nix-expr-tests" - "nix-fetchers-tests" - "nix-flake-tests" - "nix-functional-tests" - "nix-perl-bindings" - ] (_: null)) c - ); - }; - - # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop - version = lib.fileContents ../.version; - name = finalAttrs.pname; - - installFlags = "sysconfdir=$(out)/etc"; - shellHook = '' - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH - - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share - - # Make the default phases do the right thing. - # FIXME: this wouldn't be needed if the ninja package set buildPhase() instead of $buildPhase. - # FIXME: mesonConfigurePhase shouldn't cd to the build directory. It would be better to pass '-C ' to ninja. - - cdToBuildDir() { - if [[ ! -e build.ninja ]]; then - cd build - fi - } - - configurePhase() { - mesonConfigurePhase - } - - buildPhase() { - cdToBuildDir - ninjaBuildPhase - } - - checkPhase() { - cdToBuildDir - mesonCheckPhase - } - - installPhase() { - cdToBuildDir - ninjaInstallPhase + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ../.version; + name = finalAttrs.pname; + + installFlags = "sysconfdir=$(out)/etc"; + shellHook = '' + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH + + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share + + # Make the default phases do the right thing. + # FIXME: this wouldn't be needed if the ninja package set buildPhase() instead of $buildPhase. + # FIXME: mesonConfigurePhase shouldn't cd to the build directory. It would be better to pass '-C ' to ninja. + + cdToBuildDir() { + if [[ ! -e build.ninja ]]; then + cd build + fi + } + + configurePhase() { + mesonConfigurePhase + } + + buildPhase() { + cdToBuildDir + ninjaBuildPhase + } + + checkPhase() { + cdToBuildDir + mesonCheckPhase + } + + installPhase() { + cdToBuildDir + ninjaInstallPhase + } + ''; + + # We use this shell with the local checkout, not unpackPhase. + src = null; + + # Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html + # Remove when gdb fix is rolled out everywhere. + separateDebugInfo = false; + + mesonBuildType = "debugoptimized"; + + env = { + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" + modular.pre-commit.settings.rawConfig + }"; } - ''; - - # We use this shell with the local checkout, not unpackPhase. - src = null; - - # Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html - # Remove when gdb fix is rolled out everywhere. - separateDebugInfo = false; + // lib.optionalAttrs stdenv.hostPlatform.isLinux { + CC_LD = "mold"; + CXX_LD = "mold"; + }; - mesonBuildType = "debugoptimized"; + dontUseCmakeConfigure = true; - env = { - # For `make format`, to work without installing pre-commit - _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" - modular.pre-commit.settings.rawConfig - }"; - } - // lib.optionalAttrs stdenv.hostPlatform.isLinux { - CC_LD = "mold"; - CXX_LD = "mold"; - }; - - dontUseCmakeConfigure = true; - - mesonFlags = [ - (lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks")) - ] - ++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags) - ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags) - ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags) - ++ lib.optionals havePerl ( - map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags) - ) - ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags) - ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags); - - nativeBuildInputs = - let - inputs = - dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) ( - lib.lists.concatMap ( - # Nix manual has a build-time dependency on nix, but we - # don't want to do a native build just to enter the cross - # dev shell. - # - # TODO: think of a more principled fix for this. - c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs - ) activeComponents - ) - ) - ++ lib.optional ( - !buildCanExecuteHost - # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 - && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) - && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages - && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages) - ) pkgs.buildPackages.mesonEmulatorHook - ++ [ - pkgs.buildPackages.gnused - modular.pre-commit.settings.package - (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) - pkgs.buildPackages.nixfmt-rfc-style - pkgs.buildPackages.shellcheck - pkgs.buildPackages.include-what-you-use - ] - ++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb - ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( - lib.hiPrio pkgs.buildPackages.clang-tools - ) - ++ lib.optional stdenv.hostPlatform.isLinux pkgs.buildPackages.mold-wrapped; - in - # FIXME: separateDebugInfo = false doesn't actually prevent -Wa,--compress-debug-sections - # from making its way into NIX_CFLAGS_COMPILE. - lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs; - - propagatedNativeBuildInputs = dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) ( - lib.lists.concatMap (c: c.propagatedNativeBuildInputs) activeComponents + mesonFlags = [ + (lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks")) + ] + ++ map (transformFlag "libutil") (ignoreCrossFile nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl ( + map (transformFlag "perl") (ignoreCrossFile nixComponents.nix-perl-bindings.mesonFlags) ) - ); + ++ map (transformFlag "libexpr") (ignoreCrossFile nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags); + + nativeBuildInputs = + let + inputs = + dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) ( + lib.lists.concatMap ( + # Nix manual has a build-time dependency on nix, but we + # don't want to do a native build just to enter the cross + # dev shell. + # + # TODO: think of a more principled fix for this. + c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs + ) activeComponents + ) + ) + ++ lib.optional ( + !buildCanExecuteHost + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages) + ) pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.gnused + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) + pkgs.buildPackages.nixfmt-rfc-style + pkgs.buildPackages.shellcheck + pkgs.buildPackages.include-what-you-use + ] + ++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb + ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( + lib.hiPrio pkgs.buildPackages.clang-tools + ) + ++ lib.optional stdenv.hostPlatform.isLinux pkgs.buildPackages.mold-wrapped; + in + # FIXME: separateDebugInfo = false doesn't actually prevent -Wa,--compress-debug-sections + # from making its way into NIX_CFLAGS_COMPILE. + lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs; + + propagatedNativeBuildInputs = dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) ( + lib.lists.concatMap (c: c.propagatedNativeBuildInputs) activeComponents + ) + ); - buildInputs = - # TODO change Nixpkgs to mark gbenchmark as building on Windows - lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark - ++ dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) - ) - ++ lib.optional havePerl pkgs.perl; + buildInputs = + # TODO change Nixpkgs to mark gbenchmark as building on Windows + lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark + ++ dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) + ) + ++ lib.optional havePerl pkgs.perl; - propagatedBuildInputs = dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) - ); - } -) + propagatedBuildInputs = dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) + ); + } + )) +) { } diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 75830b98cf00..fa41b070cc57 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -173,6 +173,8 @@ rec { # Boost coroutines fail with ASAN on darwin. withASan = !pkgs.stdenv.buildPlatform.isDarwin; withUBSan = true; + # Build without unity to catch include issues. + withUnityBuild = false; nix-expr = super.nix-expr.override { enableGC = false; }; # Unclear how to make Perl bindings work with a dynamically linked ASAN. nix-perl-bindings = null; From 0f67c95181a2ecc371160573d6e1f97268e3c1b4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Feb 2026 22:54:15 +0100 Subject: [PATCH 188/555] Drop unnecessary nlohmann include --- src/libcmd/include/nix/cmd/installable-attr-path.hh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libcmd/include/nix/cmd/installable-attr-path.hh b/src/libcmd/include/nix/cmd/installable-attr-path.hh index 474bb358ec91..ef9dac813346 100644 --- a/src/libcmd/include/nix/cmd/installable-attr-path.hh +++ b/src/libcmd/include/nix/cmd/installable-attr-path.hh @@ -21,8 +21,6 @@ #include #include -#include - namespace nix { class InstallableAttrPath : public InstallableValue From 9e903deff686bf653c56156f5af5514f3d14e38c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2026 18:38:37 +0200 Subject: [PATCH 189/555] Drop another unnecessary nlohmann/json.hpp include --- src/libstore/include/nix/store/derivation-options.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index 4e6ec22f9ccd..d96c4892e1c9 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -2,7 +2,6 @@ ///@file #include -#include #include #include From 75cda3c7d7624df0dca2ad77e57ad1c71250e3fe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 27 Mar 2026 14:19:57 -0400 Subject: [PATCH 190/555] doxygen: suppress warnings in dev builds They are probably good to keep in standalone release builds, but they clutter scroll back massively in dev shells. Co-authored-by: Amaan Qureshi --- src/external-api-docs/doxygen.cfg.in | 1 + src/external-api-docs/meson.build | 1 + src/internal-api-docs/doxygen.cfg.in | 1 + src/internal-api-docs/meson.build | 1 + 4 files changed, 4 insertions(+) diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in index 3af2f5b813fa..18d94e5eeeb3 100644 --- a/src/external-api-docs/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -61,3 +61,4 @@ USE_MDFILE_AS_MAINPAGE = @src@/src/external-api-docs/README.md WARN_IF_UNDOCUMENTED = NO WARN_IF_INCOMPLETE_DOC = NO QUIET = YES +WARNINGS = @WARNINGS@ diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build index cba6f646b93b..1903b36e589b 100644 --- a/src/external-api-docs/meson.build +++ b/src/external-api-docs/meson.build @@ -14,6 +14,7 @@ doxygen_cfg = configure_file( 'PROJECT_NUMBER' : meson.project_version(), 'OUTPUT_DIRECTORY' : meson.current_build_dir(), 'src' : fs.parent(fs.parent(meson.project_source_root())), + 'WARNINGS' : meson.is_subproject() ? 'NO' : 'YES', }, ) diff --git a/src/internal-api-docs/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in index 2769edd9f727..aa81d77c5684 100644 --- a/src/internal-api-docs/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -112,3 +112,4 @@ PREDEFINED = DOXYGEN_SKIP WARN_IF_UNDOCUMENTED = NO WARN_IF_INCOMPLETE_DOC = NO QUIET = YES +WARNINGS = @WARNINGS@ diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build index daab4c93c309..844cb262ee38 100644 --- a/src/internal-api-docs/meson.build +++ b/src/internal-api-docs/meson.build @@ -15,6 +15,7 @@ doxygen_cfg = configure_file( 'OUTPUT_DIRECTORY' : meson.current_build_dir(), 'BUILD_ROOT' : meson.build_root(), 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + 'WARNINGS' : meson.is_subproject() ? 'NO' : 'YES', }, ) From 0a8b126661ba657cc7960d5f589d7ddcb4f6a7f4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Mar 2026 14:16:51 -0400 Subject: [PATCH 191/555] Add `BasicDerivation` JSON serialization Add JSON serialization for `BasicDerivation` with a flat "inputs" array format (vs `Derivation`'s nested `{srcs, drvs}` format), and a corresponding JSON schema (`derivation-resolved-v4.yaml`) that references common field definitions factored out of `derivation-v4.yaml`. Co-authored-by: Amaan Qureshi --- doc/manual/source/SUMMARY.md.in | 1 + .../protocols/json/derivation/resolved.md | 1 + .../json/fixup-json-schema-generated-doc.sed | 3 + doc/manual/source/protocols/json/meson.build | 1 + .../json/schema/derivation-resolved-v4.yaml | 58 ++++++ .../protocols/json/schema/derivation-v4.yaml | 185 ++++++++++-------- src/json-schema-checks/meson.build | 11 ++ src/libstore/derivations.cc | 123 +++++++----- src/libstore/include/nix/store/derivations.hh | 1 + 9 files changed, 255 insertions(+), 129 deletions(-) create mode 100644 doc/manual/source/protocols/json/derivation/resolved.md create mode 100644 doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 1a81bad10b5f..5a17426b9020 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -128,6 +128,7 @@ - [Signature](protocols/json/signature.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation/index.md) + - [Resolved Derivation](protocols/json/derivation/resolved.md) - [Derivation Options](protocols/json/derivation/options.md) - [Deriving Path](protocols/json/deriving-path.md) - [Build Trace Entry](protocols/json/build-trace-entry.md) diff --git a/doc/manual/source/protocols/json/derivation/resolved.md b/doc/manual/source/protocols/json/derivation/resolved.md new file mode 100644 index 000000000000..45e18d9af047 --- /dev/null +++ b/doc/manual/source/protocols/json/derivation/resolved.md @@ -0,0 +1 @@ +{{#include ../derivation-resolved-v4-fixed.md}} diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed index 96b6f1801a57..0162036d420b 100644 --- a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -16,3 +16,6 @@ s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](@docroot@/proto s^\(./hash-v1.yaml\)^[JSON format for `Hash`](@docroot@/protocols/json/hash.html)^g s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](@docroot@/protocols/json/content-address.html#method)^g s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](@docroot@/protocols/json/content-address.html)^g +s^\(./store-path-v1.yaml\)^[JSON format for `StorePath`](@docroot@/protocols/json/store-path.html)^g +s^\(./derivation-v4.yaml\)\?#/\$defs/common/properties/[a-zA-Z]*^[JSON format for `Derivation`](@docroot@/protocols/json/derivation/index.html)^g +s^\(./derivation-v4.yaml\)^[JSON format for `Derivation`](@docroot@/protocols/json/derivation/index.html)^g diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 36e9220c028b..9fcacb20387f 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -16,6 +16,7 @@ schemas = [ 'signature-v2', 'store-object-info-v3', 'derivation-v4', + 'derivation-resolved-v4', 'derivation-options-v1', 'deriving-path-v1', 'build-trace-entry-v3', diff --git a/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml new file mode 100644 index 000000000000..781f07fb374f --- /dev/null +++ b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml @@ -0,0 +1,58 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-resolved-v4.json" +title: Resolved Derivation +description: | + Experimental JSON representation of a resolved Nix derivation (version 4). + + This schema describes the JSON representation of Nix's `BasicDerivation` type, + which is a derivation with all input derivation dependencies resolved to store paths. + This is the result of `Derivation::tryResolve`. + + This is called "version 4" because we wish to keep it in sync with the + [primary (not necessarily resolved) JSON schema](@docroot@/protocols/json/derivation/index.md), + but actually it is the first version for this format. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > and subject to change. + +type: object +required: + - name + - version + - outputs + - inputs + - system + - builder + - args + - env +properties: + name: { "$ref": "derivation-v4.yaml#/$defs/common/properties/name" } + version: { "$ref": "derivation-v4.yaml#/$defs/common/properties/version" } + outputs: { "$ref": "derivation-v4.yaml#/$defs/common/properties/outputs" } + system: { "$ref": "derivation-v4.yaml#/$defs/common/properties/system" } + builder: { "$ref": "derivation-v4.yaml#/$defs/common/properties/builder" } + args: { "$ref": "derivation-v4.yaml#/$defs/common/properties/args" } + env: { "$ref": "derivation-v4.yaml#/$defs/common/properties/env" } + structuredAttrs: { "$ref": "derivation-v4.yaml#/$defs/common/properties/structuredAttrs" } + + inputs: + type: array + title: Input source paths + description: | + List of store paths on which this resolved derivation depends. + Since all derivation inputs have been resolved, only source paths remain. + + > **Example** + > + > ```json + > "inputs": [ + > "b8nwz167km1yciqpwzjj24f8jcy8pq1h-separate-debug-info.sh", + > "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out" + > ] + > ``` + items: + $ref: "store-path-v1.yaml" +additionalProperties: false diff --git a/doc/manual/source/protocols/json/schema/derivation-v4.yaml b/doc/manual/source/protocols/json/schema/derivation-v4.yaml index c41eef31bfc6..170c145c2389 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v4.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v4.yaml @@ -23,57 +23,14 @@ required: - args - env properties: - name: - type: string - title: Derivation name - description: | - The name of the derivation. - Used when calculating store paths for the derivation’s outputs. - - version: - const: 4 - title: Format version (must be 4) - description: | - Must be `4`. - This is a guard that allows us to continue evolving this format. - The choice of `3` is fairly arbitrary, but corresponds to this informal version: - - - Version 0: ATerm format - - - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from ATerm format. - - - Version 2: Separate `method` and `hashAlgo` fields in output specs - - - Version 3: Drop store dir from store paths, just include base name. - - - Version 4: Two cleanups, batched together to lesson churn: - - - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) - - - Use canonical content address JSON format for floating content addressed derivation outputs. - - Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. - - outputs: - type: object - title: Output specifications - description: | - Information about the output paths of the derivation. - This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described. - - > **Example** - > - > ```json - > "outputs": { - > "out": { - > "method": "nar", - > "hashAlgo": "sha256", - > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" - > } - > } - > ``` - additionalProperties: - "$ref": "#/$defs/output/overall" + name: { "$ref": "#/$defs/common/properties/name" } + version: { "$ref": "#/$defs/common/properties/version" } + outputs: { "$ref": "#/$defs/common/properties/outputs" } + system: { "$ref": "#/$defs/common/properties/system" } + builder: { "$ref": "#/$defs/common/properties/builder" } + args: { "$ref": "#/$defs/common/properties/args" } + env: { "$ref": "#/$defs/common/properties/env" } + structuredAttrs: { "$ref": "#/$defs/common/properties/structuredAttrs" } inputs: type: object @@ -126,47 +83,105 @@ properties: - "$ref": "#/$defs/dynamicOutputs" additionalProperties: false additionalProperties: false +additionalProperties: false - system: - type: string - title: Build system type +"$defs": + common: + title: Common Derivation Fields description: | - The system type on which this derivation is to be built - (e.g. `x86_64-linux`). + Fields shared between `Derivation` and `BasicDerivation`. + properties: + name: + type: string + title: Derivation name + description: | + The name of the derivation. + Used when calculating store paths for the derivation's outputs. - builder: - type: string - title: Build program path - description: | - Absolute path of the program used to perform the build. - Typically this is the `bash` shell - (e.g. `/nix/store/p4xlj4imjbnm4v0x5jf4qysvyjjlgq1d-bash-4.4-p23/bin/bash`). + version: + const: 4 + title: Format version (must be 4) + description: | + Must be `4`. + This is a guard that allows us to continue evolving this format. + The choice of `3` is fairly arbitrary, but corresponds to this informal version: - args: - type: array - title: Builder arguments - description: | - Command-line arguments passed to the `builder`. - items: - type: string + - Version 0: ATerm format - env: - type: object - title: Environment variables - description: | - Environment variables passed to the `builder`. - additionalProperties: - type: string + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from ATerm format. - structuredAttrs: - title: Structured attributes - description: | - [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. - Structured attributes are JSON, and thus embedded as-is. - type: object - additionalProperties: true + - Version 2: Separate `method` and `hashAlgo` fields in output specs + + - Version 3: Drop store dir from store paths, just include base name. + + - Version 4: Two cleanups, batched together to lesson churn: + + - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) + + - Use canonical content address JSON format for floating content addressed derivation outputs. + + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. + + outputs: + type: object + title: Output specifications + description: | + Information about the output paths of the derivation. + This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described. + + > **Example** + > + > ```json + > "outputs": { + > "out": { + > "method": "nar", + > "hashAlgo": "sha256", + > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + > } + > } + > ``` + additionalProperties: + "$ref": "#/$defs/output/overall" + + system: + type: string + title: Build system type + description: | + The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + + builder: + type: string + title: Build program path + description: | + Absolute path of the program used to perform the build. + Typically this is the `bash` shell + (e.g. `/nix/store/p4xlj4imjbnm4v0x5jf4qysvyjjlgq1d-bash-4.4-p23/bin/bash`). + + args: + type: array + title: Builder arguments + description: | + Command-line arguments passed to the `builder`. + items: + type: string + + env: + type: object + title: Environment variables + description: | + Environment variables passed to the `builder`. + additionalProperties: + type: string + + structuredAttrs: + title: Structured attributes + description: | + [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. + Structured attributes are JSON, and thus embedded as-is. + type: object + additionalProperties: true -"$defs": output: overall: title: Derivation Output diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 921709b9d305..8a0bde04b8ed 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -98,6 +98,17 @@ schemas += [ 'files' : [ 'dyn-dep-derivation.json', 'simple-derivation.json', + 'try-resolve' / 'no-inputs-before.json', + 'try-resolve' / 'with-inputs-before.json', + 'try-resolve' / 'resolution-failure-before.json', + ], + }, + { + 'stem' : 'derivation', + 'schema' : schema_dir / 'derivation-resolved-v4.yaml', + 'files' : [ + 'try-resolve' / 'no-inputs-after.json', + 'try-resolve' / 'with-inputs-after.json', ], }, { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 16eaff181175..b618e9818023 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1526,32 +1526,52 @@ adl_serializer::from_json(const json & _json, const Experiment } } -void adl_serializer::to_json(json & res, const Derivation & d) +static void inputSrcsToJson(json & res, const StorePathSet & inputSrcs) +{ + res = nlohmann::json::array(); + for (auto & input : inputSrcs) + res.emplace_back(input); +} + +static void basicDerivationToJson(json & res, const BasicDerivation & d) { res = nlohmann::json::object(); res["name"] = d.name; - res["version"] = expectedJsonVersionDerivation; { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); - for (auto & [outputName, output] : d.outputs) { + for (auto & [outputName, output] : d.outputs) outputsObj[outputName] = output; - } } + res["system"] = d.platform; + res["builder"] = d.builder; + res["args"] = d.args; + res["env"] = d.env; + + if (d.structuredAttrs) + res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; +} + +void adl_serializer::to_json(json & res, const BasicDerivation & d) +{ + basicDerivationToJson(res, d); + + inputSrcsToJson(res["inputs"], d.inputSrcs); +} + +void adl_serializer::to_json(json & res, const Derivation & d) +{ + basicDerivationToJson(res, d); + { auto & inputsObj = res["inputs"]; inputsObj = nlohmann::json::object(); - { - auto & inputsList = inputsObj["srcs"]; - inputsList = nlohmann::json::array(); - for (auto & input : d.inputSrcs) - inputsList.emplace_back(input); - } + inputSrcsToJson(inputsObj["srcs"], d.inputSrcs); auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { auto value = nlohmann::json::object(); @@ -1567,28 +1587,21 @@ void adl_serializer::to_json(json & res, const Derivation & d) auto & inputDrvsObj = inputsObj["drvs"]; inputDrvsObj = nlohmann::json::object(); - for (auto & [inputDrv, inputNode] : d.inputDrvs.map) { + for (auto & [inputDrv, inputNode] : d.inputDrvs.map) inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); - } } - - res["system"] = d.platform; - res["builder"] = d.builder; - res["args"] = d.args; - res["env"] = d.env; - - if (d.structuredAttrs) - res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; } -Derivation adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +static void inputSrcsFromJson(const json & inputSrcsJson, StorePathSet & inputSrcs) { - using nlohmann::detail::value_t; - - Derivation res; - - auto & json = getObject(_json); + auto arr = getArray(inputSrcsJson); + for (auto & input : arr) + inputSrcs.insert(input); +} +static void basicDerivationFromJson( + const json::object_t & json, BasicDerivation & res, const ExperimentalFeatureSettings & xpSettings) +{ res.name = getString(valueAt(json, "name")); { @@ -1610,13 +1623,50 @@ Derivation adl_serializer::from_json(const json & _json, const Exper throw; } + res.platform = getString(valueAt(json, "system")); + res.builder = getString(valueAt(json, "builder")); + res.args = getStringList(valueAt(json, "args")); + + auto envJson = valueAt(json, "env"); + try { + res.env = getStringMap(envJson); + } catch (Error & e) { + e.addTrace({}, "while reading key 'env'"); + throw; + } + + if (auto structuredAttrs = get(json, "structuredAttrs")) + res.structuredAttrs = StructuredAttrs{*structuredAttrs}; +} + +BasicDerivation +adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +{ + BasicDerivation res; + auto & json = getObject(_json); + basicDerivationFromJson(json, res, xpSettings); + + try { + inputSrcsFromJson(valueAt(json, "inputs"), res.inputSrcs); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputs'"); + throw; + } + + return res; +} + +Derivation adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +{ + Derivation res; + auto & json = getObject(_json); + basicDerivationFromJson(json, res, xpSettings); + try { auto inputsObj = getObject(valueAt(json, "inputs")); try { - auto inputSrcs = getArray(valueAt(inputsObj, "srcs")); - for (auto & input : inputSrcs) - res.inputSrcs.insert(input); + inputSrcsFromJson(valueAt(inputsObj, "srcs"), res.inputSrcs); } catch (Error & e) { e.addTrace({}, "while reading key 'srcs'"); throw; @@ -1647,21 +1697,6 @@ Derivation adl_serializer::from_json(const json & _json, const Exper throw; } - res.platform = getString(valueAt(json, "system")); - res.builder = getString(valueAt(json, "builder")); - res.args = getStringList(valueAt(json, "args")); - - auto envJson = valueAt(json, "env"); - try { - res.env = getStringMap(envJson); - } catch (Error & e) { - e.addTrace({}, "while reading key 'env'"); - throw; - } - - if (auto structuredAttrs = get(json, "structuredAttrs")) - res.structuredAttrs = StructuredAttrs{*structuredAttrs}; - return res; } diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index b74fb420e69a..4cfc79acbe15 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -620,4 +620,5 @@ constexpr unsigned expectedJsonVersionDerivation = 4; } // namespace nix JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput) +JSON_IMPL_WITH_XP_FEATURES(nix::BasicDerivation) JSON_IMPL_WITH_XP_FEATURES(nix::Derivation) From 0624f2260ea81126219dfb8c2e88e7145b1c42c5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Mar 2026 14:17:09 -0400 Subject: [PATCH 192/555] Add `tryResolve` unit tests Add dedicated unit tests for `Derivation::tryResolve` covering: - No-input derivations - Multi-input/multi-output derivations with structured attrs and embedded placeholders - Resolution failure when inputs cannot be resolved As usual, the tests are JSON based so they are less implementation-specific, gettting use closer to an implementation-agnostic "conformance test suite". Co-authored-by: Amaan Qureshi --- .../try-resolve/no-inputs-after.json | 17 ++ .../try-resolve/no-inputs-before.json | 20 ++ .../try-resolve/no-inputs-buildTrace.json | 1 + .../resolution-failure-before.json | 25 ++ .../resolution-failure-buildTrace.json | 1 + .../try-resolve/with-inputs-after.json | 33 +++ .../try-resolve/with-inputs-before.json | 46 ++++ .../try-resolve/with-inputs-buildTrace.json | 23 ++ src/libstore-tests/derivations.cc | 251 ++++++++++++++++++ src/libstore-tests/meson.build | 1 + 10 files changed, 418 insertions(+) create mode 100644 src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json create mode 100644 src/libstore-tests/derivations.cc diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json new file mode 100644 index 000000000000..ed9b9bc48a56 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json @@ -0,0 +1,17 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "FOO": "bar" + }, + "inputs": [], + "name": "no-inputs", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json new file mode 100644 index 000000000000..8ade9a6d90a9 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json @@ -0,0 +1,20 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "FOO": "bar" + }, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "no-inputs", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json new file mode 100644 index 000000000000..fe51488c7066 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json @@ -0,0 +1 @@ +[] diff --git a/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json new file mode 100644 index 000000000000..cb57937ff53b --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json @@ -0,0 +1,25 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": {}, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "resolution-failure", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json new file mode 100644 index 000000000000..fe51488c7066 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json @@ -0,0 +1 @@ +[] diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json new file mode 100644 index 000000000000..5c318ff84084 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json @@ -0,0 +1,33 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "DEP1_DEV": "/nix/store/j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev", + "DEP1_OUT": "prefix-/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out-suffix", + "DEP2": "/nix/store/i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out", + "i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out", + "j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev" + ], + "name": "with-inputs", + "outputs": { + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "dep1out": "/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out", + "nested": { + "dep2": "before /nix/store/i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out after" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json new file mode 100644 index 000000000000..6b39e5fc067f --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json @@ -0,0 +1,46 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "DEP1_DEV": "/1r6jj1sjvjx4d1jqlnak71d8vzysdl2gyz2m01spjf31nhr1g55f", + "DEP1_OUT": "prefix-/10qd59vslmmj9w7hx8lpr0yxazwvhf3gfrnvbbc2s9071rhl3l55-suffix", + "DEP2": "/1zwa38661ns8faagbp31qzc8vx94hxsxcncs7hvcxhjvbcfyv9wb" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "with-inputs", + "outputs": { + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "dep1out": "/10qd59vslmmj9w7hx8lpr0yxazwvhf3gfrnvbbc2s9071rhl3l55", + "nested": { + "dep2": "before /1zwa38661ns8faagbp31qzc8vx94hxsxcncs7hvcxhjvbcfyv9wb after" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json new file mode 100644 index 000000000000..a15a69869a30 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json @@ -0,0 +1,23 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv", + "output": "dev" + }, + "j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev" + ], + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out" + ], + [ + { + "drvPath": "h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv", + "output": "out" + }, + "i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out" + ] +] diff --git a/src/libstore-tests/derivations.cc b/src/libstore-tests/derivations.cc new file mode 100644 index 000000000000..a898b8ee9c27 --- /dev/null +++ b/src/libstore-tests/derivations.cc @@ -0,0 +1,251 @@ +#include +#include + +#include "nix/store/derivations.hh" +#include "nix/store/downstream-placeholder.hh" +#include "nix/store/parsed-derivations.hh" +#include "nix/store/tests/libstore.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +class TryResolveTest : public LibStoreTest, public CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "derivation" / "try-resolve"; + +public: + + /** + * A simple in-memory *derived* build trace. (e.g. we have + * `SingleDerivedPath::Built` not `StorePath` keys, because that is + * how the `tryResolved` callback works.) + * + * For a real-world build trace, derived build traces are much more + * work to get correct, and should be at most a caching layer atop + * an underlying source-of-truth build trace. (This is as described + * in the manual.) + * + * However, this is just for some simple in-memory unit tests, with + * tiny amounts of data we can hand-review, so it's fine. The point + * is unit testing `tryResolve`, not unit testing "deep queries" + * from a non-derived build trace (what is done in + * `outputs-query.cc`) anyways --- that would be a separate unit + * test. + */ + struct BuildTrace + { + std::map dict; + + bool operator==(const BuildTrace &) const = default; + }; + +protected: + + EnableExperimentalFeature caFeature{"ca-derivations"}; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } + + /** + * Just here because we do this a few times in a tests. + */ + static DerivationOutput caFloatingOutput() + { + return DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}; + } + + /** + * Build a callback from a BuildTrace lookup. + */ + static auto makeCallback(const BuildTrace & table) + { + return + [&table](ref drvPath, const std::string & outputName) -> std::optional { + if (auto p = get(table.dict, SingleDerivedPath::Built{drvPath, outputName})) + return *p; + return std::nullopt; + }; + } + + /** + * Checkpoint before/buildTrace/after and assert the resolved derivation + * matches expected. + */ + void resolveExpect( + std::string_view stem, const Derivation & drv, const BuildTrace & buildTrace, const BasicDerivation & expected) + { + nix::checkpointJson(*this, std::string{stem} + "-before", drv); + nix::checkpointJson(*this, std::string{stem} + "-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + ASSERT_TRUE(resolved); + + nix::checkpointJson(*this, std::string{stem} + "-after", *resolved); + + EXPECT_EQ(*resolved, expected); + } +}; + +} // namespace nix + +JSON_IMPL(nix::TryResolveTest::BuildTrace); + +namespace nlohmann { + +using nix::SingleDerivedPath; +using nix::StorePath; +using nix::TryResolveTest; + +void adl_serializer::to_json(json & j, const TryResolveTest::BuildTrace & t) +{ + j = t.dict; +} + +TryResolveTest::BuildTrace adl_serializer::from_json(const json & j) +{ + return TryResolveTest::BuildTrace{ + .dict = j.get>(), + }; +} + +} // namespace nlohmann + +namespace nix { + +TEST_F(TryResolveTest, noInputs) +{ + resolveExpect( + "no-inputs", + [&] { + Derivation drv; + drv.name = "no-inputs"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.env = {{"FOO", "bar"}}; + return drv; + }(), + {}, + [&] { + BasicDerivation expected; + expected.name = "no-inputs"; + expected.platform = "x86_64-linux"; + expected.builder = "/bin/bash"; + expected.outputs = {{"out", caFloatingOutput()}}; + expected.env = {{"FOO", "bar"}}; + return expected; + }()); +} + +TEST_F(TryResolveTest, withInputs) +{ + // dep1 has two outputs (out, dev), dep2 has one (out) + StorePath dep1DrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv"}; + StorePath dep1OutPath{"f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out"}; + StorePath dep1DevPath{"j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev"}; + StorePath dep2DrvPath{"h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv"}; + StorePath dep2OutPath{"i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out"}; + + auto placeholder1Out = DownstreamPlaceholder::unknownCaOutput(dep1DrvPath, "out").render(); + auto placeholder1Dev = DownstreamPlaceholder::unknownCaOutput(dep1DrvPath, "dev").render(); + auto placeholder2Out = DownstreamPlaceholder::unknownCaOutput(dep2DrvPath, "out").render(); + + DerivationOutputs multiOutputs = { + {"out", caFloatingOutput()}, + {"dev", caFloatingOutput()}, + }; + + resolveExpect( + "with-inputs", + [&] { + Derivation drv; + drv.name = "with-inputs"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = multiOutputs; + drv.inputDrvs = { + .map = { + {dep1DrvPath, {.value = {"out", "dev"}}}, + {dep2DrvPath, {.value = {"out"}}}, + }}; + drv.env = { + {"DEP1_OUT", "prefix-" + placeholder1Out + "-suffix"}, + {"DEP1_DEV", placeholder1Dev}, + {"DEP2", placeholder2Out}, + }; + drv.structuredAttrs = StructuredAttrs{{ + {"dep1out", placeholder1Out}, + {"nested", nlohmann::json::object({{"dep2", "before " + placeholder2Out + " after"}})}, + }}; + return drv; + }(), + {.dict{ + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep1DrvPath), + .output = "out", + }, + dep1OutPath, + }, + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep1DrvPath), + .output = "dev", + }, + dep1DevPath, + }, + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep2DrvPath), + .output = "out", + }, + dep2OutPath, + }, + }}, + [&] { + BasicDerivation expected; + expected.name = "with-inputs"; + expected.platform = "x86_64-linux"; + expected.builder = "/bin/bash"; + expected.outputs = multiOutputs; + expected.inputSrcs = {dep1OutPath, dep1DevPath, dep2OutPath}; + expected.env = { + {"DEP1_OUT", "prefix-" + store->printStorePath(dep1OutPath) + "-suffix"}, + {"DEP1_DEV", store->printStorePath(dep1DevPath)}, + {"DEP2", store->printStorePath(dep2OutPath)}, + }; + expected.structuredAttrs = StructuredAttrs{{ + {"dep1out", store->printStorePath(dep1OutPath)}, + {"nested", + nlohmann::json::object({{"dep2", "before " + store->printStorePath(dep2OutPath) + " after"}})}, + }}; + return expected; + }()); +} + +TEST_F(TryResolveTest, resolutionFailure) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + + Derivation drv; + drv.name = "resolution-failure"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + + BuildTrace buildTrace; + + checkpointJson(*this, "resolution-failure-before", drv); + checkpointJson(*this, "resolution-failure-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + EXPECT_FALSE(resolved); +} + +} // namespace nix diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index fd1f0a05e7ad..a126a87aca8b 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -60,6 +60,7 @@ sources = files( 'derivation-advanced-attrs.cc', 'derivation/external-formats.cc', 'derivation/invariants.cc', + 'derivations.cc', 'derived-path.cc', 'downstream-placeholder.cc', 'dummy-store.cc', From edd2e2f6d8ad7a95224efbd5523a94b2a85a200a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 30 Mar 2026 23:46:35 +0300 Subject: [PATCH 193/555] libstore: Use std::weak_ptr in ItemHandle I noticed cole-h came across this issue in detnix. Silly mistake on my part, the TransferItem can die by the time we might want to unpause it. Haven't seen this fail in in the wild, but the weak_ptr approach is the correct one. The enqueueing thread mustn't take shared ownership. Same for enqueueing for wakeup. Only the worker thread must have the ownership of the TransferItem. --- src/libstore/filetransfer.cc | 21 +++++++++++++------ .../include/nix/store/filetransfer.hh | 4 ++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index d5eb1c538b31..88853620facc 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -925,7 +925,7 @@ struct curlFileTransfer : public FileTransfer }; std::priority_queue, std::vector>, EmbargoComparator> incoming; - std::vector> unpause; + std::vector> unpause; private: bool quitting = false; public: @@ -1098,8 +1098,15 @@ struct curlFileTransfer : public FileTransfer return res; }(); - for (auto & item : unpause) - item->unpause(); + for (auto & item : unpause) { + /* The transfer might have completed (failed) between it getting + enqueued for unpause and by the time the worker thread picked + it up. */ + auto ptr = item.lock(); + if (!ptr) + continue; + static_cast(*ptr).unpause(); + } } debug("download thread shutting down"); @@ -1139,7 +1146,7 @@ struct curlFileTransfer : public FileTransfer } wakeupMulti(); - return ItemHandle(static_cast(*item)); + return ItemHandle(item.get_ptr()); } ItemHandle enqueueFileTransfer(const FileTransferRequest & request, Callback callback) override @@ -1154,7 +1161,7 @@ struct curlFileTransfer : public FileTransfer return enqueueItem(make_ref(*this, request, std::move(callback))); } - void unpauseTransfer(ref item) + void unpauseTransfer(std::weak_ptr item) { auto state(state_.lock()); state->unpause.push_back(std::move(item)); @@ -1163,7 +1170,9 @@ struct curlFileTransfer : public FileTransfer void unpauseTransfer(ItemHandle handle) override { - unpauseTransfer(ref{static_cast(handle.item.get()).shared_from_this()}); + /* The transfer might have completed (more likely failed) when we want + to wake it up. That's why we must use a weak_ptr throughout. */ + unpauseTransfer(handle.item); } }; diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 64f1cadf5693..a423249e6634 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -401,10 +401,10 @@ public: */ struct ItemHandle { - std::reference_wrapper item; + std::weak_ptr item; friend struct FileTransfer; - ItemHandle(Item & item) + explicit ItemHandle(std::weak_ptr item) : item(item) { } From 3672f815d69357b7afa43c797a51078fd135067d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 31 Mar 2026 01:08:02 +0300 Subject: [PATCH 194/555] libutil: Fix FreeBSD build --- src/libutil/file-system.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 16e4e6ce8dee..dfba75e50066 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -462,7 +462,7 @@ std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const will be owned by "wheel"; but if the user is not in "wheel", then "tar" will fail to unpack archives that have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + if (::chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) throw SysError("setting group of directory %1%", PathFmt(tmpDir)); #endif return tmpDir; From 2239fa6d7b4d7d44b32f7209f76b1cd7ceae8856 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 22:42:43 +0000 Subject: [PATCH 195/555] build(deps): bump cachix/install-nix-action from 31.10.2 to 31.10.3 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.10.2 to 31.10.3. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/51f3067b56fe8ae331890c77d4e454f6d60615ff...96951a368ba55167b55f1c916f7d416bac6505fe) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.10.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21f567f7cad2..51cd9b48c72c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,7 +192,7 @@ jobs: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@51f3067b56fe8ae331890c77d4e454f6d60615ff # v31.10.2 + - uses: cachix/install-nix-action@96951a368ba55167b55f1c916f7d416bac6505fe # v31.10.3 if: ${{ !matrix.experimental-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} @@ -258,7 +258,7 @@ jobs: id: installer-tarball-url run: | echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@51f3067b56fe8ae331890c77d4e454f6d60615ff # v31.10.2 + - uses: cachix/install-nix-action@96951a368ba55167b55f1c916f7d416bac6505fe # v31.10.3 with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} From c1c1d477db466eb0b0fc69dba3eccab6081f1136 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 22:42:48 +0000 Subject: [PATCH 196/555] build(deps): bump korthout/backport-action from 4.2.0 to 4.3.0 Bumps [korthout/backport-action](https://github.com/korthout/backport-action) from 4.2.0 to 4.3.0. - [Release notes](https://github.com/korthout/backport-action/releases) - [Commits](https://github.com/korthout/backport-action/compare/4aaf0e03a94ff0a619c9a511b61aeb42adea5b02...3c06f323a58619da1e8522229ebc8d5de2633e46) --- updated-dependencies: - dependency-name: korthout/backport-action dependency-version: 4.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 3fe089df5348..2d01cfefc295 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -26,7 +26,7 @@ jobs: # required to find all branches fetch-depth: 0 - name: Create backport PRs - uses: korthout/backport-action@4aaf0e03a94ff0a619c9a511b61aeb42adea5b02 # v4.2.0 + uses: korthout/backport-action@3c06f323a58619da1e8522229ebc8d5de2633e46 # v4.3.0 id: backport with: # Config README: https://github.com/korthout/backport-action#backport-action From edd0af196d1ed7052adced04a71c0f771aae6c5b Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Tue, 31 Mar 2026 23:23:21 +0800 Subject: [PATCH 197/555] doc: render C API doc for nix-main-c and nix-fetchers-c --- src/external-api-docs/doxygen.cfg.in | 2 ++ src/external-api-docs/package.nix | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in index 3af2f5b813fa..df6d1c5b7dfe 100644 --- a/src/external-api-docs/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -42,6 +42,8 @@ INPUT = \ @src@/src/libexpr-c \ @src@/src/libflake-c \ @src@/src/libstore-c \ + @src@/src/libfetchers-c \ + @src@/src/libmain-c \ @src@/src/external-api-docs/README.md FILE_PATTERNS = nix_api_*.h *.md diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index b194e16d4608..df6aa9838d6a 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -31,7 +31,9 @@ mkMesonDerivation (finalAttrs: { # Source is not compiled, but still must be available for Doxygen # to gather comments. (cpp ../libexpr-c) + (cpp ../libfetchers-c) (cpp ../libflake-c) + (cpp ../libmain-c) (cpp ../libstore-c) (cpp ../libutil-c) ]; From 26213c114f94fdd282b05564203fd9c0cd6667ef Mon Sep 17 00:00:00 2001 From: Pierre Penninckx Date: Tue, 17 Mar 2026 13:14:28 +0100 Subject: [PATCH 198/555] doc: add example for git tags --- src/nix/flake.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/flake.md b/src/nix/flake.md index 290e4694eb51..88d0e1be58a0 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -95,6 +95,8 @@ Here are some examples of flake references in their URL-like representation: branch of a Git repository. * `git+https://github.com/NixOS/patchelf?ref=master&rev=f34751b88bd07d7f44f5cd3200fb4122bf916c7e`: A specific branch *and* revision of a Git repository. +* `git+https://github.com/NixOS/patchelf?ref=refs/tags/0.18.0`: A specific + tag of a Git repository. * `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball flake. From 09de76b98f2f0736e23105a5316318cf2569bcd5 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:23 +0300 Subject: [PATCH 199/555] libstore: Memoise Goal::key() --- src/libstore/build/goal.cc | 4 +--- src/libstore/include/nix/store/build/goal.hh | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index ce6b3e58833d..946314afe140 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -157,9 +157,7 @@ std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller) bool CompareGoalPtrs::operator()(const GoalPtr & a, const GoalPtr & b) const { - std::string s1 = a->key(); - std::string s2 = b->key(); - return s1 < s2; + return a->keyCached() < b->keyCached(); } void addToWeakGoals(WeakGoals & goals, GoalPtr p) diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 3e32f4fecb61..0b7367ff7e02 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -80,6 +80,11 @@ private: */ Goals waitees; + /** + * Memoised result of key(). + */ + std::optional cachedKey; + public: typedef enum { ecBusy, ecSuccess, ecFailed, ecNoSubstituters } ExitCode; @@ -600,6 +605,17 @@ public: */ virtual std::string key() = 0; + /** + * Memoising variant of key(). We really don't want to pay the overhead of + * allocating strings just to compare Goals. + */ + std::string_view keyCached() & + { + if (cachedKey) + return *cachedKey; + return *(cachedKey = key()); + } + /** * @brief Hint for the scheduler, which concurrency limit applies. * @see JobCategory From 6fae3a25f1a07472b45c56862c0ffe2050159e0f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:27 +0300 Subject: [PATCH 200/555] libstore: Optimise memory usage of the Worker Several things to consider: - Derivation is quite big and we used to copy it unnecessarily. We can just have a single instance and pass immutable shared_ptr around. - childTerminated would wake all goals that are waiting for a build slot. This would trigger a lot of coroutine frame allocations in DerivationBuildingGoal::buildLocally. We should wake at most one waiter for each terminated child. Everything else is wasted work. --- src/libstore-tests/worker-substitution.cc | 4 +- .../build/derivation-building-goal.cc | 4 +- src/libstore/build/derivation-goal.cc | 15 ++++-- .../build/derivation-trampoline-goal.cc | 4 +- src/libstore/build/worker.cc | 49 +++++++++++++------ .../store/build/derivation-building-goal.hh | 8 ++- .../nix/store/build/derivation-goal.hh | 4 +- .../include/nix/store/build/worker.hh | 9 +++- 8 files changed, 65 insertions(+), 32 deletions(-) diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index 02b65054eaa6..3534d44d0d8f 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -241,7 +241,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Create a derivation goal for the CA derivation output // The worker should substitute the output rather than building - auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true); + auto goal = worker.makeDerivationGoal(drvPath, make_ref(drv), "out", bmNormal, true); // Run the worker Goals goals; @@ -403,7 +403,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Create a derivation goal for the root derivation output // The worker should substitute the output rather than building - auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false); + auto goal = worker.makeDerivationGoal(rootDrvPath, make_ref(rootDrv), "out", bmNormal, false); // Run the worker Goals goals; diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 64801e4be373..2235cf872b63 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -30,10 +30,10 @@ namespace nix { DerivationBuildingGoal::DerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode, bool storeDerivation) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode, bool storeDerivation) : Goal(worker, gaveUpOnSubstitution(storeDerivation)) , drvPath(drvPath) - , drv{std::make_unique(drv)} + , drv{std::move(drv)} , buildMode(buildMode) { name = fmt("building derivation '%s'", worker.store.printStorePath(drvPath)); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index e824dfdc9ca2..115e48612fcd 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -29,7 +29,7 @@ namespace nix { DerivationGoal::DerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, Worker & worker, BuildMode buildMode, @@ -37,7 +37,7 @@ DerivationGoal::DerivationGoal( : Goal(worker, haveDerivation(storeDerivation)) , drvPath(drvPath) , wantedOutput(wantedOutput) - , drv{std::make_unique(drv)} + , drv{std::move(drv)} , buildMode(buildMode) { @@ -167,8 +167,13 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) if (resolutionGoal->resolvedDrv) { auto & [pathResolved, drvResolved] = *resolutionGoal->resolvedDrv; - auto resolvedDrvGoal = - worker.makeDerivationGoal(pathResolved, drvResolved, wantedOutput, buildMode, /*storeDerivation=*/true); + auto resolvedDrvGoal = worker.makeDerivationGoal( + pathResolved, + make_ref(drvResolved), + wantedOutput, + buildMode, + /*storeDerivation=*/true); + { Goals waitees{resolvedDrvGoal}; co_await await(std::move(waitees)); @@ -227,7 +232,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* Give up on substitution for the output we want, actually build this derivation */ - auto g = worker.makeDerivationBuildingGoal(drvPath, *drv, buildMode, storeDerivation); + auto g = worker.makeDerivationBuildingGoal(drvPath, drv, buildMode, storeDerivation); /* We will finish with it ourselves, as if we were the derivational goal. */ g->preserveFailure = true; diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index 4c6553a52182..edf8d1e86ebc 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -148,8 +148,10 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation /* Build this step! */ + auto sharedDrv = make_ref(std::move(drv)); + for (auto & output : resolvedWantedOutputs) { - auto g = upcast_goal(worker.makeDerivationGoal(drvPath, drv, output, buildMode, false)); + auto g = upcast_goal(worker.makeDerivationGoal(drvPath, sharedDrv, output, buildMode, false)); g->preserveFailure = true; /* We will finish with it ourselves, as if we were the derivational goal. */ concreteDrvGoals.insert(std::move(g)); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 20131aaf3f7e..de23487ac8e5 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -88,13 +88,19 @@ std::shared_ptr Worker::makeDerivationTrampolineGoal( std::shared_ptr Worker::makeDerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, BuildMode buildMode, bool storeDerivation) { return initGoalIfNeeded( - derivationGoals[drvPath][wantedOutput], drvPath, drv, wantedOutput, *this, buildMode, storeDerivation); + derivationGoals[drvPath][wantedOutput], + drvPath, + std::move(drv), + wantedOutput, + *this, + buildMode, + storeDerivation); } std::shared_ptr @@ -104,9 +110,10 @@ Worker::makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation } std::shared_ptr Worker::makeDerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, BuildMode buildMode, bool storeDerivation) + const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation) { - return initGoalIfNeeded(derivationBuildingGoals[drvPath], drvPath, drv, *this, buildMode, storeDerivation); + return initGoalIfNeeded( + derivationBuildingGoals[drvPath], drvPath, std::move(drv), *this, buildMode, storeDerivation); } std::shared_ptr @@ -287,27 +294,37 @@ void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSlee children.erase(i); if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) + auto & waiting = jobCategory == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild; + + /* Wake up goals waiting for a build slot. Wake at most one waiter to avoid + starting unnecessary work (that is accompanied by coroutine frame allocation). */ + auto it = waiting.begin(); + while (it != waiting.end()) { + if (auto goal = it->lock()) { + waiting.erase(it); wakeUp(goal); + break; + } + it = waiting.erase(it); } - - wantingToBuild.clear(); } } void Worker::waitForBuildSlot(GoalPtr goal) { goal->trace("wait for build slot"); - bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution; - if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) - || (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs)) - wakeUp(goal); /* we can do it right away */ + + bool slotAvailable = [&] { + if (goal->jobCategory() == JobCategory::Substitution) + return getNrSubstitutions() < settings.maxSubstitutionJobs; + else + return getNrLocalBuilds() < settings.maxBuildJobs; + }(); + + if (slotAvailable) + wakeUp(goal); /* Can do it right away. */ else - addToWeakGoals(wantingToBuild, goal); + addToWeakGoals(goal->jobCategory() == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild, goal); } void Worker::waitForAnyGoal(GoalPtr goal) diff --git a/src/libstore/include/nix/store/build/derivation-building-goal.hh b/src/libstore/include/nix/store/build/derivation-building-goal.hh index 826cf2c50f07..50621d97d20b 100644 --- a/src/libstore/include/nix/store/build/derivation-building-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-building-goal.hh @@ -41,7 +41,11 @@ struct DerivationBuildingGoal : public Goal * faithfully reconstruct the build history. */ DerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode, bool storeDerivation); + const StorePath & drvPath, + ref drv, + Worker & worker, + BuildMode buildMode, + bool storeDerivation); ~DerivationBuildingGoal(); private: @@ -52,7 +56,7 @@ private: /** * The derivation stored at drvPath. */ - const std::unique_ptr drv; + const ref drv; /** * The remainder is state held during the build. diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index c6bd412182f3..94aba5c3160d 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -45,7 +45,7 @@ struct DerivationGoal : public Goal */ DerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, Worker & worker, BuildMode buildMode, @@ -64,7 +64,7 @@ private: /** * The derivation stored at drvPath. */ - std::unique_ptr drv; + ref drv; const BuildMode buildMode; diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 2f5272cef5d7..104ef64accbc 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -92,6 +92,11 @@ private: */ WeakGoals wantingToBuild; + /** + * Goals waiting for a substitution slot. + */ + WeakGoals wantingToSubstitute; + /** * Child processes currently running. */ @@ -256,7 +261,7 @@ public: std::shared_ptr makeDerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, BuildMode buildMode, bool storeDerivation); @@ -271,7 +276,7 @@ public: * @ref DerivationBuildingGoal "derivation building goal" */ std::shared_ptr makeDerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, BuildMode buildMode, bool storeDerivation); + const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation); /** * @ref PathSubstitutionGoal "substitution goal" From 9db1f7c8af673776a9c18ec31c4be432d0e60d72 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:29 +0300 Subject: [PATCH 201/555] libstore: Get rid of unused wakeSleepers parameter to Worker::childTerminated It was never actually used. Also Administration goals never make their way into this code path (it would make no sense if they did). --- src/libstore/build/worker.cc | 38 +++++++++---------- .../include/nix/store/build/worker.hh | 10 ++--- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index de23487ac8e5..154cb80158e0 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -254,21 +254,21 @@ void Worker::childStarted( nrLocalBuilds++; break; case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; default: + /* Doesn't make sense, since there are only building and substitution slots. */ unreachable(); } } } -void Worker::childTerminated(Goal * goal, bool wakeSleepers) +void Worker::childTerminated(Goal * goal) { - childTerminated(goal, goal->jobCategory(), wakeSleepers); + childTerminated(goal, goal->jobCategory()); } -void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers) +void Worker::childTerminated(Goal * goal, JobCategory jobCategory) { + // FIXME: Inefficient. Make children a map from Goal -> Child instead. auto i = std::find_if(children.begin(), children.end(), [&](const Child & child) { return child.goal2 == goal; }); if (i == children.end()) return; @@ -284,29 +284,25 @@ void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSlee nrLocalBuilds--; break; case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; default: + /* Doesn't make sense, since there are only building and substitution slots. */ unreachable(); } } children.erase(i); - - if (wakeSleepers) { - auto & waiting = jobCategory == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild; - - /* Wake up goals waiting for a build slot. Wake at most one waiter to avoid - starting unnecessary work (that is accompanied by coroutine frame allocation). */ - auto it = waiting.begin(); - while (it != waiting.end()) { - if (auto goal = it->lock()) { - waiting.erase(it); - wakeUp(goal); - break; - } - it = waiting.erase(it); + auto & waiting = jobCategory == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild; + + /* Wake up goals waiting for a build slot. Wake at most one waiter to avoid + starting unnecessary work (that is accompanied by coroutine frame allocation). */ + auto it = waiting.begin(); + while (it != waiting.end()) { + if (auto goal = it->lock()) { + waiting.erase(it); + wakeUp(goal); + break; } + it = waiting.erase(it); } } diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 104ef64accbc..b58249823220 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -331,15 +331,13 @@ public: bool respectTimeouts); /** - * Unregisters a running child process. `wakeSleepers` should be - * false if there is no sense in waking up goals that are sleeping - * because they can't run yet (e.g., there is no free build slot, - * or the hook would still say `postpone`). + * Unregisters a running child process. Wakes at most a single goal that is + * awaiting on the corresponding build slot type (building or substitution). * * This overload requires `goal` to point to a fully constructed, * valid goal object, as it calls `goal->jobCategory()`. */ - void childTerminated(Goal * goal, bool wakeSleepers = true); + void childTerminated(Goal * goal); /** * Unregisters a running child process, like the other overload. @@ -348,7 +346,7 @@ public: * weak goal references, so it is safe to call from destructors * where the goal object may be partially destroyed. */ - void childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers = true); + void childTerminated(Goal * goal, JobCategory jobCategory); /** * Put `goal` to sleep until a build slot becomes available (which From 9ec16e28dd8363e0a921295079dac38ad0ffd26c Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:31 +0300 Subject: [PATCH 202/555] libstore: Get rid of unused Worker::waitForAnyGoal Nothing in-tree actually uses this. --- src/libstore/build/worker.cc | 15 --------------- src/libstore/include/nix/store/build/worker.hh | 11 ----------- 2 files changed, 26 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 154cb80158e0..7f63b790a474 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -207,15 +207,6 @@ void Worker::removeGoal(GoalPtr goal) if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) topGoals.clear(); } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) - wakeUp(goal); - } - - waitingForAnyGoal.clear(); } void Worker::wakeUp(GoalPtr goal) @@ -323,12 +314,6 @@ void Worker::waitForBuildSlot(GoalPtr goal) addToWeakGoals(goal->jobCategory() == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild, goal); } -void Worker::waitForAnyGoal(GoalPtr goal) -{ - goal->trace("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); -} - void Worker::waitForAWhile(GoalPtr goal) { goal->trace("wait for a while"); diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index b58249823220..4c836986811f 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -126,11 +126,6 @@ private: std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; - /** - * Goals waiting for busy paths to be unlocked. - */ - WeakGoals waitingForAnyGoal; - /** * Goals sleeping for a few seconds (polling a lock). */ @@ -354,12 +349,6 @@ public: */ void waitForBuildSlot(GoalPtr goal); - /** - * Wait for any goal to finish. Pretty indiscriminate way to - * wait for some resource that some other goal is holding. - */ - void waitForAnyGoal(GoalPtr goal); - /** * Wait for a few seconds and then retry this goal. Used when * waiting for a lock held by another process. This kind of From d5a64770a4635799c7f283480561ae2da02198ae Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:34 +0300 Subject: [PATCH 203/555] libstore: Get rid of quardatic complexity in the Worker::removeGoal When the number of derivations is huge, derivationTrampolineGoals is also huge. removeGoal would then iterate over all of the goals and try to prune them one-by-one. This would slow down the scheduler to a crawl on just something like 50k derivations. That is totally unacceptable, especially when we can make the removal O(log n) instead of O(n). --- src/libstore/build/worker.cc | 79 ++++++------------- src/libstore/derived-path-map.cc | 28 +++++++ .../store/build/derivation-building-goal.hh | 2 + .../store/build/derivation-resolution-goal.hh | 2 + .../build/drv-output-substitution-goal.hh | 3 +- .../include/nix/store/derived-path-map.hh | 13 +++ 6 files changed, 70 insertions(+), 57 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 7f63b790a474..4666725ed05a 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -141,64 +141,31 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) req.raw()); } -/** - * This function is polymorphic (both via type parameters and - * overloading) and recursive in order to work on a various types of - * trees - * - * @return Whether the tree node we are processing is not empty / should - * be kept alive. In the case of this overloading the node in question - * is the leaf, the weak reference itself. If the weak reference points - * to the goal we are looking for, our caller can delete it. In the - * inductive case where the node is an interior node, we'll likewise - * return whether the interior node is non-empty. If it is empty - * (because we just deleted its last child), then our caller can - * likewise delete it. - */ -template -static bool removeGoal(std::shared_ptr goal, std::weak_ptr & gp) -{ - return gp.lock() != goal; -} - -template -static bool removeGoal(std::shared_ptr goal, std::map & goalMap) -{ - /* !!! inefficient */ - for (auto i = goalMap.begin(); i != goalMap.end();) { - if (!removeGoal(goal, i->second)) - i = goalMap.erase(i); - else - ++i; - } - return !goalMap.empty(); -} - -template -static bool -removeGoal(std::shared_ptr goal, typename DerivedPathMap>>::ChildNode & node) -{ - bool valueKeep = removeGoal(goal, node.value); - bool childMapKeep = removeGoal(goal, node.childMap); - return valueKeep || childMapKeep; -} - void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, derivationTrampolineGoals.map); - else if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, derivationGoals); - else if (auto drvResolutionGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvResolutionGoal, derivationResolutionGoals); - else if (auto drvBuildingGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvBuildingGoal, derivationBuildingGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, substitutionGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, drvOutputSubstitutionGoals); - else - assert(false); + if (auto drvGoal = std::dynamic_pointer_cast(goal)) { + derivationTrampolineGoals.removeSlot(*drvGoal->drvReq, [&](auto & node) { + node.value.erase(drvGoal->wantedOutputs); + /* Return true if ancestors don't need to be pruned. */ + return !node.value.empty(); + }); + } else if (auto drvGoal = std::dynamic_pointer_cast(goal)) { + if (auto it = derivationGoals.find(drvGoal->drvPath); it != derivationGoals.end()) { + it->second.erase(drvGoal->wantedOutput); + if (it->second.empty()) + derivationGoals.erase(it); + } + } else if (auto drvResolutionGoal = std::dynamic_pointer_cast(goal)) { + derivationResolutionGoals.erase(drvResolutionGoal->drvPath); + } else if (auto drvBuildingGoal = std::dynamic_pointer_cast(goal)) { + derivationBuildingGoals.erase(drvBuildingGoal->drvPath); + } else if (auto subGoal = std::dynamic_pointer_cast(goal)) { + substitutionGoals.erase(subGoal->storePath); + } else if (auto subGoal = std::dynamic_pointer_cast(goal)) { + drvOutputSubstitutionGoals.erase(subGoal->id); + } else { + unreachable(); + } if (topGoals.find(goal) != topGoals.end()) { topGoals.erase(goal); diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 3d799ab1c583..684c4462fbf8 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -48,6 +48,34 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single return initIter(k); } +template +void DerivedPathMap::removeSlot(const SingleDerivedPath & k, fun callback) +{ + auto removeIter = [&map = + map](this auto & self, const SingleDerivedPath & k, fun onNode) -> void { + std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) { + if (auto it = map.find(bo.path); it != map.end() && !onNode(it->second)) + map.erase(it); + }, + [&](const SingleDerivedPath::Built & bfd) { + self(*bfd.drvPath, [&](ChildNode & parent) -> bool { + auto it = parent.childMap.find(bfd.output); + if (it == parent.childMap.end()) + return !parent.value.empty() || !parent.childMap.empty(); + if (!onNode(it->second)) + parent.childMap.erase(it); + return !parent.value.empty() || !parent.childMap.empty(); + }); + }, + }, + k.raw()); + }; + + removeIter(k, [&](ChildNode & node) -> bool { return callback(node) || !node.childMap.empty(); }); +} + } // namespace nix // instantiations diff --git a/src/libstore/include/nix/store/build/derivation-building-goal.hh b/src/libstore/include/nix/store/build/derivation-building-goal.hh index 50621d97d20b..6a17f73eeb51 100644 --- a/src/libstore/include/nix/store/build/derivation-building-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-building-goal.hh @@ -31,6 +31,8 @@ typedef enum { rpAccept, rpDecline, rpPostpone } HookReply; */ struct DerivationBuildingGoal : public Goal { + friend class Worker; + /** * @param storeDerivation Whether to store the derivation in * `worker.store`. This is useful for newly-resolved derivations. In this diff --git a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh index b79e6bbb79e4..843e4031aa7e 100644 --- a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh @@ -35,6 +35,8 @@ struct BuilderFailureError; */ struct DerivationResolutionGoal : public Goal { + friend class Worker; + DerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode); /** diff --git a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh index 5f36bbf06d8b..6652da6a20c8 100644 --- a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh +++ b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh @@ -20,11 +20,12 @@ class Worker; * If the output store object itself should also be substituted, that is * the responsibility of the caller to do so. * - * @todo rename this `BuidlTraceEntryGoal`, which will make sense + * @todo rename this `BuildTraceEntryGoal`, which will make sense * especially once `Realisation` is renamed to `BuildTraceEntry`. */ class DrvOutputSubstitutionGoal : public Goal { + friend class Worker; /** * The drv output we're trying to substitute diff --git a/src/libstore/include/nix/store/derived-path-map.hh b/src/libstore/include/nix/store/derived-path-map.hh index c10af84ca26c..65be5df3c4fb 100644 --- a/src/libstore/include/nix/store/derived-path-map.hh +++ b/src/libstore/include/nix/store/derived-path-map.hh @@ -93,6 +93,19 @@ struct DerivedPathMap * `ChildNode::value`. */ ChildNode * findSlot(const SingleDerivedPath & k); + + /** + * Find the node for `k` and invoke @ref callback on it, pruning empty + * ancestors afterwards. + * + * @param callback Invoked on the found node. Should return true if + * the node's value is still non-empty (i.e. the node should be kept). + * If it returns false and the node has no children, the node is erased + * and empty ancestors are pruned recursively. + * + * No-op if the node does not exist. + */ + void removeSlot(const SingleDerivedPath & k, fun callback); }; template<> From 4c9afebaaa0707caf85ff69cb687cc1f9a118794 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:36 +0300 Subject: [PATCH 204/555] libstore: Add useful vomit prints in the Worker::run loop Useful for getting traces like this out of the worker: worker event loop worked goal 'building derivation '/nix/store/llw9vkyi5gj0jv01lrvyj4r0s7pahdmn-X-Reload-Triggers-caddy.drv'' for 182.477ms worker event loop worked goal 'building derivation '/nix/store/jinkpxfafkwflh5sg781knqv7x07w5yx-X-Restart-Triggers-dbus.drv'' for 252.866ms worker event loop worked goal 'building derivation '/nix/store/7jkb4bk87vlb32vzd36hs5iymgrpfyx4-X-Restart-Triggers-nginx-config-reload.drv'' for 196.006ms worker event loop worked goal 'building derivation '/nix/store/267q376i5hk2kzwqiybb8j11387jkdf6-closure-info.drv'' for 221.300ms worker event loop worked goal 'building derivation '/nix/store/5kbnzasxm28qsplwzg78jnzfcvbbkpyl-limesurvey-phpfpm-exec-pre.drv'' for 172.403ms worker event loop worked goal 'building derivation '/nix/store/4ms5j2x70kn1hsj93wkzpwv7ibgwhz9c-mailman-hyperkitty.cfg.drv'' for 177.715ms worker event loop worked goal 'building derivation '/nix/store/hix8qi0cpl19cz6qviwz20hf5lirv57c-nixos-26.05pre-git.drv'' for 170.551ms --- src/libstore/build/worker.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 4666725ed05a..0c63beb4dd1e 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -333,9 +333,24 @@ void Worker::run(const Goals & _topGoals) awake2.insert(goal); } awake.clear(); + for (auto & goal : awake2) { checkInterrupt(); + + std::chrono::time_point startTime; + if (verbosity >= lvlVomit) + startTime = std::chrono::steady_clock::now(); + goal->work(); + + /* Useful for tracing which goals hod the event loop. */ + vomit( + "worker event loop worked goal '%1%' for %2$.3fms", + goal->name, + std::chrono::duration_cast>( + std::chrono::steady_clock::now() - startTime) + .count()); + if (topGoals.empty()) break; // stuff may have been cancelled } From 4a09c1f083679b179d3fc4c42d663ff3832469e6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Apr 2026 22:01:13 +0200 Subject: [PATCH 205/555] Don't detach the auto-GC thread It could still be running after `this` is destroyed. So we need to join it in the LocalStore destructor. --- src/libstore/gc.cc | 9 +++++++-- src/libstore/include/nix/store/local-store.hh | 1 + src/libstore/local-store.cc | 6 ++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index a88fe6a64acb..2d918a1391cb 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -852,12 +852,17 @@ void LocalStore::autoGC(bool sync) if (avail > state->availAfterGC * 0.97) return; + /* Note: since gcRunning is false here, any previous GC thread has exited / is exiting so the join() should be + * almost instantenous. */ + if (state->gcThread.joinable()) + state->gcThread.join(); + state->gcRunning = true; std::promise promise; future = state->gcFuture = promise.get_future().share(); - std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { + state->gcThread = std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { try { /* Wake up any threads waiting for the auto-GC to finish. */ @@ -884,7 +889,7 @@ void LocalStore::autoGC(bool sync) // future, but we don't really care. (what??) ignoreExceptionInDestructor(); } - }).detach(); + }); } sync: diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 63a1da67d8bf..b5c80ba9e56c 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -212,6 +212,7 @@ private: */ bool gcRunning = false; std::shared_future gcFuture; + std::thread gcThread; /** * How much disk space was available after the previous diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e9eb48bfe632..ef8407532bbb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -432,6 +432,12 @@ LocalStore::~LocalStore() future.get(); } + { + auto state(_state->lock()); + if (state->gcThread.joinable()) + state->gcThread.join(); + } + try { auto fdTempRoots(_fdTempRoots.lock()); if (*fdTempRoots) { From f4759088bf26b674294c9652d9f65da0ff6d0eaf Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 2 Apr 2026 02:29:32 +0300 Subject: [PATCH 206/555] libutil: Make `MemorySourceAccessor` exceptions more precise, `makeFSSourceAccessor` tracks metadata Changes `makeFSSourceAccessor` to behave better on symlinks. We had an unfortunate footgun of `PosixSourceAccessor` following interior links if root was a symlink. Prior commits changed that to first load the symlink in `MemorySourceAccessor`, but its exceptions were imprecise and `pathExists` didn't reject interior symlinks and instead returned false in such cases. `mtime` tracking of symlinks is now also restored, with more precise `showPath` and `getPhysicalPath`. Use some new `fstatat` wrappers. I, @ericson2314, forgot they weren't actaully commited yet, so I added them. Co-authored-by: John Ericson --- src/libutil-tests/memory-source-accessor.cc | 40 ++++++- .../include/nix/util/file-system-at.hh | 29 +++++ src/libutil/memory-source-accessor.cc | 107 +++++++++++++----- src/libutil/posix-source-accessor.cc | 73 ++++++++++-- src/libutil/unix/file-system-at.cc | 27 ++++- src/libutil/unix/file-system.cc | 10 +- 6 files changed, 233 insertions(+), 53 deletions(-) diff --git a/src/libutil-tests/memory-source-accessor.cc b/src/libutil-tests/memory-source-accessor.cc index d9a48da0cebd..e80bfaeb9adb 100644 --- a/src/libutil-tests/memory-source-accessor.cc +++ b/src/libutil-tests/memory-source-accessor.cc @@ -144,7 +144,7 @@ TEST_F(MemorySourceAccessorTestErrors, addFileParentNotDirectory) [&] { accessor->addFile(CanonPath("file/child"), "contents"); }, ThrowsMessage(AllOf( HasSubstrIgnoreANSIMatcher("somepath/file/child"), - HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); + HasSubstrIgnoreANSIMatcher("file 'somepath/file' is not a directory")))); } TEST_F(MemorySourceAccessorTestErrors, addFileNotARegularFile) @@ -165,7 +165,7 @@ TEST_F(MemorySourceAccessorTestErrors, createDirectoryParentNotDirectory) [&] { sink.createDirectory(CanonPath("file/child")); }, ThrowsMessage(AllOf( HasSubstrIgnoreANSIMatcher("somepath/file/child"), - HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); + HasSubstrIgnoreANSIMatcher("file 'somepath/file' is not a directory")))); } TEST_F(MemorySourceAccessorTestErrors, createDirectoryNotADirectory) @@ -185,8 +185,8 @@ TEST_F(MemorySourceAccessorTestErrors, createRegularFileParentNotDirectory) EXPECT_THAT( [&] { sink.createRegularFile(CanonPath("file/child"), [](CreateRegularFileSink &) {}); }, ThrowsMessage(AllOf( - HasSubstrIgnoreANSIMatcher("file/child"), - HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); + HasSubstrIgnoreANSIMatcher("somepath/file/child"), + HasSubstrIgnoreANSIMatcher("file 'somepath/file' is not a directory")))); } TEST_F(MemorySourceAccessorTestErrors, createRegularFileNotARegularFile) @@ -207,7 +207,7 @@ TEST_F(MemorySourceAccessorTestErrors, createSymlinkParentNotDirectory) [&] { sink.createSymlink(CanonPath("file/child"), "target"); }, ThrowsMessage(AllOf( HasSubstrIgnoreANSIMatcher("somepath/file/child"), - HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); + HasSubstrIgnoreANSIMatcher("file 'somepath/file' is not a directory")))); } TEST_F(MemorySourceAccessorTestErrors, createSymlinkNotASymlink) @@ -220,6 +220,36 @@ TEST_F(MemorySourceAccessorTestErrors, createSymlinkNotASymlink) AllOf(HasSubstrIgnoreANSIMatcher("somepath/file"), HasSubstrIgnoreANSIMatcher("is not a symbolic link")))); } +TEST_F(MemorySourceAccessorTestErrors, pathExistsThroughRegularFile) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + EXPECT_FALSE(accessor->pathExists(CanonPath("file/child"))); +} + +TEST_F(MemorySourceAccessorTestErrors, maybeLstatThroughRegularFile) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + EXPECT_FALSE(accessor->maybeLstat(CanonPath("file/child"))); +} + +TEST_F(MemorySourceAccessorTestErrors, pathExistsThroughSymlink) +{ + sink.createSymlink(CanonPath("link"), "target"); + EXPECT_THAT( + [&] { accessor->pathExists(CanonPath("link/child")); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/link"), HasSubstrIgnoreANSIMatcher("is a symlink")))); +} + +TEST_F(MemorySourceAccessorTestErrors, createFileThroughSymlink) +{ + sink.createSymlink(CanonPath("link"), "target"); + EXPECT_THAT( + [&] { sink.createRegularFile(CanonPath("link/child"), [](CreateRegularFileSink &) {}); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/link"), HasSubstrIgnoreANSIMatcher("is a symlink")))); +} + /* ---------------------------------------------------------------------------- * JSON * --------------------------------------------------------------------------*/ diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh index a8cb17b95e1f..c78cafe6a559 100644 --- a/src/libutil/include/nix/util/file-system-at.hh +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -34,6 +34,35 @@ namespace nix { */ PosixStat fstat(Descriptor fd); +#ifndef _WIN32 + +/** + * Get status of a file relative to a directory file descriptor. + * + * @param dirFd Directory file descriptor + * @param path Relative path to stat + * + * @return nullopt if the path does not exist. + * @throws SystemError on other I/O errors. + * + * @pre `path` must be relative (not absolute) and non-empty. + */ +std::optional maybeFstatat(Descriptor dirFd, const std::filesystem::path & path); + +/** + * Get status of a file relative to a directory file descriptor. + * + * @param dirFd Directory file descriptor + * @param path Relative path to stat + * + * @throws SystemError if the path does not exist or on other I/O errors. + * + * @pre `path` must be relative (not absolute) and non-empty. + */ +PosixStat fstatat(Descriptor dirFd, const std::filesystem::path & path); + +#endif + /** * Read a symlink relative to a directory file descriptor. * diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index 06d81ba5583f..f0f7a952296a 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -1,6 +1,8 @@ #include "nix/util/memory-source-accessor.hh" #include "nix/util/json-utils.hh" +#include + namespace nix { MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, std::optional create) @@ -24,10 +26,20 @@ MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, bool newF = false; + unsigned j = 0; for (std::string_view name : path) { auto * curDirP = std::get_if(&cur->raw); - if (!curDirP) + if (!curDirP) { + CanonPath curDirPath = CanonPath::root; + for (auto name2 : std::views::take(path, j)) + curDirPath.push(name2); + if (std::holds_alternative(cur->raw)) + throw SymlinkNotAllowed(curDirPath, "file '%s' is a symlink", showPath(curDirPath)); + if (create) + throw NotADirectory("file '%s' is not a directory", showPath(curDirPath)); return nullptr; + } + auto & curDir = *curDirP; auto i = curDir.entries.find(name); @@ -45,6 +57,7 @@ MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, } } cur = &i->second; + ++j; } if (newF && create) @@ -58,12 +71,17 @@ void MemorySourceAccessor::readFile(const CanonPath & path, Sink & sink, fun(&f->raw)) { - sizeCallback(r->contents.size()); - StringSource source{r->contents}; - source.drainInto(sink); - } else - throw NotARegularFile("file '%s' is not a regular file", showPath(path)); + std::visit( + overloaded{ + [&](const File::Regular & r) { + sizeCallback(r.contents.size()); + StringSource source{r.contents}; + source.drainInto(sink); + }, + [&](const File::Directory &) { throw NotARegularFile("file '%s' is not a regular file", showPath(path)); }, + [&](const File::Symlink &) { throw SymlinkNotAllowed(path, "file '%s' is a symlink", showPath(path)); }, + }, + f->raw); } bool MemorySourceAccessor::pathExists(const CanonPath & path) @@ -108,14 +126,22 @@ MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const Canon auto * f = open(path, std::nullopt); if (!f) throw FileNotFound("file '%s' does not exist", showPath(path)); - if (auto * d = std::get_if(&f->raw)) { - DirEntries res; - for (auto & [name, file] : d->entries) - res.insert_or_assign(name, file.lstat().type); - return res; - } else - throw NotADirectory("file '%s' is not a directory", showPath(path)); - return {}; + return std::visit( + overloaded{ + [&](const File::Directory & d) { + DirEntries res; + for (auto & [name, file] : d.entries) + res.insert_or_assign(name, file.lstat().type); + return res; + }, + [&](const File::Regular &) -> DirEntries { + throw NotADirectory("file '%s' is not a directory", showPath(path)); + }, + [&](const File::Symlink &) -> DirEntries { + throw SymlinkNotAllowed(path, "file '%s' is a symlink", showPath(path)); + }, + }, + f->raw); } std::string MemorySourceAccessor::readLink(const CanonPath & path) @@ -135,9 +161,15 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents if (!root && !path.isRoot()) open(CanonPath::root, File::Directory{}); - auto * f = open(path, File{File::Regular{}}); - if (!f) - throw Error("file '%s' cannot be created because some parent file is not a directory", showPath(path)); + MemorySourceAccessor::File * f = nullptr; + try { + f = open(path, File{File::Regular{}}); + if (!f) + throw Error("file '%s' cannot be created because some parent directories don't exist", showPath(path)); + } catch (SourceAccessorError & e) { + e.addTrace({}, "while creating file '%s'", showPath(path)); + throw; + } if (auto * r = std::get_if(&f->raw)) r->contents = std::move(contents); else @@ -150,10 +182,16 @@ using File = MemorySourceAccessor::File; void MemorySink::createDirectory(const CanonPath & path) { - auto * f = dst.open(path, File{File::Directory{}}); - if (!f) - throw Error("directory '%s' cannot be created because some parent file is not a directory", dst.showPath(path)); - + MemorySourceAccessor::File * f = nullptr; + try { + f = dst.open(path, File{File::Directory{}}); + if (!f) + throw Error( + "directory '%s' cannot be created because some parent directories don't exist", dst.showPath(path)); + } catch (SourceAccessorError & e) { + e.addTrace({}, "while creating directory '%s'", dst.showPath(path)); + throw; + } if (!std::holds_alternative(f->raw)) throw NotADirectory("file '%s' is not a directory", dst.showPath(path)); }; @@ -174,9 +212,15 @@ struct CreateMemoryRegularFile : CreateRegularFileSink void MemorySink::createRegularFile(const CanonPath & path, fun func) { - auto * f = dst.open(path, File{File::Regular{}}); - if (!f) - throw Error("file '%s' cannot be created because some parent file is not a directory", path); + MemorySourceAccessor::File * f = nullptr; + try { + f = dst.open(path, File{File::Regular{}}); + if (!f) + throw Error("file '%s' cannot be created because some parent directories don't exist", dst.showPath(path)); + } catch (SourceAccessorError & e) { + e.addTrace({}, "while creating regular file '%s'", dst.showPath(path)); + throw; + } if (auto * rp = std::get_if(&f->raw)) { CreateMemoryRegularFile crf{*rp}; func(crf); @@ -201,9 +245,16 @@ void CreateMemoryRegularFile::operator()(std::string_view data) void MemorySink::createSymlink(const CanonPath & path, const std::string & target) { - auto * f = dst.open(path, File{File::Symlink{}}); - if (!f) - throw Error("symlink '%s' cannot be created because some parent file is not a directory", dst.showPath(path)); + MemorySourceAccessor::File * f = nullptr; + try { + f = dst.open(path, File{File::Symlink{}}); + if (!f) + throw Error( + "symlink '%s' cannot be created because some parent directories don't exist", dst.showPath(path)); + } catch (SourceAccessorError & e) { + e.addTrace({}, "while creating symlink '%s'", dst.showPath(path)); + throw; + } if (auto * s = std::get_if(&f->raw)) s->target = target; else diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 534151dfcfd2..45af4e88ffd6 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -315,20 +315,69 @@ ref makeFSSourceAccessor(std::filesystem::path root, bool trackL AutoCloseFD fd = openFileReadonly(root, FinalSymlink::DontFollow); if (!fd) { - if (errno == ELOOP) { - /* This branch is taken either if the final component is a symlink - or we hit a symlink loop. If that's the latter this will also - throw. This can be done with O_PATH descriptor for the symlink - itself, but it's not portable. */ - auto linkTarget = readLink(root); - auto res = make_ref(); - /* Create an in-memory accessor with the symlink at the root. */ - MemorySink sink{*res}; - sink.createSymlink(CanonPath::root, os_string_to_string(linkTarget)); - return res; + if (errno != ELOOP) + throw NativeSysError("opening file %1%", PathFmt(root)); + + /* A helper class that holds the symlink destination in memory. */ + class SymlinkSourceAccessor : public MemorySourceAccessor + { + bool trackLastModified; + std::time_t mtime; + std::filesystem::path fsPath; + + public: + SymlinkSourceAccessor( + std::string target, std::filesystem::path fsPath_, bool trackLastModified, std::time_t mtime) + : trackLastModified(trackLastModified) + , mtime(mtime) + , fsPath(std::move(fsPath_)) + { + MemorySink sink{*this}; + sink.createSymlink(CanonPath::root, target); + displayPrefix = fsPath.native(); + } + + std::optional getLastModified() override + { + return trackLastModified ? std::optional{mtime} : std::nullopt; + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + if (path.isRoot()) + return fsPath; + return fsPath / path.rel(); /* RHS must be a relative path. */ + } + + std::string showPath(const CanonPath & path) override + { + /* When rendering the file itself omit the trailing slash. */ + return path.isRoot() ? displayPrefix : SourceAccessor::showPath(path); + } + }; + + assert(root.has_parent_path()); + assert(root.has_filename()); + + /* This branch is taken either if the final component is a symlink + or we hit a symlink loop. If that's the latter this will also + throw. This can be done with O_PATH descriptor for the symlink + itself, but it's not portable. Note that file must have a parent directory + (it's required to be absolute and root directories can't fail with ELOOP). */ + + auto parentFd = openDirectory(root.parent_path(), FinalSymlink::Follow); + std::time_t mtime = 0; + if (!parentFd) + throw SysError("opening %1%", PathFmt(root)); + + auto relPath = CanonPath::fromFilename(root.filename().native()); + if (trackLastModified) { + auto st = fstatat(parentFd.get(), root.filename()); + mtime = st.st_mtime; } - throw NativeSysError("opening file %1%", PathFmt(root)); + auto linkTarget = readLinkAt(parentFd.get(), relPath); + return make_ref(std::move(linkTarget), std::move(root), trackLastModified, mtime); } auto st = nix::fstat(fd.get()); diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index fee985fd5d65..d2401382ef99 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -168,8 +168,7 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat }); if (errno == ENOTDIR) /* Path component might be a symlink. */ { - struct ::stat st; - if (::fstatat(getParentFd(), component.c_str(), &st, AT_SYMLINK_NOFOLLOW) == 0 && S_ISLNK(st.st_mode)) + if (auto st = maybeFstatat(getParentFd(), component); st && S_ISLNK(st->st_mode)) throw SymlinkNotAllowed(path2); errno = ENOTDIR; /* Restore the errno. */ } else if (errno == ELOOP) { @@ -230,4 +229,28 @@ PosixStat fstat(Descriptor fd) return st; } +PosixStat fstatat(Descriptor dirFd, const std::filesystem::path & path) +{ + assert(path.is_relative()); + assert(!path.empty()); + PosixStat st; + if (::fstatat(dirFd, path.c_str(), &st, AT_SYMLINK_NOFOLLOW)) { + throw SysError([&] { return HintFmt("getting status of %s", PathFmt(descriptorToPath(dirFd) / path)); }); + } + return st; +} + +std::optional maybeFstatat(Descriptor dirFd, const std::filesystem::path & path) +{ + assert(path.is_relative()); + assert(!path.empty()); + PosixStat st; + if (::fstatat(dirFd, path.c_str(), &st, AT_SYMLINK_NOFOLLOW)) { + if (errno == ENOENT || errno == ENOTDIR) + return std::nullopt; + throw SysError([&] { return HintFmt("getting status of %s", PathFmt(descriptorToPath(dirFd) / path)); }); + } + return st; +} + } // namespace nix diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index 1e6c4e3d88a8..b154ee215225 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -169,12 +169,10 @@ static void _deletePath( auto name = CanonPath::fromFilename(path.filename().native()); - PosixStat st; - if (fstatat(parentfd, name.rel_c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { - if (errno == ENOENT) - return; - throw SysError("getting status of %1%", PathFmt(path)); - } + auto st_ = maybeFstatat(parentfd, name.rel()); + if (!st_) + return; + auto & st = *st_; if (!S_ISDIR(st.st_mode)) { /* We are about to delete a file. Will it likely free space? */ From 90db12ceb17880dec8eacaa746d75ca409c750e9 Mon Sep 17 00:00:00 2001 From: dramforever Date: Thu, 2 Apr 2026 16:01:05 +0800 Subject: [PATCH 207/555] eval-cache: Add derivation result as temproot Closes the window where the derivation could be garbage collected after we checked isValidPath. --- src/libexpr/eval-cache.cc | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index a97cef550e8d..a419530c6dd8 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -707,13 +707,16 @@ StorePath AttrCursor::forceDerivation() auto aDrvPath = getAttr(root->state.s.drvPath); auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); drvPath.requireDerivation(); - if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { - /* The eval cache contains 'drvPath', but the actual path has - been garbage-collected. So force it to be regenerated. */ - aDrvPath->forceValue(); - if (!root->state.store->isValidPath(drvPath)) - throw Error( - "don't know how to recreate store derivation '%s'!", root->state.store->printStorePath(drvPath)); + if (!settings.readOnlyMode) { + root->state.store->addTempRoot(drvPath); + if (!root->state.store->isValidPath(drvPath)) { + /* The eval cache contains 'drvPath', but the actual path has + been garbage-collected. So force it to be regenerated. */ + aDrvPath->forceValue(); + if (!root->state.store->isValidPath(drvPath)) + throw Error( + "don't know how to recreate store derivation '%s'!", root->state.store->printStorePath(drvPath)); + } } return drvPath; } From 0e3412a93f43a017342c267000c152e5c45327e7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 3 Apr 2026 00:21:21 +0300 Subject: [PATCH 208/555] libstore: Make temporary in-store directory not world-readable --- src/libstore/local-store.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e9eb48bfe632..d5e45773120c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1304,8 +1304,9 @@ std::pair LocalStore::createTempDirInStore() do { /* There is a slight possibility that `tmpDir' gets deleted by the GC between createTempDir() and when we acquire a lock on it. - We'll repeat until 'tmpDir' exists and we've locked it. */ - tmpDirFn = createTempDir(std::filesystem::path{config->realStoreDir.get()}, "tmp"); + We'll repeat until 'tmpDir' exists and we've locked it. + Make the directory accessible only to the current user. */ + tmpDirFn = createTempDir(std::filesystem::path{config->realStoreDir.get()}, "tmp", /*mode=*/0700); tmpDirFd = openDirectory(tmpDirFn, FinalSymlink::DontFollow); if (!tmpDirFd) { continue; From 8dbb3daee0d435ec54441146f46ecfb1a45c8d83 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sun, 29 Mar 2026 21:50:48 +0000 Subject: [PATCH 209/555] build: link mimalloc into the nix binary by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mimalloc is a compact, general-purpose allocator from Microsoft that consistently outperforms glibc malloc on allocation-heavy workloads. Since the nix evaluator is extremely allocation-heavy (strings, maps, symbol tables, derivation parsing), this is a cheap win. The allocator is linked into the nix executable where it overrides malloc for all components via symbol interposition. Boehm GC's heap (the bulk of evaluator memory) is unaffected since GC_malloc bypasses the system allocator entirely — but the ~5% of allocations that do go through malloc get noticeably faster. Benchmarked on x86_64-linux against nixpkgs master: | workload | thunks | glibc | mimalloc | wall | |-----------------------|--------|-------------|-------------|--------| | hello | 275K | 185.4 ms | 176.4 ms | 1.05× | | chromium | 1.9M | 760.3 ms | 709.3 ms | 1.07× | | firefox-unwrapped | 2.4M | 836.2 ms | 782.0 ms | 1.07× | | texliveFull | 5.5M | 1.604 s | 1.466 s | 1.09× | | large NixOS config | 39M | 12.79 s | 11.90 s | 1.07× | | nix-env -qa | 100M+ | 118.96 s | 105.86 s | 1.12× | Implementation: - new meson feature option 'mimalloc' (default: auto) - package.nix wires it with withMimalloc ? !isWindows - dev-shell includes mimalloc so local builds pick it up - static builds work via pkgsStatic.mimalloc's dev output pkg-config Also compared against tcmalloc and jemalloc: tcmalloc matches mimalloc on wall-clock; jemalloc lags slightly. mimalloc has the smallest footprint (no libunwind dep) so it's the pick here. --- doc/manual/rl-next/mimalloc.md | 15 +++++++++++++++ src/nix/meson.build | 5 +++++ src/nix/meson.options | 7 +++++++ src/nix/package.nix | 11 ++++++++++- 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 doc/manual/rl-next/mimalloc.md diff --git a/doc/manual/rl-next/mimalloc.md b/doc/manual/rl-next/mimalloc.md new file mode 100644 index 000000000000..dfcde0cab03a --- /dev/null +++ b/doc/manual/rl-next/mimalloc.md @@ -0,0 +1,15 @@ +--- +synopsis: "Link mimalloc for faster evaluation" +prs: [15596] +--- + +The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) +by default on non-Windows platforms, replacing glibc's malloc for all +non-GC allocations. + +This yields a **5–12% wall-clock improvement** on evaluation workloads, +ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS +configurations. + +The allocator can be disabled at build time with `-Dmimalloc=disabled` +or by passing `withMimalloc = false` to the Nix package. diff --git a/src/nix/meson.build b/src/nix/meson.build index 039b202b937d..3327b846c9c4 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -31,6 +31,11 @@ deps_private_maybe_subproject = [ deps_public_maybe_subproject = [] subdir('nix-meson-build-support/subprojects') +mimalloc = dependency('mimalloc', required : get_option('mimalloc')) +if mimalloc.found() + deps_private += mimalloc +endif + subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') diff --git a/src/nix/meson.options b/src/nix/meson.options index 0fc680cfe4c7..ae391bfd8a7b 100644 --- a/src/nix/meson.options +++ b/src/nix/meson.options @@ -7,3 +7,10 @@ option( value : 'etc/profile.d', description : 'the path to install shell profile files', ) + +option( + 'mimalloc', + type : 'feature', + value : 'auto', + description : 'Link against mimalloc to override the default memory allocator', +) diff --git a/src/nix/package.nix b/src/nix/package.nix index 8195e6c6ff5a..68d1e379ab47 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -8,9 +8,16 @@ nix-main, nix-cmd, + mimalloc, + # Configuration Options version, + + # Whether to link against mimalloc for malloc override. + # Significantly improves evaluation performance on allocation-heavy + # workloads (~10-15% on large evaluations). + withMimalloc ? !stdenv.hostPlatform.isWindows, }: let @@ -69,9 +76,11 @@ mkMesonExecutable (finalAttrs: { nix-expr nix-main nix-cmd - ]; + ] + ++ lib.optional withMimalloc mimalloc; mesonFlags = [ + (lib.mesonEnable "mimalloc" withMimalloc) ]; postInstall = lib.optionalString stdenv.hostPlatform.isStatic '' From a760af86b3a42aa5ac9d9002929107fe357bf128 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 3 Apr 2026 00:21:31 +0300 Subject: [PATCH 210/555] derivation-builder: Don't use copyFile for FOD output copying, put the output in a temporary directory in the store Puts the temporary FOD output copies in a temporary directory inside the store instead of the (for Linux sandboxed builds) chroot. This prevents file overwrite due to symlink following that std::filesystem::copy_file does. Also applies the same output copying approach for impure derivations that don't have network sandboxing and thus are subject to FD smuggling. Fixes GHSA-g3g9-5vj6-r3gj. --- src/libstore/include/nix/store/local-store.hh | 2 ++ src/libstore/unix/build/derivation-builder.cc | 36 ++++++++++++++----- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 63a1da67d8bf..bf3437e95771 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -512,6 +512,8 @@ private: friend struct PathSubstitutionGoal; friend struct DerivationGoal; + /* Only used for createTempDirInStore. */ + friend class DerivationBuilderImpl; }; } // namespace nix diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 8f6343e0f4ea..8288a4a31851 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1597,6 +1597,13 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() assert(output && scratchPath); auto actualPath = realPathInHost(store.printStorePath(*scratchPath)); + /* An optional file descriptor of a directory used for intermediate + operations. */ + AutoCloseFD tempDirFd; + /* RAII cleanup of a temporary directory inside the store that is used + for intermediate operations. */ + AutoDelete delTempDir; + auto finish = [&](StorePath finalStorePath) { /* Store the final path */ finalOutputs.insert_or_assign(outputName, finalStorePath); @@ -1744,6 +1751,25 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() return newInfo0; }; + auto moveOutputToTempDir = [&]() -> void { + std::filesystem::path tempDir; + std::tie(tempDir, tempDirFd) = store.createTempDirInStore(); + delTempDir = AutoDelete(tempDir); + + auto tmpOutput = tempDir / "x"; + + /* Serialise and create a fresh copy of the output to break + any stale writable file descriptors. Copy through the + serialisation/deserialisation. TODO: Use copyRecursive here and + make use of reflinking. */ + auto source = sinkToSource([&](Sink & nextSink) { dumpPath(actualPath, nextSink); }); + restorePath(tmpOutput, *source, store.config->getLocalSettings().fsyncStorePaths); + /* This makes it slightly harder to make sense of the control flow. The rule + of thumb is that actualPath points to the current location of the stuff + that we'll end up registering. */ + actualPath = std::move(tmpOutput); + }; + ValidPathInfo newInfo = std::visit( overloaded{ @@ -1771,14 +1797,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() [&](const DerivationOutput::CAFixed & dof) { auto & wanted = dof.ca.hash; - - // Replace the output by a fresh copy of itself to make sure - // that there's no stale file descriptor pointing to it - std::filesystem::path tmpOutput = actualPath.native() + ".tmp"; - copyFile(actualPath, tmpOutput, true); - - std::filesystem::rename(tmpOutput, actualPath); - + moveOutputToTempDir(); return newInfoFromCA( DerivationOutput::CAFloating{ .method = dof.ca.method, @@ -1795,6 +1814,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() }, [&](const DerivationOutput::Impure & doi) { + moveOutputToTempDir(); return newInfoFromCA( DerivationOutput::CAFloating{ .method = doi.method, From d43bb18ac272ba25c3040bc26fdbb5f548a6839e Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:51:17 +1000 Subject: [PATCH 211/555] packaging/dependencies: bump mimalloc to 3.1.6 3.1.5 seems to be broken with freebsd cross https://hydra.nixos.org/build/325420458/nixlog/1 --- packaging/dependencies.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 5338f70e57b3..9903a93ba9ed 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -16,6 +16,16 @@ in scope: { inherit stdenv; + mimalloc = pkgs.mimalloc.overrideAttrs rec { + version = "3.1.6"; + src = pkgs.fetchFromGitHub { + owner = "microsoft"; + repo = "mimalloc"; + tag = "v${version}"; + hash = "sha256-7zG0Sqloanz/b+fkJ4wzO86uBmtf9fdYNAT9ixLouyY="; + }; + }; + boehmgc = (pkgs.boehmgc.override { enableLargeConfig = true; From 323b8cfcb4b2a30da480570ded8f6eee1c18f8dd Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Sat, 4 Apr 2026 02:02:38 -0700 Subject: [PATCH 212/555] doc: fix typo: A -> The --- doc/manual/source/development/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index dd965862a34c..3b80e8a266b0 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -80,7 +80,7 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. -A environment variables that Google Test accepts are also worth knowing: +The environment variables that Google Test accepts are also worth knowing: 1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) From 79e2d2f4943e42661062e3bd2acc81ca2a664378 Mon Sep 17 00:00:00 2001 From: Antonio Nuno Monteiro Date: Sun, 5 Apr 2026 23:02:22 -0700 Subject: [PATCH 213/555] libmain/progress-bar: invalidate redraw cache when clearing the bar --- src/libmain/progress-bar.cc | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 71c7789de254..9d6c8f157576 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -131,7 +131,7 @@ class ProgressBar : public Logger auto state(state_.lock()); if (state->active) { state->active = false; - writeToStderr("\r\e[K"); + clearProgressDisplay(); updateCV.notify_one(); quitCV.notify_one(); } @@ -150,7 +150,7 @@ class ProgressBar : public Logger } if (state->active) - writeToStderr("\r\e[K"); + clearProgressDisplay(); } void resume() override @@ -164,7 +164,7 @@ class ProgressBar : public Logger } if (state->suspensions == 0) { if (state->active) - writeToStderr("\r\e[K"); + clearProgressDisplay(); state->haveUpdate = true; updateCV.notify_one(); } @@ -196,6 +196,7 @@ class ProgressBar : public Logger void log(State & state, Verbosity lvl, std::string_view s) { if (state.active) { + invalidateRedrawCache(); writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); draw(state); } else { @@ -408,6 +409,17 @@ class ProgressBar : public Logger } } + void invalidateRedrawCache() + { + *lastOutput_.lock() = ""; + } + + void clearProgressDisplay() + { + invalidateRedrawCache(); + writeToStderr("\r\e[K"); + } + std::chrono::milliseconds draw(State & state) { auto nextWakeup = std::chrono::milliseconds::max(); @@ -647,6 +659,7 @@ class ProgressBar : public Logger { auto state(state_.lock()); if (state->active) { + invalidateRedrawCache(); std::cerr << "\r\e[K"; Logger::writeToStdout(s); draw(*state); @@ -660,6 +673,7 @@ class ProgressBar : public Logger auto state(state_.lock()); if (!state->active) return {}; + invalidateRedrawCache(); std::cerr << fmt("\r\e[K%s ", msg); auto s = trim(readLine(getStandardInput(), true)); if (s.size() != 1) From 44017ca497c8b44d5dac179f5afc63e91fe45ed6 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 5 Apr 2026 16:39:58 +0300 Subject: [PATCH 214/555] libstore: Use landlock with LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET for new enough kernels This partially fixes the issue with cooperating processes being able to communicate via abstract sockets. The fix is partial, because processes outside the landlock domain of the sandboxed process can still connect to a socket created by the FOD. There's no equivalent way of restricting inbound connections. This closes the gap when there's no cooperating process on the host (i.e. 2 separate FODs). >= 6.12 kernel is widespread enough (NixOS 25.11 ships it by default) that we have no reason not to apply this hardening, even though it's incomplete. ca-fd-leak test exercises this exact code path and now the smuggling process fails with (on new enough kernels that have landlock support enabled): vm-test-run-ca-fd-leak> machine # sandbox setup: applied landlock sandboxing vm-test-run-ca-fd-leak> machine # building '/nix/store/s7brgi6pdr5f3n8yqlgmdlz8blb89njc-smuggled.drv'... vm-test-run-ca-fd-leak> machine # building derivation '/nix/store/s7brgi6pdr5f3n8yqlgmdlz8blb89njc-smuggled.drv': woken up vm-test-run-ca-fd-leak> machine # connect: Operation not permitted vm-test-run-ca-fd-leak> machine # sendmsg: Socket not connected --- src/libstore/meson.build | 5 + .../unix/build/linux-derivation-builder.cc | 101 ++++++++++++++++++ tests/nixos/ca-fd-leak/default.nix | 8 +- 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 445798544f2d..753c786876ab 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -79,6 +79,11 @@ foreach funcspec : check_funcs configdata_priv.set(define_name, define_value) endforeach +if host_machine.system() == 'linux' + has_landlock = cxx.has_header('linux/landlock.h') + configdata_priv.set('HAVE_LANDLOCK', has_landlock.to_int()) +endif + has_acl_support = cxx.has_header('sys/xattr.h') \ and cxx.has_function('llistxattr') \ and cxx.has_function('lremovexattr') diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 9cfd3cbcd27c..c71d23e15e02 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -1,5 +1,7 @@ #ifdef __linux__ +# include "store-config-private.hh" + # include "nix/store/globals.hh" # include "nix/store/personality.hh" # include "nix/store/filetransfer.hh" @@ -11,6 +13,8 @@ # include # include +# include + # include # include # include @@ -19,11 +23,16 @@ # include # include # include +# include # if HAVE_SECCOMP # include # endif +# if HAVE_LANDLOCK +# include +# endif + # define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) namespace nix { @@ -129,6 +138,77 @@ static void setupSeccomp(const LocalSettings & localSettings) # endif } +# if HAVE_LANDLOCK && defined(LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET) + +# define DO_LANDLOCK 1 + +/* We are using LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET on best-effort basis. There are no glibc wrappers for now. */ + +static int landlockCreateRuleset(const ::landlock_ruleset_attr * attr, std::size_t size, std::uint32_t flags) +{ + return ::syscall(__NR_landlock_create_ruleset, attr, size, flags); +} + +static int landlockRestrictSelf(Descriptor rulesetFd, std::uint32_t flags) +{ + return ::syscall(__NR_landlock_restrict_self, rulesetFd, flags); +} + +static int getLandlockAbiVersion() +{ + int abiVersion = landlockCreateRuleset(nullptr, 0, LANDLOCK_CREATE_RULESET_VERSION); + return abiVersion; +} + +static void setupLandlock() +{ + bool landlockSupportsScopeAbstractUnixSocket = []() { + int abiVersion = getLandlockAbiVersion(); + if (abiVersion >= 6) + /* All good, we can use LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET. See + https://docs.kernel.org/userspace-api/landlock.html#abstract-unix-socket-abi-6 */ + return true; + + if (abiVersion == -1) { + debug("landlock is not available"); + return false; + } + + debug("landlock version %d does not support LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET", abiVersion); + return false; + }(); + + /* Bail out early if landlock is not enabled or LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET wouldn't work. + TODO: Consider adding more landlock rules for filesystem access as defense-in-depth on top. */ + if (!landlockSupportsScopeAbstractUnixSocket) + return; + + ::landlock_ruleset_attr attr = { + /* This prevents multiple FODs from communicating with each other + via abstract sockets. Note that cooperating processes outside the + sandbox can still connect to an abstract socket created by the FOD. To + mitigate that issue entirely we'd still need network namespaces. */ + .scoped = LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET, + }; + + /* This better not fail - if the kernel reports a new enough ABI version we + should treat any errors as fatal from now on. */ + AutoCloseFD rulesetFd = landlockCreateRuleset(&attr, sizeof(attr), 0); + if (!rulesetFd) + throw SysError("failed to create a landlock ruleset"); + + if (landlockRestrictSelf(rulesetFd.get(), 0) == -1) + throw SysError("failed to apply landlock"); + + debug("applied landlock sandboxing"); +} + +# else + +# define DO_LANDLOCK 0 + +# endif + static void doBind(const std::filesystem::path & source, const std::filesystem::path & target, bool optional = false) { debug("bind mounting %1% to %2%", PathFmt(source), PathFmt(target)); @@ -169,8 +249,27 @@ struct LinuxDerivationBuilder : virtual DerivationBuilderImpl { auto & localSettings = store.config->getLocalSettings(); + /* Set the NO_NEW_PRIVS before doing seccomp/landlock setup. + landlock_restrict_self requires either NO_NEW_PRIVS or CAP_SYS_ADMIN. + With user namespaces we do get CAP_SYS_ADMIN. */ + if (!localSettings.allowNewPrivileges) + if (::prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) == -1) + throw SysError("failed to set PR_SET_NO_NEW_PRIVS"); + setupSeccomp(localSettings); +# if DO_LANDLOCK + try { + setupLandlock(); + } catch (SysError & e) { + if (e.errNo != EPERM) + throw; + /* If allowNewPrivileges is true and we don't have CAP_SYS_ADMIN + this code path might be hit. */ + warn("setting up landlock: %s", e.message()); + } +# endif + linux::setPersonality({ .system = drv.platform, .impersonateLinux26 = localSettings.impersonateLinux26, @@ -765,4 +864,6 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu } // namespace nix +# undef DO_LANDLOCK + #endif diff --git a/tests/nixos/ca-fd-leak/default.nix b/tests/nixos/ca-fd-leak/default.nix index 902aacdc650f..dc944290f7e2 100644 --- a/tests/nixos/ca-fd-leak/default.nix +++ b/tests/nixos/ca-fd-leak/default.nix @@ -78,7 +78,7 @@ in # Build the smuggled derivation. # This will connect to the smuggler server and send it the file descriptor - machine.succeed(r""" + sender_output = machine.succeed(r""" nix-build -E ' builtins.derivation { name = "smuggled"; @@ -89,9 +89,13 @@ in outputHash = builtins.hashString "sha256" "hello, world\n"; builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; args = [ "-c" "echo \"hello, world\" > $out; ''${${sender}} ${socketName}" ]; - }' + }' 2>&1 """.strip()) + # Landlock's LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET prevents a sandboxed process + # from connecting to an abstract socket created in an unrelated landlock domain. + # There's no such flag for preventing inbound connections. + assert "connect: Operation not permitted" in sender_output # Tell the smuggler server that we're done machine.execute("echo done | ${pkgs.socat}/bin/socat - ABSTRACT-CONNECT:${socketName}") From f4bde1f72d3f7172366d485891b67e56eff3ed40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 8 Apr 2026 10:54:00 +0200 Subject: [PATCH 215/555] Split release artifacts into a dedicated Hydra jobset Cutting a release currently blocks on the full hydraJobs evaluation (~900 builds including sanitizers, clang-tidy, static, NixOS VM and installer tests), even though upload-release only consumes ~25 of them. On recent maintenance evals the long CI tail and darwin queue depth pushed the wait into the multi-day range while the actual artifacts were ready within an hour. Hydra hard-codes flake jobsets to outputs.hydraJobs, so the subset is exposed through a legacy jobset expression that re-enters the flake via builtins.getFlake on the locked GitHub ref. Going through the ref rather than the checked-out store path preserves rev/lastModified and thus the version suffix, keeping derivations bit-identical to the flake jobset so both share builds through the binary cache. A release aggregate job provides a single gating signal for upload-release. The release process now creates a release-$VERSION jobset alongside maintenance-$VERSION and waits on that instead of the full matrix. Requires adding https://releases.nixos.org/ to allowed-uris on hydra.nixos.org, since the locked nixpkgs input is a tarball from there and legacy jobsets run under restrict-eval. --- maintainers/release-process.md | 49 ++++++++++++---------- packaging/release-jobs.nix | 76 ++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 23 deletions(-) create mode 100644 packaging/release-jobs.nix diff --git a/maintainers/release-process.md b/maintainers/release-process.md index f8b6b6bec572..19e5e050534e 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -81,27 +81,30 @@ release: $ git push --set-upstream origin $VERSION-maintenance ``` -* Create a jobset for the release branch on Hydra as follows: - - * Go to the jobset of the previous release - (e.g. https://hydra.nixos.org/jobset/nix/maintenance-2.11). - - * Select `Actions -> Clone this jobset`. - - * Set identifier to `maintenance-$VERSION`. - - * Set description to `$VERSION release branch`. - - * Set flake URL to `github:NixOS/nix/$VERSION-maintenance`. - - * Hit `Create jobset`. - -* Wait for the new jobset to evaluate and build. If impatient, go to - the evaluation and select `Actions -> Bump builds to front of - queue`. - -* When the jobset evaluation has succeeded building, take note of the - evaluation ID (e.g. `1780832` in +* Create two jobsets for the release branch on Hydra: + + `maintenance-$VERSION` runs the full `hydraJobs` CI matrix. + `release-$VERSION` builds only the artifacts consumed by + `upload-release`, so a release can be cut without waiting on the full + matrix. + + * Clone the previous `maintenance-*` jobset, set identifier + `maintenance-$VERSION`, description `$VERSION release branch`, flake + URL `github:NixOS/nix/$VERSION-maintenance`. + + * Clone the previous `release-*` jobset (or create a new **legacy** + jobset), set identifier `release-$VERSION`, description `$VERSION + release artifacts`, Nix expression `packaging/release-jobs.nix` in + input `src`, and add input `src` of type *Git checkout* pointing at + `https://github.com/NixOS/nix $VERSION-maintenance`. + +* Wait for the `release-$VERSION` jobset to evaluate and build. If + impatient, go to the evaluation and select `Actions -> Bump builds to + front of queue`. The aggregate job `release` turns green once every + required artifact is available. + +* When the release jobset evaluation has succeeded building, take note of + the evaluation ID (e.g. `1780832` in `https://hydra.nixos.org/eval/1780832`). * Tag the release: @@ -174,8 +177,8 @@ release: $ git push ``` -* Wait for the desired evaluation of the maintenance jobset to finish - building. +* Wait for the desired evaluation of the `release-$VERSION` jobset to + finish building (the `release` aggregate job is the gating signal). * Tag the release diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix new file mode 100644 index 000000000000..3da8c80de315 --- /dev/null +++ b/packaging/release-jobs.nix @@ -0,0 +1,76 @@ +# Hydra jobset containing only the artifacts consumed by +# `maintainers/upload-release.{pl,py}`, so a release can be cut without +# waiting on the full `hydraJobs` CI matrix. +# +# Evaluated as a legacy (non-flake) jobset because Hydra hard-codes flake +# jobsets to `outputs.hydraJobs`; we re-enter the flake via +# `builtins.getFlake` so derivations stay identical to the flake jobset +# and share builds through the binary cache. +# +# Hydra jobset configuration: +# Type: Legacy +# Nix expression: packaging/release-jobs.nix in input `src` +# Inputs: +# src (Git checkout) https://github.com/NixOS/nix +{ + src ? { + outPath = ./..; + }, +}: +let + # Fetch by GitHub ref rather than the bare store path Hydra hands us, + # so `rev`/`lastModified` (and thus the version suffix) match the flake + # jobset and derivations are shared. + flake = builtins.getFlake ( + if src ? rev then + "github:NixOS/nix/${src.rev}" + else + # Local evaluation / testing. + builtins.unsafeDiscardStringContext (toString src) + ); + inherit (flake) hydraJobs; + inherit (flake.inputs.nixpkgs) lib; + + jobs = { + # `nix-everything` per system: provides the store paths for + # `fallback-paths.nix` and (on x86_64-linux) the rendered manual via + # its `doc` output. + build.nix-everything = hydraJobs.build.nix-everything; + buildCross.nix-everything.riscv64-unknown-linux-gnu = + hydraJobs.buildCross.nix-everything.riscv64-unknown-linux-gnu; + + inherit (hydraJobs) + manual + binaryTarball + binaryTarballCross + installerScript + installerScriptForGHA + dockerImage + ; + + # Aggregate gating job: green ⇒ every artifact the upload script + # needs is available. `upload-release` can wait on this single job + # instead of the whole evaluation. Constituents are referenced by + # job *name* so that an evaluation failure in one of them does not + # take down the aggregate's own evaluation. + release = flake.inputs.nixpkgs.legacyPackages.x86_64-linux.releaseTools.aggregate { + name = "nix-release-${flake.packages.x86_64-linux.nix-everything.version}"; + meta.description = "Artifacts required for a Nix release"; + constituents = + let + collectJobNames = + prefix: x: + if lib.isDerivation x then + [ prefix ] + else if lib.isAttrs x then + lib.concatLists ( + lib.mapAttrsToList (n: collectJobNames (if prefix == "" then n else "${prefix}.${n}")) x + ) + else + [ ]; + in + collectJobNames "" (builtins.removeAttrs jobs [ "release" ]); + }; + }; +in +jobs From d0cbf284efe80241c78fe3850b0745b1984b8fb7 Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 8 Apr 2026 21:45:41 +0000 Subject: [PATCH 216/555] tests/functional: cover `nix-store --restore` with bare relative destination (#15645) Regression test for `restorePath` when the destination has no directory component. --- tests/functional/nars.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index 68bd191a2543..ecae66c14151 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -42,6 +42,11 @@ expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | gre mkdir -p "$TEST_ROOT/out2" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" +# Check that a bare relative destination (no directory component) works. +rm -rf "$TEST_ROOT/out" +(cd "$TEST_ROOT" && nix-store --restore out < tmp.nar) +[[ -f "$TEST_ROOT/out" ]] + # The same, but for a symlink. ln -sfn foo "$TEST_ROOT/symlink" nix-store --dump "$TEST_ROOT/symlink" > "$TEST_ROOT/tmp.nar" @@ -62,6 +67,12 @@ expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | gre mkdir -p "$TEST_ROOT/out2" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" +# Likewise for a bare relative destination. +rm -rf "$TEST_ROOT/out" +(cd "$TEST_ROOT" && nix-store --restore out < tmp.nar) +[[ -L "$TEST_ROOT/out" ]] +[[ $(readlink "$TEST_ROOT/out") = foo ]] + # Check whether restoring and dumping a NAR that contains case # collisions is round-tripping, even on a case-insensitive system. rm -rf "$TEST_ROOT/case" @@ -77,6 +88,11 @@ nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > "$TEST_ROOT/case.nar" cmp case.nar "$TEST_ROOT/case.nar" [ "$(nix-hash "${opts[@]}" --type sha256 "$TEST_ROOT/case")" = "$(nix-hash --flat --type sha256 case.nar)" ] +# Likewise for a bare relative destination. +rm -rf "$TEST_ROOT/case" +(cd "$TEST_ROOT" && nix-store "${opts[@]}" --restore case) < case.nar +[[ -e "$TEST_ROOT/case/xt_CONNMARK.h" ]] + # Check whether we detect true collisions (e.g. those remaining after # removal of the suffix). touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" From 96752a0f640114c4cc8a089148af48d3e40bdb27 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 8 Apr 2026 23:59:17 -0400 Subject: [PATCH 217/555] Fix #15003 (#15637) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix some missing `std::less<>` with `std::map` We need a `clang-tidy` for this, or similar. * libstore: instantiate `DerivationOptions` JSON serializers for both input types The `to_json`/`from_json` definitions in `derivation-options.cc` were only specialized for `DerivationOptions`, even though `derivation-options.hh` declared `JSON_IMPL` for both `DerivationOptions` and `DerivationOptions` (and likewise for the nested `OutputChecks`). The `StorePath` declarations were dangling — referencing them caused link errors. Refactor the per-type specializations into shared template helpers (`derivationOptionsFromJson` / `derivationOptionsToJson` / `outputChecksFromJson` / `outputChecksToJson`) and instantiate both `SingleDerivedPath` and `StorePath` `adl_serializer` specializations as thin wrappers. Add JSON characterization tests for `DerivationOptions` mirroring the existing `SingleDerivedPath` tests. The macro now takes the input type as a parameter. As noted in the comment near the new test invocations, the IA test data files are reused as-is because `DrvRef` and `DrvRef` (when only the `Opaque` case is used) JSON-encode identically. * libstore: handle CA placeholder + subpath in `exportReferencesGraph` Fixes #15003. When parsing the `exportReferencesGraph` derivation attribute, `parseSingleDerivedPath` looked up the path string in the placeholder map by exact match. For CA derivations, `dep` renders as a bare `/HASH` placeholder; if the user appended a subpath (e.g. `"${dep}/foo"`), the resulting `/HASH/foo` failed the exact-match lookup. (The fallback to `StoreDirConfig::toStorePath` then threw because placeholders don't start with `/nix/store/`.) This regressed `contentAddressedByDefault = true;` Nixpkgs usage in Nix 2.33, which as those Nixpkgs drvs use `${closureInfo}/...` style references in `exportReferencesGraph`. Fix the issue by extracting the leading path component (everything before the second `/`) before the lookup, mirroring how `toStorePath` strips subpaths from real store paths. Apply the same treatment to `parseRef` via a shared `findPlaceholder` helper. Add `TryResolveTest::exportReferencesGraphPlaceholderSubpath` (and a structured-attrs counterpart) covering both the parse step and a subsequent `Derivation::tryResolve` to confirm the placeholder gets substituted with the concrete output path end-to-end. --- .../export-ref-subpath-before-options.json | 31 ++++ .../export-ref-subpath-before.json | 27 +++ .../export-ref-subpath-buildTrace.json | 9 + .../export-ref-subpath-resolved-options.json | 28 ++++ .../export-ref-subpath-resolved.json | 19 +++ .../export-ref-subpath-sa-before-options.json | 23 +++ .../export-ref-subpath-sa-before.json | 34 ++++ .../export-ref-subpath-sa-buildTrace.json | 9 + ...xport-ref-subpath-sa-resolved-options.json | 20 +++ .../export-ref-subpath-sa-resolved.json | 26 +++ .../derivation-advanced-attrs.cc | 155 +++++++++++++++--- src/libstore-tests/derivations.cc | 114 +++++++++++++ src/libstore/build/derivation-check.cc | 3 +- src/libstore/derivation-options.cc | 120 ++++++++++---- .../include/nix/store/derivation-options.hh | 6 +- 15 files changed, 567 insertions(+), 57 deletions(-) create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json new file mode 100644 index 000000000000..44006012fb8c --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json @@ -0,0 +1,31 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + } + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json new file mode 100644 index 000000000000..0706e1b64aa8 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json @@ -0,0 +1,27 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "exportReferencesGraph": "refs /19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "export-ref-subpath", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json new file mode 100644 index 000000000000..3f15525837fa --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json @@ -0,0 +1,9 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] +] diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json new file mode 100644 index 000000000000..2bfe56fc8d44 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json @@ -0,0 +1,28 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json new file mode 100644 index 000000000000..453efae619de --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json @@ -0,0 +1,19 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "exportReferencesGraph": "refs /nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ], + "name": "export-ref-subpath", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json new file mode 100644 index 000000000000..7bc4cad36c51 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json @@ -0,0 +1,23 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + } + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "perOutput": {} + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json new file mode 100644 index 000000000000..3e92638f9d6f --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json @@ -0,0 +1,34 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "__json": "{\"exportReferencesGraph\":{\"refs\":[\"/19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo\"]}}" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "export-ref-subpath-sa", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "exportReferencesGraph": { + "refs": [ + "/19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo" + ] + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json new file mode 100644 index 000000000000..3f15525837fa --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json @@ -0,0 +1,9 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] +] diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json new file mode 100644 index 000000000000..b3970f4a3788 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json @@ -0,0 +1,20 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "perOutput": {} + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json new file mode 100644 index 000000000000..d9befe3c96e7 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json @@ -0,0 +1,26 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "__json": "{\"exportReferencesGraph\":{\"refs\":[\"/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo\"]}}" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ], + "name": "export-ref-subpath-sa", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "exportReferencesGraph": { + "refs": [ + "/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo" + ] + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 16988613c633..aa24943a72ea 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -307,7 +307,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) }; DerivationOptions advancedAttributes_structuredAttrs_defaults = { - .outputChecks = std::map::OutputChecks>{}, + .outputChecks = std::map::OutputChecks, std::less<>>{}, .unsafeDiscardReferences = {}, .passAsFile = {}, .exportReferencesGraph = {}, @@ -352,7 +352,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) { DerivationOptions expected = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"dev", DerivationOptions::OutputChecks{ .maxSize = 789, @@ -384,7 +384,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) { // Delete all keys but "dev" in options.outputChecks auto * outputChecksMapP = - std::get_if::OutputChecks>>( + std::get_if::OutputChecks, std::less<>>>( &options.outputChecks); ASSERT_TRUE(outputChecksMapP); auto & outputChecksMap = *outputChecksMapP; @@ -405,7 +405,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) DerivationOptions advancedAttributes_structuredAttrs_ia = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"out", DerivationOptions::OutputChecks{ .allowedReferences = std::set>{pathFoo}, @@ -447,7 +447,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) DerivationOptions advancedAttributes_structuredAttrs_ca = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"out", DerivationOptions::OutputChecks{ .allowedReferences = std::set>{placeholderFoo}, @@ -489,24 +489,137 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) {"rainbow", "uid-range", "ca-derivations"}); }; -#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2) \ - TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json) \ - { \ - nix::readJsonTest>( \ - *this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ - } \ - TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json) \ - { \ - nix::readJsonTest>( \ - *this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ +#define TEST_JSON_OPTIONS(FIXUTURE, INPUT, VAR, VAR2) \ + TEST_F(FIXUTURE, DerivationOptions_##INPUT##_##VAR##_from_json) \ + { \ + nix::readJsonTest>(*this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ + } \ + TEST_F(FIXUTURE, DerivationOptions_##INPUT##_##VAR##_to_json) \ + { \ + nix::readJsonTest>(*this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ } -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, all_set, ia) -TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, all_set, ca) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_defaults, structuredAttrs_defaults) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ia) -TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ca) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, defaults, defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, all_set, ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, SingleDerivedPath, all_set, ca) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_defaults, structuredAttrs_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_all_set, structuredAttrs_ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_all_set, structuredAttrs_ca) + +/** + * `DerivationOptions` versions of the IA fixtures, used to + * exercise the `StorePath` JSON serializers. The IA test data files are + * reused as-is (see comment near the test invocations). + */ +static const StorePath spFoo{"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}, + spFooDev{"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}, spBar{"r5cff30838majxk5mp3ip2diffi8vpaj-bar"}, + spBarDev{"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}, spBarDrv{"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}; + +static const DerivationOptions advancedAttributes_sp_defaults = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, +}; + +static const DerivationOptions advancedAttributes_sp_all_set = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + .allowedReferences = std::set>{spFoo}, + .disallowedReferences = std::set>{spBar, OutputName{"dev"}}, + .allowedRequisites = std::set>{spFooDev, OutputName{"bin"}}, + .disallowedRequisites = std::set>{spBarDev}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph{ + {"refs1", {spFoo}}, + {"refs2", {spBarDrv}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + +static const DerivationOptions advancedAttributes_sp_structuredAttrs_defaults = { + .outputChecks = std::map::OutputChecks, std::less<>>{}, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, +}; + +static const DerivationOptions advancedAttributes_sp_structuredAttrs_all_set = { + .outputChecks = + std::map::OutputChecks, std::less<>>{ + {"out", + DerivationOptions::OutputChecks{ + .allowedReferences = std::set>{spFoo}, + .allowedRequisites = std::set>{spFooDev, OutputName{"bin"}}, + }}, + {"bin", + DerivationOptions::OutputChecks{ + .disallowedReferences = std::set>{spBar, OutputName{"dev"}}, + .disallowedRequisites = std::set>{spBarDev}, + }}, + {"dev", + DerivationOptions::OutputChecks{ + .maxSize = 789, + .maxClosureSize = 5909, + }}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = + { + {"refs1", {spFoo}}, + {"refs2", {spBarDrv}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + +/** + * Same JSON characterization tests, but for `DerivationOptions`. + * + * Since `DrvRef` and `DrvRef` (when only + * the `Opaque` case is used) JSON-encode identically, the IA test data + * files can be reused as-is. + */ +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, defaults, sp_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, all_set, sp_all_set) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, structuredAttrs_defaults, sp_structuredAttrs_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, structuredAttrs_all_set, sp_structuredAttrs_all_set) #undef TEST_JSON_OPTIONS diff --git a/src/libstore-tests/derivations.cc b/src/libstore-tests/derivations.cc index a898b8ee9c27..60b86f571205 100644 --- a/src/libstore-tests/derivations.cc +++ b/src/libstore-tests/derivations.cc @@ -2,6 +2,7 @@ #include #include "nix/store/derivations.hh" +#include "nix/store/derivation-options.hh" #include "nix/store/downstream-placeholder.hh" #include "nix/store/parsed-derivations.hh" #include "nix/store/tests/libstore.hh" @@ -72,6 +73,11 @@ class TryResolveTest : public LibStoreTest, public CharacterizationTest }; } + /** + * Helper for the `exportReferencesGraph` + placeholder subpath tests. + */ + void exportRefGraphSubpathTest(std::string_view stem, const Derivation & drv, const StructuredAttrs * parsed); + /** * Checkpoint before/buildTrace/after and assert the resolved derivation * matches expected. @@ -248,4 +254,112 @@ TEST_F(TryResolveTest, resolutionFailure) EXPECT_FALSE(resolved); } +/** + * Test that `derivationOptionsFromStructuredAttrs` can parse + * `exportReferencesGraph` entries that reference a CA derivation output + * with a subpath appended (e.g. `${dep}/foo`), and that the parsed + * options resolve correctly. + * + * Regression test for #15003: the placeholder + subpath was not found + * in the placeholder map (exact-match only), and the fallback to + * `toStorePath()` failed because placeholders don't start with + * `/nix/store/`. + */ +void TryResolveTest::exportRefGraphSubpathTest( + std::string_view stem, const Derivation & drv, const StructuredAttrs * parsed) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + StorePath depOutPath{"f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out"}; + + nix::checkpointJson(*this, std::string{stem} + "-before", drv); + + auto options = derivationOptionsFromStructuredAttrs(*store, drv.inputDrvs, drv.env, parsed, true); + + nix::checkpointJson(*this, std::string{stem} + "-before-options", options); + + SingleDerivedPath expectedPath = SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(depDrvPath), + .output = "out", + }; + + ASSERT_EQ(options.exportReferencesGraph.size(), 1); + auto it = options.exportReferencesGraph.find("refs"); + ASSERT_NE(it, options.exportReferencesGraph.end()); + ASSERT_EQ(it->second.size(), 1); + EXPECT_EQ(*it->second.begin(), expectedPath); + + // Also test that resolution works + BuildTrace buildTrace{.dict{ + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(depDrvPath), + .output = "out", + }, + depOutPath, + }, + }}; + + nix::checkpointJson(*this, std::string{stem} + "-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + ASSERT_TRUE(resolved); + + nix::checkpointJson(*this, std::string{stem} + "-resolved", *resolved); + + // Re-parse options from the resolved derivation, where placeholders + // have been substituted with concrete store paths. + auto resolvedOptions = derivationOptionsFromStructuredAttrs( + *store, + /* inputDrvs */ {}, + resolved->env, + resolved->structuredAttrs ? &*resolved->structuredAttrs : nullptr, + true); + + nix::checkpointJson(*this, std::string{stem} + "-resolved-options", resolvedOptions); + + EXPECT_EQ( + resolvedOptions.exportReferencesGraph, + (decltype(resolvedOptions.exportReferencesGraph){ + {"refs", std::set{SingleDerivedPath::Opaque{depOutPath}}}})); +} + +TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); + + Derivation drv; + drv.name = "export-ref-subpath"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.env = { + {"exportReferencesGraph", "refs " + placeholder + "/foo"}, + }; + + exportRefGraphSubpathTest("export-ref-subpath", drv, nullptr); +} + +TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath_structuredAttrs) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); + + Derivation drv; + drv.name = "export-ref-subpath-sa"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.structuredAttrs = StructuredAttrs{{ + {"exportReferencesGraph", nlohmann::json::object({{"refs", nlohmann::json::array({placeholder + "/foo"})}})}, + }}; + drv.env = { + {std::string{StructuredAttrs::envVarName}, nlohmann::json(drv.structuredAttrs->structuredAttrs).dump()}, + }; + + exportRefGraphSubpathTest("export-ref-subpath-sa", drv, &*drv.structuredAttrs); +} + } // namespace nix diff --git a/src/libstore/build/derivation-check.cc b/src/libstore/build/derivation-check.cc index c422897e624e..dffbbc3beb26 100644 --- a/src/libstore/build/derivation-check.cc +++ b/src/libstore/build/derivation-check.cc @@ -186,7 +186,8 @@ void checkOutputs( std::visit( overloaded{ [&](const DerivationOptions::OutputChecks & checks) { applyChecks(checks); }, - [&](const std::map::OutputChecks> & checksPerOutput) { + [&](const std::map::OutputChecks, std::less<>> & + checksPerOutput) { if (auto outputChecks = get(checksPerOutput, outputName)) applyChecks(*outputChecks); diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 5208440c7d94..86543bc3632c 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -73,7 +73,8 @@ template using OutputChecks = DerivationOptions::OutputChecks; template -using OutputChecksVariant = std::variant, std::map>>; +using OutputChecksVariant = + std::variant, std::map, std::less<>>>; DerivationOptions derivationOptionsFromStructuredAttrs( const StoreDirConfig & store, @@ -124,7 +125,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( { DerivationOptions defaults = {}; - std::map placeholders; + std::map> placeholders; if (mockXpSettings.isEnabled(Xp::CaDerivations)) { /* Initialize placeholder map from inputDrvs */ auto initPlaceholders = [&](this const auto & initPlaceholders, @@ -156,16 +157,27 @@ DerivationOptions derivationOptionsFromStructuredAttrs( } } + /* Extract the placeholder key from a path that may have a subpath + appended (e.g. `/HASH/foo` → `/HASH`), mirroring how + `StoreDirConfig::toStorePath` strips subpaths from store paths. */ + auto findPlaceholder = [&](std::string_view pathS) -> const SingleDerivedPath::Built * { + auto slash = pathS.find('/', 1); + auto key = pathS.substr(0, slash); + if (auto it = placeholders.find(key); it != placeholders.end()) + return &it->second; + return nullptr; + }; + auto parseSingleDerivedPath = [&](const std::string & pathS) -> SingleDerivedPath { - if (auto it = placeholders.find(pathS); it != placeholders.end()) - return it->second; + if (auto * built = findPlaceholder(pathS)) + return *built; else return SingleDerivedPath::Opaque{store.toStorePath(pathS).first}; }; auto parseRef = [&](const std::string & pathS) -> DrvRef { - if (auto it = placeholders.find(pathS); it != placeholders.end()) - return it->second; + if (auto * built = findPlaceholder(pathS)) + return *built; if (store.isStorePath(pathS)) return SingleDerivedPath::Opaque{store.toStorePath(pathS).first}; else @@ -206,7 +218,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( if (parsed) { auto & structuredAttrs = parsed->structuredAttrs; - std::map> res; + std::map, std::less<>> res; if (auto * outputChecks = get(structuredAttrs, "outputChecks")) { for (auto & [outputName, output_] : getObject(*outputChecks)) { auto & output = getObject(output_); @@ -265,7 +277,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( }(), .unsafeDiscardReferences = [&] { - std::map res; + std::map> res; if (parsed) { if (auto * udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) { @@ -298,7 +310,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( }(), .exportReferencesGraph = [&] { - std::map> ret; + std::map, std::less<>> ret; if (parsed) { auto * e = get(parsed->structuredAttrs, "exportReferencesGraph"); @@ -444,9 +456,10 @@ std::optional> tryResolve( }; // Helper function to resolve exportReferencesGraph using functional style - auto tryResolveExportReferencesGraph = [&](const std::map> & exportGraph) - -> std::optional>> { - std::map> resolved; + auto tryResolveExportReferencesGraph = + [&](const std::map, std::less<>> & exportGraph) + -> std::optional, std::less<>>> { + std::map, std::less<>> resolved; for (const auto & [name, inputPaths] : exportGraph) { std::set resolvedPaths; for (const auto & inputPath : inputPaths) { @@ -466,19 +479,20 @@ std::optional> tryResolve( [&](const DerivationOptions::OutputChecks & checks) -> std::optional::OutputChecks, - std::map::OutputChecks>>> { + std::map::OutputChecks, std::less<>>>> { auto resolved = tryResolveOutputChecks(checks); if (!resolved) return std::nullopt; return std::variant< DerivationOptions::OutputChecks, - std::map::OutputChecks>>(*resolved); + std::map::OutputChecks, std::less<>>>(*resolved); }, - [&](const std::map::OutputChecks> & checksMap) + [&](const std::map::OutputChecks, std::less<>> & + checksMap) -> std::optional::OutputChecks, - std::map::OutputChecks>>> { - std::map::OutputChecks> resolvedMap; + std::map::OutputChecks, std::less<>>>> { + std::map::OutputChecks, std::less<>> resolvedMap; for (const auto & [outputName, checks] : checksMap) { auto resolved = tryResolveOutputChecks(checks); if (!resolved) @@ -487,7 +501,7 @@ std::optional> tryResolve( } return std::variant< DerivationOptions::OutputChecks, - std::map::OutputChecks>>(resolvedMap); + std::map::OutputChecks, std::less<>>>(resolvedMap); }}, drvOptions.outputChecks); @@ -525,21 +539,22 @@ namespace nlohmann { using namespace nix; -DerivationOptions adl_serializer>::from_json(const json & json_) +template +static DerivationOptions derivationOptionsFromJson(const nlohmann::json & json_) { auto & json = getObject(json_); return { - .outputChecks = [&]() -> OutputChecksVariant { + .outputChecks = [&]() -> OutputChecksVariant { auto outputChecks = getObject(valueAt(json, "outputChecks")); auto forAllOutputsOpt = get(outputChecks, "forAllOutputs"); auto perOutputOpt = get(outputChecks, "perOutput"); if (forAllOutputsOpt && !perOutputOpt) { - return static_cast>(*forAllOutputsOpt); + return static_cast>(*forAllOutputsOpt); } else if (perOutputOpt && !forAllOutputsOpt) { - return static_cast>>(*perOutputOpt); + return static_cast, std::less<>>>(*perOutputOpt); } else { throw Error("Exactly one of 'perOutput' or 'forAllOutputs' is required"); } @@ -561,17 +576,17 @@ DerivationOptions adl_serializer>::to_json( - json & json, const DerivationOptions & o) +template +static void derivationOptionsToJson(nlohmann::json & json, const DerivationOptions & o) { json["outputChecks"] = std::visit( overloaded{ - [&](const OutputChecks & checks) { + [&](const OutputChecks & checks) { nlohmann::json outputChecks; outputChecks["forAllOutputs"] = checks; return outputChecks; }, - [&](const std::map> & checksPerOutput) { + [&](const std::map, std::less<>> & checksPerOutput) { nlohmann::json outputChecks; outputChecks["perOutput"] = checksPerOutput; return outputChecks; @@ -594,7 +609,8 @@ void adl_serializer>::to_json( json["allowSubstitutes"] = o.allowSubstitutes; } -OutputChecks adl_serializer>::from_json(const json & json_) +template +static OutputChecks outputChecksFromJson(const nlohmann::json & json_) { auto & json = getObject(json_); @@ -602,16 +618,15 @@ OutputChecks adl_serializer>: .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")), .maxSize = ptrToOwned(getNullable(valueAt(json, "maxSize"))), .maxClosureSize = ptrToOwned(getNullable(valueAt(json, "maxClosureSize"))), - .allowedReferences = - ptrToOwned>>(getNullable(valueAt(json, "allowedReferences"))), + .allowedReferences = ptrToOwned>>(getNullable(valueAt(json, "allowedReferences"))), .disallowedReferences = valueAt(json, "disallowedReferences"), - .allowedRequisites = - ptrToOwned>>(getNullable(valueAt(json, "allowedRequisites"))), + .allowedRequisites = ptrToOwned>>(getNullable(valueAt(json, "allowedRequisites"))), .disallowedRequisites = valueAt(json, "disallowedRequisites"), }; } -void adl_serializer>::to_json(json & json, const OutputChecks & c) +template +static void outputChecksToJson(nlohmann::json & json, const OutputChecks & c) { json["ignoreSelfRefs"] = c.ignoreSelfRefs; json["maxSize"] = c.maxSize; @@ -622,4 +637,45 @@ void adl_serializer>::to_json(json & json, const json["disallowedRequisites"] = c.disallowedRequisites; } +DerivationOptions adl_serializer>::from_json(const json & json_) +{ + return derivationOptionsFromJson(json_); +} + +void adl_serializer>::to_json( + json & json, const DerivationOptions & o) +{ + derivationOptionsToJson(json, o); +} + +DerivationOptions adl_serializer>::from_json(const json & json_) +{ + return derivationOptionsFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const DerivationOptions & o) +{ + derivationOptionsToJson(json, o); +} + +OutputChecks adl_serializer>::from_json(const json & json_) +{ + return outputChecksFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const OutputChecks & c) +{ + outputChecksToJson(json, c); +} + +OutputChecks adl_serializer>::from_json(const json & json_) +{ + return outputChecksFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const OutputChecks & c) +{ + outputChecksToJson(json, c); +} + } // namespace nlohmann diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index d96c4892e1c9..e29f660c4848 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -84,12 +84,12 @@ struct DerivationOptions * Either one set of checks for all outputs, or separate checks * per-output. */ - std::variant> outputChecks = OutputChecks{}; + std::variant>> outputChecks = OutputChecks{}; /** * Whether to avoid scanning for references for a given output. */ - std::map unsafeDiscardReferences; + std::map> unsafeDiscardReferences; /** * In non-structured mode, all bindings specified in the derivation @@ -122,7 +122,7 @@ struct DerivationOptions * attributes give to the builder. The set of paths in the original JSON * is replaced with a list of `PathInfo` in JSON format. */ - std::map> exportReferencesGraph; + std::map, std::less<>> exportReferencesGraph; /** * env: __sandboxProfile From 7b642b5ab3702fa6f9efa6090aaed4dea3109b8b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 9 Apr 2026 00:35:58 -0400 Subject: [PATCH 218/555] Chmod cleanup (#15647) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * libutil-test-support: Add `ThrowsSysError` gmock matcher helper Pulls out the `::testing::Throws(::testing::Field(&SysError::errNo, N))` pattern into a small inline helper so tests can write `EXPECT_THAT(..., ThrowsSysError(ENOENT))` for specific-errno assertions on `SysError` exceptions. This will be used in subsequent commits. * libutil: Use `AT_SYMLINK_NOFOLLOW` in `fchmodatTryNoFollow` on all non-Linux Unixes Previously the final `::fchmodat` fallback only passed `AT_SYMLINK_NOFOLLOW` on macOS and FreeBSD. The flag is equally supported on NetBSD, OpenBSD, and DragonFly BSD — chmod of a symlink's own inode is a standard BSD feature — so extend the guard to `!defined(__linux__)`. On Linux we deliberately keep passing `0` rather than `AT_SYMLINK_NOFOLLOW`, even though we would *prefer* not to follow symlinks here either. Ideally, on a Linux symlink we'd want this fallback to fail outright rather than silently chmod the target. Unfortunately, if we reach this path at all (i.e. both the `fchmodat2` and `/proc/self/fd` fallbacks above were unavailable), glibc's `AT_SYMLINK_NOFOLLOW` compat shim will make the call fail *even for non-symlink targets* — so we can't use it as a belt-and-suspenders check. A comment at the call site explains this. Also refresh the `fchmodatTryNoFollow` test: - Replace the `EXPECT_NO_THROW(try { ... } catch { if (...) throw; })` gymnastic with a per-OS whitelist via a small `expectSymlinkChmod` lambda: Linux asserts `ThrowsSysError(EOPNOTSUPP)`; Darwin / FreeBSD / NetBSD / OpenBSD / DragonFly BSD assert `EXPECT_NO_THROW`; any other platform logs a `GTEST_LOG_(WARNING)` that chmod-on-symlink behaviour isn't verified for this OS and just exercises the call. The real invariant — the target file's mode is unchanged — is asserted unconditionally afterwards in every branch. - Add an `ENOENT` expectation via `ThrowsSysError` for the nonexistent-path failure mode. - Drop the odd `(expr, 0)` wrapping around one of the no-throw calls — it was a leftover from an older style. * Update src/libutil/unix/file-system-at.cc Co-authored-by: Sergei Zimmerman --------- Co-authored-by: Sergei Zimmerman --- .../include/nix/util/tests/gmock-matchers.hh | 13 +++++ src/libutil-tests/unix/file-system-at.cc | 55 +++++++++++++++---- src/libutil/unix/file-system-at.cc | 9 ++- 3 files changed, 64 insertions(+), 13 deletions(-) diff --git a/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh b/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh index 27d765e9e816..e48faca7c68e 100644 --- a/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh +++ b/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "nix/util/error.hh" #include "nix/util/terminal.hh" #include @@ -53,4 +54,16 @@ HasSubstrIgnoreANSIMatcher(const std::string & substring) return ::testing::MakePolymorphicMatcher(internal::HasSubstrIgnoreANSIMatcher(substring)); } +/** + * Matches a callable that throws `SysError` whose `errNo` equals `expected`. + * + * Example: + * + * EXPECT_THAT([&]{ openFile("nope"); }, ThrowsSysError(ENOENT)); + */ +inline auto ThrowsSysError(int expected) +{ + return ::testing::Throws(::testing::Field(&SysError::errNo, expected)); +} + } // namespace nix::testing diff --git a/src/libutil-tests/unix/file-system-at.cc b/src/libutil-tests/unix/file-system-at.cc index b72851358ac5..5fb0d26b9ced 100644 --- a/src/libutil-tests/unix/file-system-at.cc +++ b/src/libutil-tests/unix/file-system-at.cc @@ -1,9 +1,11 @@ #include +#include #include "nix/util/file-system-at.hh" #include "nix/util/file-system.hh" #include "nix/util/fs-sink.hh" #include "nix/util/processes.hh" +#include "nix/util/tests/gmock-matchers.hh" #ifdef __linux__ # include "nix/util/linux-namespaces.hh" @@ -42,21 +44,48 @@ TEST(fchmodatTryNoFollow, works) struct ::stat st; - /* Check that symlinks are not followed and targets are not changed. */ + using nix::testing::ThrowsSysError; + + /* Check that symlinks are not followed and targets are not changed. + + Whitelist per OS rather than "not Linux", so that an unrecognised + platform fails loudly at compile time instead of silently getting the + wrong expectation: + + - Linux: always throws `SysError(EOPNOTSUPP)` — Linux symlinks have no + mutable mode bits and `fchmodat` with `AT_SYMLINK_NOFOLLOW` is + unsupported. + + - BSDs and descendants (Darwin, FreeBSD, NetBSD, OpenBSD, DragonFly BSD): + succeeds, modifying the symlink's own inode mode without touching the + target. + + - Anything else: unverified — warn and exercise the call so we at least + detect crashes, but don't assert an outcome. + + The `ASSERT_EQ`/`EXPECT_EQ` below run in every branch and enforce the + invariant that the *target* file's mode is unchanged, which is the + property we actually care about. */ + auto expectSymlinkChmod = [&](std::string_view path, mode_t mode) { +#if defined(__linux__) + EXPECT_THAT([&] { fchmodatTryNoFollow(dirFd.get(), CanonPath(path), mode); }, ThrowsSysError(EOPNOTSUPP)); +#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) \ + || defined(__DragonFly__) + EXPECT_NO_THROW(fchmodatTryNoFollow(dirFd.get(), CanonPath(path), mode)); +#else + GTEST_LOG_(WARNING) << "unknown platform: chmod-on-symlink behaviour is not verified for this OS"; + try { + fchmodatTryNoFollow(dirFd.get(), CanonPath(path), mode); + } catch (SysError &) { + } +#endif + }; - EXPECT_NO_THROW( - try { fchmodatTryNoFollow(dirFd.get(), CanonPath("filelink"), 0777); } catch (SysError & e) { - if (e.errNo != EOPNOTSUPP) - throw; - }); + expectSymlinkChmod("filelink", 0777); ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); EXPECT_EQ(st.st_mode & 0777, 0644); - EXPECT_NO_THROW( - try { fchmodatTryNoFollow(dirFd.get(), CanonPath("dirlink"), 0777); } catch (SysError & e) { - if (e.errNo != EOPNOTSUPP) - throw; - }); + expectSymlinkChmod("dirlink", 0777); ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); EXPECT_EQ(st.st_mode & 0777, 0755); @@ -66,9 +95,11 @@ TEST(fchmodatTryNoFollow, works) ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); EXPECT_EQ(st.st_mode & 0777, 0600); - EXPECT_NO_THROW((fchmodatTryNoFollow(dirFd.get(), CanonPath("dir"), 0700), 0)); + EXPECT_NO_THROW(fchmodatTryNoFollow(dirFd.get(), CanonPath("dir"), 0700)); ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); EXPECT_EQ(st.st_mode & 0777, 0700); + + EXPECT_THAT([&] { fchmodatTryNoFollow(dirFd.get(), CanonPath("nonexistent"), 0600); }, ThrowsSysError(ENOENT)); } #ifdef __linux__ diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index d2401382ef99..d8be6fdeb70b 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -121,9 +121,16 @@ void unix::fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t dirFd, path.rel_c_str(), mode, -#if defined(__APPLE__) || defined(__FreeBSD__) +#if defined(AT_SYMLINK_NOFOLLOW) && !defined(__linux__) AT_SYMLINK_NOFOLLOW #else + /* We would like to avoid following symlinks on Linux too. (Even though + Linux doesn't support chmoding symlinks, we should still fail if we + try, and not falsely succeed by following.) However, if we reach + this point, rather than the Linux-specific cases above, it means we + will likely hit glibc compat paths that will make using + AT_SYMLINK_NOFOLLOW cause failures even if there is no symlink + being followed! */ 0 #endif ); From 84acfc03f6af30042714d82d79eebb799b64f7a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Apr 2026 10:37:26 +0200 Subject: [PATCH 219/555] dev-shell.nix: Propagate mesonFlags from the nix component (#15646) This ensures that meson flags defined in src/nix/package.nix are respected in the dev shell. --- packaging/dev-shell.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index b058bbe53db7..a29d65cb2a8d 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -291,7 +291,8 @@ nixComponents.callPackage ( map (transformFlag "perl") (ignoreCrossFile nixComponents.nix-perl-bindings.mesonFlags) ) ++ map (transformFlag "libexpr") (ignoreCrossFile nixComponents.nix-expr.mesonFlags) - ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags); + ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags) + ++ map (transformFlag "nix") (ignoreCrossFile nixComponents.nix-cli.mesonFlags); nativeBuildInputs = let From 6d1d4c323550fb21426edaa1d19a626eb6c717b3 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Thu, 9 Apr 2026 10:13:21 -0400 Subject: [PATCH 220/555] Add _WIN32 guard to spawn.cc Signed-off-by: Lisanna Dettwyler --- src/libutil-tests/spawn.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil-tests/spawn.cc b/src/libutil-tests/spawn.cc index 715c0d17fb2f..db6502390c6d 100644 --- a/src/libutil-tests/spawn.cc +++ b/src/libutil-tests/spawn.cc @@ -1,11 +1,11 @@ -#include +#ifdef _WIN32 +# include -#include "nix/util/os-string.hh" -#include "nix/util/processes.hh" +# include "nix/util/os-string.hh" +# include "nix/util/processes.hh" namespace nix { -#ifdef _WIN32 TEST(SpawnTest, spawnEcho) { auto output = @@ -34,5 +34,5 @@ TEST(SpawnTest, windowsEscape) auto space = windowsEscape(L"hello world", false); ASSERT_EQ(space, LR"("hello world")"); } -#endif } // namespace nix +#endif From 303fc45c71cb7ac6f366ac8901dab92e428c9355 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Thu, 9 Apr 2026 14:20:48 -0400 Subject: [PATCH 221/555] Cleanup unnecessary includes Signed-off-by: Lisanna Dettwyler --- src/libcmd/built-path.cc | 2 -- src/libcmd/common-eval-args.cc | 2 -- src/libcmd/installable-attr-path.cc | 11 ----------- src/libcmd/installable-flake.cc | 12 ------------ src/libcmd/installables.cc | 5 ----- src/libcmd/repl-interacter.cc | 2 -- src/libcmd/unix/unix-socket-server.cc | 1 - src/libexpr-c/nix_api_external.cc | 3 --- src/libexpr-c/nix_api_value.cc | 3 --- src/libexpr-test-support/tests/value/context.cc | 5 +++-- src/libexpr-tests/derived-path.cc | 4 +++- src/libexpr-tests/eval.cc | 1 - src/libexpr-tests/main.cc | 2 +- src/libexpr-tests/nix_api_external.cc | 2 -- src/libexpr-tests/nix_api_value.cc | 1 - src/libexpr-tests/value/context.cc | 3 +-- src/libexpr/eval-error.cc | 1 - src/libexpr/eval-gc.cc | 2 -- src/libexpr/get-drvs.cc | 1 - src/libexpr/json-to-value.cc | 1 - src/libexpr/primops/fetchClosure.cc | 1 - src/libexpr/primops/fetchMercurial.cc | 2 -- src/libexpr/primops/fetchTree.cc | 1 - src/libexpr/value-to-json.cc | 1 - src/libexpr/value/context.cc | 2 -- src/libfetchers-tests/access-tokens.cc | 2 -- src/libfetchers-tests/git-utils.cc | 1 - src/libfetchers-tests/git.cc | 1 - src/libfetchers-tests/nix_api_fetchers.cc | 1 - src/libfetchers-tests/public-key.cc | 1 - src/libfetchers/attrs.cc | 1 - src/libfetchers/fetchers.cc | 2 +- src/libfetchers/git-utils.cc | 2 -- src/libfetchers/git.cc | 7 +------ src/libfetchers/github.cc | 2 -- src/libfetchers/input-cache.cc | 1 - src/libfetchers/mercurial.cc | 2 -- src/libfetchers/registry.cc | 2 +- src/libflake/config.cc | 2 -- src/libflake/flake-primops.cc | 6 ------ src/libflake/flake.cc | 2 -- src/libflake/lockfile.cc | 5 ----- src/libmain-c/nix_api_main.cc | 2 -- src/libmain/progress-bar.cc | 1 - src/libmain/shared.cc | 2 -- src/libstore-test-support/derived-path.cc | 5 +++-- src/libstore-test-support/path.cc | 8 +++----- src/libstore-tests/common-protocol.cc | 3 --- src/libstore-tests/derivation-advanced-attrs.cc | 3 --- src/libstore-tests/derivation-parser-bench.cc | 1 - src/libstore-tests/derivation/invariants.cc | 2 -- src/libstore-tests/derived-path.cc | 2 -- src/libstore-tests/http-binary-cache-store.cc | 1 - src/libstore-tests/machines.cc | 2 +- src/libstore-tests/realisation.cc | 2 -- src/libstore-tests/register-valid-paths-bench.cc | 1 - src/libstore-tests/s3-binary-cache-store.cc | 1 - src/libstore-tests/serve-protocol.cc | 1 - src/libstore-tests/store-open.cc | 1 - src/libstore-tests/worker-protocol.cc | 1 - src/libstore/binary-cache-store.cc | 4 +--- src/libstore/build/derivation-building-goal.cc | 1 - src/libstore/build/derivation-goal.cc | 11 ----------- src/libstore/build/drv-output-substitution-goal.cc | 2 -- src/libstore/build/entry-points.cc | 1 - src/libstore/build/substitution-goal.cc | 5 +---- src/libstore/builtins/fetchurl.cc | 1 - src/libstore/common-protocol.cc | 3 +-- src/libstore/daemon.cc | 2 -- src/libstore/derivation-options.cc | 1 - src/libstore/derivations.cc | 2 -- src/libstore/derived-path.cc | 4 +--- src/libstore/filetransfer.cc | 2 -- src/libstore/gc.cc | 4 ++-- src/libstore/globals.cc | 4 ---- src/libstore/local-binary-cache-store.cc | 3 +-- src/libstore/local-fs-store.cc | 2 -- src/libstore/local-gc.cc | 1 - src/libstore/local-overlay-store.cc | 1 - src/libstore/machines.cc | 1 - src/libstore/misc.cc | 2 -- src/libstore/nar-info.cc | 3 +-- src/libstore/optimise-store.cc | 4 +--- src/libstore/path-references.cc | 4 ---- src/libstore/path-with-outputs.cc | 2 -- src/libstore/pathlocks.cc | 2 -- src/libstore/posix-fs-canonicalise.cc | 3 --- src/libstore/profiles.cc | 3 --- src/libstore/references.cc | 2 -- src/libstore/remote-store.cc | 1 - src/libstore/s3-url.cc | 1 - src/libstore/serve-protocol.cc | 1 - src/libstore/sqlite.cc | 3 +-- src/libstore/ssh-store.cc | 1 - src/libstore/ssh.cc | 1 - src/libstore/store-api.cc | 2 -- src/libstore/uds-remote-store.cc | 2 +- src/libstore/unix/pathlocks.cc | 2 -- src/libstore/unix/user-lock.cc | 2 +- src/libstore/worker-protocol.cc | 1 - src/libutil-test-support/hash.cc | 6 +++--- .../include/nix/util/tests/json-characterization.hh | 1 + src/libutil-tests/args.cc | 1 - src/libutil-tests/config.cc | 2 -- src/libutil-tests/file-system.cc | 7 ------- src/libutil-tests/hash.cc | 2 -- src/libutil-tests/monitorfdhup.cc | 1 - src/libutil-tests/nar-listing.cc | 1 - src/libutil-tests/nix_api_util.cc | 1 - src/libutil-tests/nix_api_util_internal.cc | 4 ---- src/libutil-tests/terminal.cc | 5 ----- src/libutil-tests/topo-sort.cc | 1 - src/libutil-tests/util.cc | 6 ------ src/libutil/archive.cc | 2 -- src/libutil/args.cc | 1 - src/libutil/compression.cc | 1 - src/libutil/current-process.cc | 3 --- src/libutil/environment-variables.cc | 1 - src/libutil/error.cc | 1 - src/libutil/executable-path.cc | 1 - src/libutil/file-system.cc | 1 - src/libutil/git.cc | 2 -- src/libutil/hash.cc | 2 -- src/libutil/linux/linux-namespaces.cc | 4 ---- src/libutil/logging.cc | 2 -- src/libutil/memory-source-accessor.cc | 1 - src/libutil/nar-accessor.cc | 1 - src/libutil/posix-source-accessor.cc | 1 - src/libutil/signature/signer.cc | 1 - src/libutil/strings.cc | 3 --- src/libutil/suggestions.cc | 1 - src/libutil/terminal.cc | 1 - src/libutil/unix/file-descriptor.cc | 3 --- src/libutil/unix/file-path.cc | 6 ------ src/libutil/unix/muxable-pipe.cc | 1 - src/libutil/unix/processes.cc | 2 -- src/libutil/unix/users.cc | 2 +- src/libutil/unix/xdg-dirs.cc | 2 +- src/libutil/users.cc | 2 -- src/libutil/util.cc | 3 --- src/libutil/windows/current-process.cc | 1 - src/nix/add-to-store.cc | 2 -- src/nix/build-remote/build-remote.cc | 1 - src/nix/build.cc | 1 - src/nix/bundle.cc | 1 - src/nix/cat.cc | 1 - src/nix/config.cc | 2 -- src/nix/derivation-add.cc | 1 - src/nix/derivation-show.cc | 1 - src/nix/develop.cc | 3 +-- src/nix/diff-closures.cc | 1 - src/nix/formatter.cc | 2 -- src/nix/hash.cc | 2 -- src/nix/log.cc | 1 - src/nix/main.cc | 1 - src/nix/nix-build/nix-build.cc | 1 - src/nix/nix-channel/nix-channel.cc | 1 - src/nix/nix-collect-garbage/nix-collect-garbage.cc | 1 - src/nix/nix-env/user-env.cc | 2 -- src/nix/nix-instantiate/nix-instantiate.cc | 2 -- src/nix/nix-store/nix-store.cc | 1 - src/nix/optimise-store.cc | 2 -- src/nix/path-info.cc | 1 - src/nix/prefetch.cc | 2 -- src/nix/registry.cc | 2 -- src/nix/repl.cc | 2 -- src/nix/run.cc | 4 ---- src/nix/search.cc | 2 -- src/nix/self-exe.cc | 3 +-- src/nix/store-copy-log.cc | 4 ---- src/nix/store-delete.cc | 1 - src/nix/store-gc.cc | 1 - src/nix/unix/daemon.cc | 5 +---- 173 files changed, 39 insertions(+), 355 deletions(-) diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index fae65fca2b0d..2c52678105f4 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -6,8 +6,6 @@ #include -#include - namespace nix { // Custom implementation to avoid `ref` ptr equality diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 7e4bd7a162a3..984bed34882e 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,9 +1,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/expr/eval-settings.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/main/shared.hh" #include "nix/util/config-global.hh" -#include "nix/store/filetransfer.hh" #include "nix/expr/eval.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/registry.hh" diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 28c3db3fc79a..aa5da2e56646 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -1,24 +1,13 @@ -#include "nix/store/globals.hh" #include "nix/cmd/installable-attr-path.hh" #include "nix/store/outputs-spec.hh" #include "nix/util/util.hh" #include "nix/cmd/command.hh" #include "nix/expr/attr-path.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/store/derivations.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" #include "nix/expr/get-drvs.hh" -#include "nix/store/store-api.hh" -#include "nix/main/shared.hh" #include "nix/flake/flake.hh" -#include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" -#include "nix/fetchers/registry.hh" -#include "nix/store/build-result.hh" - -#include -#include #include diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 77a7c8d6ec1b..1ef34a1ce014 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -1,25 +1,13 @@ -#include "nix/store/globals.hh" #include "nix/cmd/installable-flake.hh" -#include "nix/cmd/installable-derived-path.hh" #include "nix/store/outputs-spec.hh" #include "nix/util/util.hh" #include "nix/cmd/command.hh" #include "nix/expr/attr-path.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/store/derivations.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" -#include "nix/expr/get-drvs.hh" -#include "nix/store/store-api.hh" -#include "nix/main/shared.hh" #include "nix/flake/flake.hh" #include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" -#include "nix/fetchers/registry.hh" -#include "nix/store/build-result.hh" - -#include -#include #include diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 8f8309bd96d7..a25971ad7f89 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -13,18 +13,13 @@ #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" -#include "nix/expr/get-drvs.hh" #include "nix/store/store-api.hh" #include "nix/main/shared.hh" #include "nix/flake/flake.hh" #include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" #include "nix/fetchers/registry.hh" #include "nix/store/build-result.hh" -#include -#include - #include #include "nix/util/strings-inline.hh" diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 58d55e0a2dfc..81240af7f547 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -18,12 +18,10 @@ extern "C" { } #endif -#include "nix/util/signals.hh" #include "nix/util/finally.hh" #include "nix/cmd/repl-interacter.hh" #include "nix/util/file-system.hh" #include "nix/util/serialise.hh" -#include "nix/cmd/repl.hh" #include "nix/util/environment-variables.hh" namespace nix { diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index e33f4317dad9..5d1fba462207 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -5,7 +5,6 @@ #include "nix/util/file-system.hh" #include "nix/util/logging.hh" #include "nix/util/signals.hh" -#include "nix/util/strings.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/util.hh" diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index a874d9a0861e..98b68d9f6b49 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -1,7 +1,4 @@ -#include "nix/expr/attr-set.hh" -#include "nix/util/configuration.hh" #include "nix/expr/eval.hh" -#include "nix/store/globals.hh" #include "nix/expr/value.hh" #include "nix_api_expr.h" diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 589ebf9a8ec2..3cb5705898c4 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -1,8 +1,6 @@ #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" -#include "nix/util/configuration.hh" #include "nix/expr/eval.hh" -#include "nix/store/globals.hh" #include "nix/store/path.hh" #include "nix/expr/primops.hh" #include "nix/expr/value.hh" @@ -13,7 +11,6 @@ #include "nix_api_util_internal.h" #include "nix_api_store_internal.h" #include "nix_api_value.h" -#include "nix/expr/value/context.hh" // Internal helper functions to check [in] and [out] `Value *` parameters static const nix::Value & check_value_not_null(const nix_value * value) diff --git a/src/libexpr-test-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc index d6036601a948..ca7996acc16e 100644 --- a/src/libexpr-test-support/tests/value/context.cc +++ b/src/libexpr-test-support/tests/value/context.cc @@ -1,7 +1,8 @@ -#include // Needed by rapidcheck on Darwin +#ifdef __APPLE__ +# include // Needed by rapidcheck on Darwin +#endif #include -#include "nix/store/tests/path.hh" #include "nix/expr/tests/value/context.hh" namespace rc { diff --git a/src/libexpr-tests/derived-path.cc b/src/libexpr-tests/derived-path.cc index e9f9fcd0720d..c685f6a094a8 100644 --- a/src/libexpr-tests/derived-path.cc +++ b/src/libexpr-tests/derived-path.cc @@ -1,6 +1,8 @@ #include #include -#include // Needed by rapidcheck on Darwin +#ifdef __APPLE__ +# include // Needed by rapidcheck on Darwin +#endif #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libexpr-tests/eval.cc b/src/libexpr-tests/eval.cc index 7562a9da21ad..985564f13e7d 100644 --- a/src/libexpr-tests/eval.cc +++ b/src/libexpr-tests/eval.cc @@ -3,7 +3,6 @@ #include "nix/expr/eval.hh" #include "nix/expr/tests/libexpr.hh" -#include "nix/util/memory-source-accessor.hh" namespace nix { diff --git a/src/libexpr-tests/main.cc b/src/libexpr-tests/main.cc index 365fa482de2d..5a717d6bb053 100644 --- a/src/libexpr-tests/main.cc +++ b/src/libexpr-tests/main.cc @@ -1,7 +1,7 @@ #include #include "nix/store/tests/test-main.hh" -#include "nix/util/config-global.hh" +#include "nix/util/configuration.hh" int main(int argc, char ** argv) { diff --git a/src/libexpr-tests/nix_api_external.cc b/src/libexpr-tests/nix_api_external.cc index 885b9c1d2dd3..e17a52c31dfc 100644 --- a/src/libexpr-tests/nix_api_external.cc +++ b/src/libexpr-tests/nix_api_external.cc @@ -1,5 +1,3 @@ -#include "nix_api_store.h" -#include "nix_api_util.h" #include "nix_api_expr.h" #include "nix_api_value.h" #include "nix_api_external.h" diff --git a/src/libexpr-tests/nix_api_value.cc b/src/libexpr-tests/nix_api_value.cc index 830637f3ec50..01d15744a750 100644 --- a/src/libexpr-tests/nix_api_value.cc +++ b/src/libexpr-tests/nix_api_value.cc @@ -1,4 +1,3 @@ -#include "nix_api_store.h" #include "nix_api_util.h" #include "nix_api_expr.h" #include "nix_api_value.h" diff --git a/src/libexpr-tests/value/context.cc b/src/libexpr-tests/value/context.cc index fe3072b64ffd..544f03f3f2e3 100644 --- a/src/libexpr-tests/value/context.cc +++ b/src/libexpr-tests/value/context.cc @@ -2,9 +2,8 @@ #include #include -#include "nix/store/tests/path.hh" -#include "nix/expr/tests/libexpr.hh" #include "nix/expr/tests/value/context.hh" +#include "nix/store/store-dir-config.hh" namespace nix { diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 38a60883bca7..45cb1e409dd1 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -1,7 +1,6 @@ #include "nix/expr/eval-error.hh" #include "nix/expr/eval.hh" #include "nix/expr/value.hh" -#include "nix/store/store-api.hh" namespace nix { diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 0d25f38f64de..9344b0405b17 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -1,8 +1,6 @@ -#include "nix/util/error.hh" #include "nix/util/environment-variables.hh" #include "nix/expr/eval-settings.hh" #include "nix/util/config-global.hh" -#include "nix/util/serialise.hh" #include "nix/expr/eval-gc.hh" #include "nix/expr/value.hh" diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 03a1aa455ce0..693b1946ee46 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -5,7 +5,6 @@ #include "nix/store/path-with-outputs.hh" #include -#include namespace nix { diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 4a68308c6416..08393435b48c 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -3,7 +3,6 @@ #include "nix/expr/eval.hh" #include -#include #include using json = nlohmann::json; diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 2e01d67172d3..db182ab499b2 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -2,7 +2,6 @@ #include "nix/store/store-open.hh" #include "nix/store/realisation.hh" #include "nix/store/make-content-addressed.hh" -#include "nix/util/url.hh" #include "nix/util/environment-variables.hh" namespace nix { diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 9347645cc204..97aeb0a03574 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,9 +1,7 @@ #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" -#include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" -#include "nix/util/url.hh" #include "nix/util/url-parts.hh" namespace nix { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 557717d2e4d2..afd61e90fbb5 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -16,7 +16,6 @@ #include #include -#include namespace nix { diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 4fac29a6671d..8d48a48391cc 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -4,7 +4,6 @@ #include "nix/util/signals.hh" #include -#include #include namespace nix { diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 4a17bbdc3a83..60ba5352077d 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -2,8 +2,6 @@ #include "nix/expr/value/context.hh" #include "nix/store/store-dir-config.hh" -#include - namespace nix { NixStringContextElem NixStringContextElem::parse(std::string_view s0, const ExperimentalFeatureSettings & xpSettings) diff --git a/src/libfetchers-tests/access-tokens.cc b/src/libfetchers-tests/access-tokens.cc index 7127434db9df..5e92a9bc7be7 100644 --- a/src/libfetchers-tests/access-tokens.cc +++ b/src/libfetchers-tests/access-tokens.cc @@ -3,8 +3,6 @@ #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/fetch-settings.hh" -#include "nix/util/json-utils.hh" -#include "nix/util/tests/characterization.hh" namespace nix::fetchers { diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index a80ff590f932..580769936d41 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -10,7 +10,6 @@ #include #include "nix/util/fs-sink.hh" #include "nix/util/serialise.hh" -#include "nix/fetchers/git-lfs-fetch.hh" #include #include diff --git a/src/libfetchers-tests/git.cc b/src/libfetchers-tests/git.cc index e970c3b8bdc4..e41bbe520f41 100644 --- a/src/libfetchers-tests/git.cc +++ b/src/libfetchers-tests/git.cc @@ -1,4 +1,3 @@ -#include "nix/store/store-open.hh" #include "nix/store/globals.hh" #include "nix/store/dummy-store.hh" #include "nix/fetchers/fetch-settings.hh" diff --git a/src/libfetchers-tests/nix_api_fetchers.cc b/src/libfetchers-tests/nix_api_fetchers.cc index 8f3e6e3c5839..ff2c5d8ff319 100644 --- a/src/libfetchers-tests/nix_api_fetchers.cc +++ b/src/libfetchers-tests/nix_api_fetchers.cc @@ -1,4 +1,3 @@ -#include "gmock/gmock.h" #include #include "nix_api_fetchers.h" diff --git a/src/libfetchers-tests/public-key.cc b/src/libfetchers-tests/public-key.cc index 2991223f6b35..dabbd464424c 100644 --- a/src/libfetchers-tests/public-key.cc +++ b/src/libfetchers-tests/public-key.cc @@ -1,6 +1,5 @@ #include #include "nix/fetchers/fetchers.hh" -#include "nix/util/json-utils.hh" #include "nix/util/tests/json-characterization.hh" namespace nix { diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 841808bd16a9..cc9e72af460d 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -1,5 +1,4 @@ #include "nix/fetchers/attrs.hh" -#include "nix/fetchers/fetchers.hh" #include diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 29b01e45a776..fb87f9b94506 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,12 +1,12 @@ #include "nix/fetchers/fetchers.hh" #include "nix/store/store-api.hh" +#include "nix/util/fs-sink.hh" #include "nix/util/source-path.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/json-utils.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/url.hh" -#include "nix/util/archive.hh" #include diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index fcffdcfa8aef..eaea4b6d5919 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -42,9 +42,7 @@ #include #include #include -#include #include -#include #include namespace std { diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 736d2e39f7e1..3c191d32f5a1 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,25 +1,20 @@ +#include "nix/util/environment-variables.hh" #include "nix/util/error.hh" #include "nix/fetchers/fetchers.hh" #include "nix/util/users.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" -#include "nix/util/tarfile.hh" #include "nix/store/store-api.hh" -#include "nix/util/url-parts.hh" #include "nix/store/pathlocks.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/util/git.hh" #include "nix/fetchers/git-utils.hh" #include "nix/util/logging.hh" -#include "nix/util/finally.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/util/json-utils.hh" #include "nix/util/archive.hh" #include "nix/util/mounted-source-accessor.hh" -#include -#include #include #ifndef _WIN32 diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ac8c02418dab..855475d7fcb0 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -1,6 +1,5 @@ #include "nix/store/filetransfer.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" #include "nix/store/store-api.hh" #include "nix/util/types.hh" #include "nix/util/url-parts.hh" @@ -13,7 +12,6 @@ #include #include -#include namespace nix::fetchers { diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 652d5ce7976b..85a611355e2f 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -1,7 +1,6 @@ #include "nix/fetchers/input-cache.hh" #include "nix/fetchers/registry.hh" #include "nix/util/sync.hh" -#include "nix/util/source-path.hh" namespace nix::fetchers { diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 38fe31fcadea..db0333e9087d 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -7,8 +7,6 @@ #include "nix/util/environment-variables.hh" #include "nix/util/users.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" -#include "nix/util/tarfile.hh" #include "nix/store/store-api.hh" #include "nix/util/url-parts.hh" #include "nix/fetchers/fetch-settings.hh" diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 9911586fa037..0aad8ef48acb 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,8 +1,8 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/registry.hh" #include "nix/fetchers/tarball.hh" +#include "nix/store/filetransfer.hh" #include "nix/util/users.hh" -#include "nix/store/globals.hh" #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" diff --git a/src/libflake/config.cc b/src/libflake/config.cc index b7d5cd8999e5..6367d7b4ec42 100644 --- a/src/libflake/config.cc +++ b/src/libflake/config.cc @@ -11,14 +11,12 @@ #include #include #include -#include #include "nix/util/users.hh" #include "nix/util/config-global.hh" #include "nix/flake/settings.hh" #include "nix/flake/flake.hh" #include "nix/util/ansicolor.hh" -#include "nix/util/configuration.hh" #include "nix/util/file-system.hh" #include "nix/util/fmt.hh" #include "nix/util/logging.hh" diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 4b84c081e65f..897739ed17e9 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -1,12 +1,8 @@ #include -#include #include #include -#include #include -#include #include -#include #include "nix/flake/flake-primops.hh" #include "nix/expr/eval.hh" @@ -21,12 +17,10 @@ #include "nix/expr/value.hh" #include "nix/fetchers/attrs.hh" #include "nix/fetchers/fetchers.hh" -#include "nix/util/configuration.hh" #include "nix/util/error.hh" #include "nix/util/experimental-features.hh" #include "nix/util/pos-idx.hh" #include "nix/util/pos-table.hh" -#include "nix/util/source-path.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 67522d2c4751..277adc092f2a 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include "nix/util/terminal.hh" #include "nix/util/ref.hh" @@ -57,7 +56,6 @@ #include "nix/util/logging.hh" #include "nix/util/pos-idx.hh" #include "nix/util/pos-table.hh" -#include "nix/util/position.hh" #include "nix/util/source-path.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index 4eeba3bd3ef1..75127521b942 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -8,10 +8,7 @@ #include #include #include -#include #include -#include -#include #include #include #include @@ -32,10 +29,8 @@ #include "nix/flake/flakeref.hh" #include "nix/store/path.hh" #include "nix/util/ansicolor.hh" -#include "nix/util/configuration.hh" #include "nix/util/error.hh" #include "nix/util/fmt.hh" -#include "nix/util/hash.hh" #include "nix/util/logging.hh" #include "nix/util/ref.hh" #include "nix/util/types.hh" diff --git a/src/libmain-c/nix_api_main.cc b/src/libmain-c/nix_api_main.cc index 0ee965dc82ee..872949e423b9 100644 --- a/src/libmain-c/nix_api_main.cc +++ b/src/libmain-c/nix_api_main.cc @@ -1,5 +1,3 @@ -#include "nix_api_store.h" -#include "nix_api_store_internal.h" #include "nix_api_util.h" #include "nix_api_util_internal.h" diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 71c7789de254..d177a1508b3f 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -5,7 +5,6 @@ #include "nix/store/store-api.hh" #include "nix/store/names.hh" -#include #include #include #include diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 3640efc07a00..f3293060f16a 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -6,12 +6,10 @@ #include "nix/store/store-open.hh" #include "nix/store/gc-store.hh" #include "nix/main/loggers.hh" -#include "nix/main/progress-bar.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" #include -#include #include #include diff --git a/src/libstore-test-support/derived-path.cc b/src/libstore-test-support/derived-path.cc index cb1d23ac5208..ee8018c3268f 100644 --- a/src/libstore-test-support/derived-path.cc +++ b/src/libstore-test-support/derived-path.cc @@ -1,6 +1,7 @@ -#include -#include // Needed by rapidcheck on Darwin +#ifdef __APPLE__ +# include // Needed by rapidcheck on Darwin +#endif #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libstore-test-support/path.cc b/src/libstore-test-support/path.cc index 1e5f74a28541..98a255ccc026 100644 --- a/src/libstore-test-support/path.cc +++ b/src/libstore-test-support/path.cc @@ -1,12 +1,10 @@ -#include // Needed by rapidcheck on Darwin -#include +#ifdef __APPLE__ +# include // Needed by rapidcheck on Darwin +#endif #include #include -#include "nix/store/path-regex.hh" -#include "nix/store/store-api.hh" - #include "nix/util/tests/hash.hh" #include "nix/store/tests/path.hh" diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 9146af219c20..5afde2bdf7e2 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -1,12 +1,9 @@ -#include - #include #include #include "nix/util/json-utils.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" -#include "nix/store/build-result.hh" #include "nix/store/tests/protocol.hh" #include "nix/util/tests/characterization.hh" diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index aa24943a72ea..bb7ad28f3d78 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -1,14 +1,11 @@ #include -#include -#include "nix/util/experimental-features.hh" #include "nix/store/derivations.hh" #include "nix/store/derived-path.hh" #include "nix/store/derivation-options.hh" #include "nix/store/globals.hh" #include "nix/store/parsed-derivations.hh" #include "nix/util/types.hh" -#include "nix/util/json-utils.hh" #include "nix/store/tests/libstore.hh" #include "nix/util/tests/json-characterization.hh" diff --git a/src/libstore-tests/derivation-parser-bench.cc b/src/libstore-tests/derivation-parser-bench.cc index 93fa64f6191e..5bc4cec9f38f 100644 --- a/src/libstore-tests/derivation-parser-bench.cc +++ b/src/libstore-tests/derivation-parser-bench.cc @@ -1,7 +1,6 @@ #include #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" -#include "nix/util/experimental-features.hh" #include "nix/util/tests/test-data.hh" #include "nix/store/store-open.hh" #include diff --git a/src/libstore-tests/derivation/invariants.cc b/src/libstore-tests/derivation/invariants.cc index 115d5bc4bb1f..e825655d66f4 100644 --- a/src/libstore-tests/derivation/invariants.cc +++ b/src/libstore-tests/derivation/invariants.cc @@ -6,8 +6,6 @@ #include "nix/store/dummy-store-impl.hh" #include "nix/util/tests/json-characterization.hh" -#include "derivation/test-support.hh" - namespace nix { class FillInOutputPathsTest : public LibStoreTest, public JsonCharacterizationTest diff --git a/src/libstore-tests/derived-path.cc b/src/libstore-tests/derived-path.cc index 70e789c0c4fd..541d729e030c 100644 --- a/src/libstore-tests/derived-path.cc +++ b/src/libstore-tests/derived-path.cc @@ -1,5 +1,3 @@ -#include - #include #include diff --git a/src/libstore-tests/http-binary-cache-store.cc b/src/libstore-tests/http-binary-cache-store.cc index aee6f7f4b1d3..74f3b93cd3df 100644 --- a/src/libstore-tests/http-binary-cache-store.cc +++ b/src/libstore-tests/http-binary-cache-store.cc @@ -3,7 +3,6 @@ #include "nix/store/http-binary-cache-store.hh" #include "nix/store/tests/https-store.hh" -#include "nix/util/fs-sink.hh" namespace nix { diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index e412376323de..7c9a17ed6b00 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -2,7 +2,7 @@ #include "nix/util/file-system.hh" #include "nix/util/util.hh" -#include "nix/util/tests/characterization.hh" +#include "nix/util/tests/test-data.hh" #include #include diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index b3584d6a1e42..814b6713664a 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -1,5 +1,3 @@ -#include - #include #include #include diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 400a7055569e..51bcb29aa903 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -5,7 +5,6 @@ #include "nix/store/store-open.hh" #include "nix/util/file-system.hh" #include "nix/util/hash.hh" -#include "nix/util/tests/test-data.hh" #ifndef _WIN32 diff --git a/src/libstore-tests/s3-binary-cache-store.cc b/src/libstore-tests/s3-binary-cache-store.cc index 9aa9b2dd1a3e..126d7a522aaa 100644 --- a/src/libstore-tests/s3-binary-cache-store.cc +++ b/src/libstore-tests/s3-binary-cache-store.cc @@ -1,7 +1,6 @@ #include "nix/store/s3-binary-cache-store.hh" #include "nix/store/http-binary-cache-store.hh" #include "nix/store/filetransfer.hh" -#include "nix/store/s3-url.hh" #include diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 6418cb55dc3d..e34f916cbb5c 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/src/libstore-tests/store-open.cc b/src/libstore-tests/store-open.cc index 4e82b358afee..cadeb3f040a4 100644 --- a/src/libstore-tests/store-open.cc +++ b/src/libstore-tests/store-open.cc @@ -3,7 +3,6 @@ #include "nix/store/store-open.hh" #include "nix/store/store-reference.hh" #include "nix/store/local-store.hh" -#include "nix/store/uds-remote-store.hh" #include "nix/store/globals.hh" #include "nix/util/file-system.hh" #include "nix/util/finally.hh" diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index ed288a7443ab..91acde7380f6 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -1,4 +1,3 @@ -#include #include #include diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 73fc6981d5be..624bbd6496df 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -3,11 +3,10 @@ #include "nix/util/compression.hh" #include "nix/store/derivations.hh" #include "nix/util/source-accessor.hh" -#include "nix/store/globals.hh" +#include "nix/store/nar-info-disk-cache.hh" #include "nix/store/nar-info.hh" #include "nix/util/sync.hh" #include "nix/store/remote-fs-accessor.hh" -#include "nix/store/nar-info-disk-cache.hh" #include "nix/util/nar-accessor.hh" #include "nix/util/thread-pool.hh" #include "nix/util/callback.hh" @@ -17,7 +16,6 @@ #include #include #include -#include #include #include diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 2235cf872b63..93a13e5c8de7 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -18,7 +18,6 @@ #include "nix/store/globals.hh" #include -#include #include #include #include diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 115e48612fcd..6e2d3223b10f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2,29 +2,18 @@ #include "nix/store/build/drv-output-substitution-goal.hh" #include "nix/store/build/derivation-building-goal.hh" #include "nix/store/build/derivation-resolution-goal.hh" -#ifndef _WIN32 // TODO enable build hook on Windows -# include "nix/store/build/hook-instance.hh" -# include "nix/store/build/derivation-builder.hh" -#endif -#include "nix/util/processes.hh" -#include "nix/util/config-global.hh" #include "nix/store/build/worker.hh" #include "nix/util/util.hh" -#include "nix/util/compression.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" // Don't remove is actually needed #include "nix/store/outputs-query.hh" -#include "nix/store/globals.hh" -#include #include #include #include #include -#include "nix/util/strings.hh" - namespace nix { DerivationGoal::DerivationGoal( diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 212c767a4cf1..ed653fed7555 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -1,9 +1,7 @@ #include "goal-impl.hh" #include "nix/store/build/drv-output-substitution-goal.hh" -#include "nix/util/finally.hh" #include "nix/store/build/worker.hh" -#include "nix/store/build/substitution-goal.hh" #include "nix/util/callback.hh" namespace nix { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index e6d1f62d786a..5b97966847b8 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -2,7 +2,6 @@ #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/build/derivation-trampoline-goal.hh" -#include "nix/store/local-store.hh" #include "nix/util/strings.hh" namespace nix { diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 2c1573b82ca9..4cb42975fe29 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -3,12 +3,9 @@ #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/nar-info.hh" -#include "nix/util/finally.hh" +#include "nix/store/worker-settings.hh" #include "nix/util/signals.hh" #include "nix/util/callback.hh" -#include "nix/store/globals.hh" - -#include namespace nix { diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 767fc9b8f057..d412adbb621f 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -1,7 +1,6 @@ #include "nix/store/builtins.hh" #include "nix/store/filetransfer.hh" #include "nix/store/store-api.hh" -#include "nix/store/globals.hh" #include "nix/util/archive.hh" #include "nix/util/compression.hh" #include "nix/util/file-system.hh" diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index fc88ee889056..dfddd042df81 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -1,11 +1,10 @@ #include "nix/util/serialise.hh" #include "nix/store/path-with-outputs.hh" -#include "nix/store/store-api.hh" #include "nix/store/build-result.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" -#include "nix/util/archive.hh" #include "nix/store/derivations.hh" +#include "nix/store/store-dir-config.hh" #include "nix/util/signature/local-keys.hh" #include diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 08802e953ed1..31d30097fceb 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -3,7 +3,6 @@ #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-connection.hh" #include "nix/store/worker-protocol-impl.hh" -#include "nix/store/build-result.hh" #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" #include "nix/store/filetransfer.hh" @@ -16,7 +15,6 @@ #include "nix/util/archive.hh" #include "nix/store/derivations.hh" #include "nix/util/args.hh" -#include "nix/util/git.hh" #include "nix/util/logging.hh" #include "nix/store/globals.hh" diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 86543bc3632c..b3d4261b5464 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -6,7 +6,6 @@ #include "nix/store/store-api.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" -#include "nix/util/variant-wrapper.hh" #include #include diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b618e9818023..8475532bca33 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1,10 +1,8 @@ #include "nix/store/derivations.hh" #include "nix/store/downstream-placeholder.hh" #include "nix/store/store-api.hh" -#include "nix/store/globals.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" -#include "nix/util/split.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" #include "nix/util/strings-inline.hh" diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 251e112514e1..131674aa5595 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,11 +1,9 @@ #include "nix/store/derived-path.hh" #include "nix/store/derivations.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-dir-config.hh" #include "nix/util/comparator.hh" #include "nix/util/json-utils.hh" -#include - namespace nix { // Custom implementation to avoid `ref` ptr equality diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 88853620facc..85bc650dc8d4 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -2,13 +2,11 @@ #include "nix/store/filetransfer-impl.hh" #include "nix/store/globals.hh" #include "nix/util/config-global.hh" -#include "nix/store/store-api.hh" #include "nix/util/finally.hh" #include "nix/util/callback.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" -#include "store-config-private.hh" #include "nix/store/s3-url.hh" #include #if NIX_WITH_AWS_AUTH diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index a88fe6a64acb..f3bded97e4fc 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,9 +1,9 @@ -#include "nix/store/derivations.hh" -#include "nix/store/globals.hh" #include "nix/store/local-gc.hh" +#include "nix/store/local-settings.hh" #include "nix/store/local-store.hh" #include "nix/store/path.hh" #include "nix/util/configuration.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/finally.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/signals.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8427bffaf601..8beb68aa8e33 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -4,12 +4,10 @@ #include "nix/util/config-global.hh" #include "nix/util/current-process.hh" #include "nix/util/executable-path.hh" -#include "nix/util/archive.hh" #include "nix/util/args.hh" #include "nix/util/abstract-setting-to-json.hh" #include "nix/util/compute-levels.hh" #include "nix/util/executable-path.hh" -#include "nix/util/signals.hh" #include "nix/store/filetransfer.hh" #include @@ -34,8 +32,6 @@ # include "nix/util/processes.hh" #endif -#include "nix/util/config-impl.hh" - #ifdef __APPLE__ # include #endif diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b79e271c27cd..60298efdbbfd 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -1,6 +1,5 @@ #include "nix/store/local-binary-cache-store.hh" -#include "nix/store/globals.hh" -#include "nix/store/nar-info-disk-cache.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/signals.hh" #include "nix/store/store-registration.hh" diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index c58b90d3231b..9f5864ee4425 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,8 +1,6 @@ -#include "nix/util/archive.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" -#include "nix/store/globals.hh" #include "nix/util/compression.hh" #include "nix/store/derivations.hh" diff --git a/src/libstore/local-gc.cc b/src/libstore/local-gc.cc index e3c74f35752b..8bbbdc8366ac 100644 --- a/src/libstore/local-gc.cc +++ b/src/libstore/local-gc.cc @@ -2,7 +2,6 @@ #include "nix/store/store-dir-config.hh" #include "nix/util/file-system.hh" #include "nix/util/signals.hh" -#include "nix/util/types.hh" #include "nix/store/local-gc.hh" #include #include diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 89c5a42d8ab7..afbd47b5bd60 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -5,7 +5,6 @@ #include "nix/util/os-string.hh" #include "nix/store/realisation.hh" #include "nix/util/processes.hh" -#include "nix/util/url.hh" #include "nix/store/store-open.hh" #include "nix/store/store-registration.hh" diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 00a4d4d1fe29..17864ca5ace6 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,6 +1,5 @@ #include "nix/util/base-n.hh" #include "nix/store/machines.hh" -#include "nix/store/globals.hh" #include "nix/store/store-open.hh" #include diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index f446b1cf005c..51708a2cbce4 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,12 +1,10 @@ #include "nix/store/derivations.hh" -#include "nix/util/fun.hh" #include "nix/store/outputs-query.hh" #include "nix/store/parsed-derivations.hh" #include "nix/store/derivation-options.hh" #include "nix/store/globals.hh" #include "nix/store/store-open.hh" #include "nix/store/nar-info.hh" -#include "nix/util/thread-pool.hh" #include "nix/store/realisation.hh" #include "nix/util/topo-sort.hh" #include "nix/util/callback.hh" diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index b912467d8a83..d470569ea294 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,6 +1,5 @@ -#include "nix/store/globals.hh" #include "nix/store/nar-info.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-dir-config.hh" #include "nix/util/strings.hh" #include "nix/util/json-utils.hh" diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 7d37dbf9d4a2..220e37fe9efe 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,5 +1,5 @@ #include "nix/store/local-store.hh" -#include "nix/store/globals.hh" +#include "nix/store/local-settings.hh" #include "nix/util/signals.hh" #include "nix/store/posix-fs-canonicalise.hh" #include "nix/util/posix-source-accessor.hh" @@ -11,8 +11,6 @@ #include #include #include -#include -#include #include "store-config-private.hh" diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 409f6de6479f..78b64a210176 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -1,5 +1,4 @@ #include "nix/store/path-references.hh" -#include "nix/util/hash.hh" #include "nix/util/archive.hh" #include "nix/util/source-accessor.hh" #include "nix/util/canon-path.hh" @@ -7,9 +6,6 @@ #include #include -#include -#include -#include namespace nix { diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 4309ceac5fa8..e489a8054b8e 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,5 +1,3 @@ -#include - #include "nix/store/path-with-outputs.hh" #include "nix/store/store-api.hh" #include "nix/util/strings.hh" diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index a8e828655af9..d20fe91b05e0 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,7 +1,5 @@ #include "nix/store/pathlocks.hh" #include "nix/util/util.hh" -#include "nix/util/sync.hh" -#include "nix/util/signals.hh" #include #include diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index 455cc7c4f5da..9317d30f4e20 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -2,10 +2,7 @@ #include "nix/store/build-result.hh" #include "nix/util/file-system.hh" #include "nix/util/signals.hh" -#include "nix/util/util.hh" #include "nix/store/store-api.hh" -#include "nix/store/globals.hh" -#include "store-config-private.hh" #if NIX_SUPPORT_ACL # include diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 4a60ee559611..519c2abc9805 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,15 +1,12 @@ #include "nix/store/profiles.hh" #include "nix/util/signals.hh" #include "nix/store/globals.hh" -#include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" #include "nix/util/users.hh" #include #include #include -#include -#include namespace nix { diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 1620af26e4df..4bc15a71de9d 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -3,9 +3,7 @@ #include "nix/util/hash.hh" #include "nix/util/base-nix-32.hh" -#include #include -#include #include namespace nix { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 2562e65d88e1..3bb5b0755779 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -13,7 +13,6 @@ #include "nix/store/derivations.hh" #include "nix/util/pool.hh" #include "nix/util/finally.hh" -#include "nix/util/git.hh" #include "nix/util/logging.hh" #include "nix/util/callback.hh" #include "nix/store/filetransfer.hh" diff --git a/src/libstore/s3-url.cc b/src/libstore/s3-url.cc index e6b5553661a6..d1530ceb0daa 100644 --- a/src/libstore/s3-url.cc +++ b/src/libstore/s3-url.cc @@ -4,7 +4,6 @@ #include "nix/util/error.hh" #include "nix/util/logging.hh" #include "nix/util/json-impls.hh" -#include "nix/util/split.hh" #include "nix/util/strings-inline.hh" #include diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index bddd17da8b60..8099f7b8ba08 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -5,7 +5,6 @@ #include "nix/store/common-protocol.hh" #include "nix/store/serve-protocol.hh" #include "nix/store/serve-protocol-impl.hh" -#include "nix/util/archive.hh" #include "nix/store/path-info.hh" #include "nix/util/json-utils.hh" diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 5f6119a427d0..3fd1f798ea74 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,5 +1,5 @@ #include "nix/store/sqlite.hh" -#include "nix/store/globals.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/util.hh" #include "nix/util/url.hh" #include "nix/util/signals.hh" @@ -10,7 +10,6 @@ #include -#include #include namespace nix { diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 9fe1fdbe3f5d..945271b1a929 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -2,7 +2,6 @@ #include "nix/store/local-fs-store.hh" #include "nix/store/remote-store-connection.hh" #include "nix/util/source-accessor.hh" -#include "nix/util/archive.hh" #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-impl.hh" #include "nix/util/pool.hh" diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 0240f9cca322..30ed7397cba7 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,5 +1,4 @@ #include "nix/store/ssh.hh" -#include "nix/util/finally.hh" #include "nix/util/current-process.hh" #include "nix/util/environment-variables.hh" #include "nix/util/os-string.hh" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f465c959c8dd..3f367b19a16c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -24,8 +24,6 @@ #include #include -#include "nix/util/strings.hh" - #ifdef _WIN32 # include "nix/util/windows-known-folders.hh" #endif diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 1b3c09ba4af4..e0b2d68ec37c 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,8 +1,8 @@ #include "nix/store/uds-remote-store.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/store/worker-protocol.hh" #include "nix/store/store-registration.hh" -#include "nix/store/globals.hh" #include #include diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc index 47e33853c920..5dc6cd236c42 100644 --- a/src/libstore/unix/pathlocks.cc +++ b/src/libstore/unix/pathlocks.cc @@ -1,7 +1,5 @@ #include "nix/store/pathlocks.hh" #include "nix/util/file-system-at.hh" -#include "nix/util/util.hh" -#include "nix/util/sync.hh" #include "nix/util/signals.hh" #include diff --git a/src/libstore/unix/user-lock.cc b/src/libstore/unix/user-lock.cc index c9abdddcea8a..d5a959be2e5f 100644 --- a/src/libstore/unix/user-lock.cc +++ b/src/libstore/unix/user-lock.cc @@ -3,8 +3,8 @@ #include #include "nix/store/user-lock.hh" +#include "nix/store/local-settings.hh" #include "nix/util/file-system.hh" -#include "nix/store/globals.hh" #include "nix/store/pathlocks.hh" #include "nix/util/users.hh" #include "nix/util/logging.hh" diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 3da966a62019..7f9bd163d91f 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -6,7 +6,6 @@ #include "nix/store/common-protocol.hh" #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-impl.hh" -#include "nix/util/archive.hh" #include "nix/store/path-info.hh" #include "nix/util/json-utils.hh" diff --git a/src/libutil-test-support/hash.cc b/src/libutil-test-support/hash.cc index 853da8e90778..fdc95a6f8123 100644 --- a/src/libutil-test-support/hash.cc +++ b/src/libutil-test-support/hash.cc @@ -1,6 +1,6 @@ -#include - -#include // Needed by rapidcheck on Darwin +#ifdef __APPLE__ +# include // Needed by rapidcheck on Darwin +#endif #include #include "nix/util/hash.hh" diff --git a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh index d0f4f9c495ad..9bd4d7fbf109 100644 --- a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh @@ -7,6 +7,7 @@ #include "nix/util/types.hh" #include "nix/util/ref.hh" #include "nix/util/file-system.hh" +#include "nix/util/nar-accessor.hh" #include "nix/util/tests/characterization.hh" diff --git a/src/libutil-tests/args.cc b/src/libutil-tests/args.cc index 7aa996233acd..f2af485bf434 100644 --- a/src/libutil-tests/args.cc +++ b/src/libutil-tests/args.cc @@ -1,5 +1,4 @@ #include "nix/util/args.hh" -#include "nix/util/fs-sink.hh" #include #include diff --git a/src/libutil-tests/config.cc b/src/libutil-tests/config.cc index 6e90dcf11bbe..b9eec0c4175f 100644 --- a/src/libutil-tests/config.cc +++ b/src/libutil-tests/config.cc @@ -1,7 +1,5 @@ #include "nix/util/configuration.hh" -#include "nix/util/args.hh" -#include #include #include diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 95bfa16b140c..995b75814f01 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -1,17 +1,10 @@ -#include "nix/util/util.hh" #include "nix/util/serialise.hh" -#include "nix/util/types.hh" #include "nix/util/file-system.hh" -#include "nix/util/processes.hh" -#include "nix/util/terminal.hh" -#include "nix/util/strings.hh" #include #include #include -#include - using namespace std::string_view_literals; #ifdef _WIN32 diff --git a/src/libutil-tests/hash.cc b/src/libutil-tests/hash.cc index a6bb52d1953f..51924f47f969 100644 --- a/src/libutil-tests/hash.cc +++ b/src/libutil-tests/hash.cc @@ -1,5 +1,3 @@ -#include - #include #include diff --git a/src/libutil-tests/monitorfdhup.cc b/src/libutil-tests/monitorfdhup.cc index 02cd7e22ca98..f8c94dcdce76 100644 --- a/src/libutil-tests/monitorfdhup.cc +++ b/src/libutil-tests/monitorfdhup.cc @@ -1,7 +1,6 @@ // TODO: investigate why this is hanging on cygwin #if !defined(_WIN32) && !defined(__CYGWIN__) -# include "nix/util/util.hh" # include "nix/util/monitor-fd.hh" # include diff --git a/src/libutil-tests/nar-listing.cc b/src/libutil-tests/nar-listing.cc index a2b8650481c9..3d2fbe975727 100644 --- a/src/libutil-tests/nar-listing.cc +++ b/src/libutil-tests/nar-listing.cc @@ -1,6 +1,5 @@ #include -#include "nix/util/nar-accessor.hh" #include "nix/util/tests/json-characterization.hh" namespace nix { diff --git a/src/libutil-tests/nix_api_util.cc b/src/libutil-tests/nix_api_util.cc index 48f85c403d65..14e23138b7ce 100644 --- a/src/libutil-tests/nix_api_util.cc +++ b/src/libutil-tests/nix_api_util.cc @@ -1,5 +1,4 @@ #include "nix/util/config-global.hh" -#include "nix/util/args.hh" #include "nix_api_util.h" #include "nix/util/tests/nix_api_util.hh" #include "nix/util/tests/string_callback.hh" diff --git a/src/libutil-tests/nix_api_util_internal.cc b/src/libutil-tests/nix_api_util_internal.cc index 6fb0a623f665..ad61574186dd 100644 --- a/src/libutil-tests/nix_api_util_internal.cc +++ b/src/libutil-tests/nix_api_util_internal.cc @@ -1,5 +1,3 @@ -#include "nix/util/config-global.hh" -#include "nix/util/args.hh" #include "nix_api_util.h" #include "nix_api_util_internal.h" #include "nix/util/tests/nix_api_util.hh" @@ -9,8 +7,6 @@ #include -#include "util-tests-config.hh" - namespace nixC { TEST_F(nix_api_util_context, nix_context_error) diff --git a/src/libutil-tests/terminal.cc b/src/libutil-tests/terminal.cc index 198b209515e5..7f63dfdc58cc 100644 --- a/src/libutil-tests/terminal.cc +++ b/src/libutil-tests/terminal.cc @@ -1,13 +1,8 @@ -#include "nix/util/util.hh" -#include "nix/util/types.hh" #include "nix/util/terminal.hh" -#include "nix/util/strings.hh" #include #include -#include - namespace nix { /* ---------------------------------------------------------------------------- diff --git a/src/libutil-tests/topo-sort.cc b/src/libutil-tests/topo-sort.cc index 91030247e67b..eadbe3e506b2 100644 --- a/src/libutil-tests/topo-sort.cc +++ b/src/libutil-tests/topo-sort.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include diff --git a/src/libutil-tests/util.cc b/src/libutil-tests/util.cc index a299cd978232..031dd7a444af 100644 --- a/src/libutil-tests/util.cc +++ b/src/libutil-tests/util.cc @@ -1,15 +1,9 @@ #include "nix/util/util.hh" #include "nix/util/types.hh" -#include "nix/util/file-system.hh" -#include "nix/util/terminal.hh" -#include "nix/util/strings.hh" -#include "nix/util/base-n.hh" #include #include -#include - namespace nix { /* ----------- tests for util.hh --------------------------------------------*/ diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 53aebde70aa1..f704acdf78b4 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -1,6 +1,4 @@ #include -#include -#include #include #include // for strcasecmp diff --git a/src/libutil/args.cc b/src/libutil/args.cc index c025f119f725..0fa356f68907 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -2,7 +2,6 @@ #include "nix/util/args/root.hh" #include "nix/util/hash.hh" #include "nix/util/environment-variables.hh" -#include "nix/util/signals.hh" #include "nix/util/users.hh" #include "nix/util/json-utils.hh" diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index cf2e26ba9748..f07b22dcab2e 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,7 +1,6 @@ #include "nix/util/compression.hh" #include "nix/util/signals.hh" #include "nix/util/tarfile.hh" -#include "nix/util/finally.hh" #include "nix/util/logging.hh" #include diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 37afda89bb4b..177728a63343 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -3,9 +3,7 @@ #include "nix/util/current-process.hh" #include "nix/util/util.hh" -#include "nix/util/finally.hh" #include "nix/util/file-system.hh" -#include "nix/util/processes.hh" #include "nix/util/signals.hh" #include "nix/util/environment-variables.hh" #include @@ -15,7 +13,6 @@ #endif #ifdef __linux__ -# include # include "nix/util/cgroup.hh" # include "nix/util/linux-namespaces.hh" #endif diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc index 9d44ad65d8db..8c99e6715597 100644 --- a/src/libutil/environment-variables.cc +++ b/src/libutil/environment-variables.cc @@ -1,4 +1,3 @@ -#include "nix/util/util.hh" #include "nix/util/environment-variables.hh" namespace nix { diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e36f46ee803c..f43df1de5209 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -10,7 +10,6 @@ #include #include #include -#include "nix/util/serialise.hh" #include namespace nix { diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index ec3520fd0ca6..4da0999715ea 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -1,7 +1,6 @@ #include "nix/util/environment-variables.hh" #include "nix/util/executable-path.hh" #include "nix/util/strings-inline.hh" -#include "nix/util/util.hh" #include "nix/util/file-path-impl.hh" namespace nix { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index dfba75e50066..b7e504e13a05 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -1,4 +1,3 @@ -#include "nix/util/environment-variables.hh" #include "nix/util/file-system.hh" #include "nix/util/file-path.hh" #include "nix/util/file-path-impl.hh" diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 2646babfd50c..96c6dd28791d 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -1,7 +1,5 @@ #include #include -#include -#include #include #include // for strcasecmp diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 1f89114c66be..2832de73c9f2 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -1,4 +1,3 @@ -#include #include #include @@ -8,7 +7,6 @@ #include "nix/util/args.hh" #include "nix/util/hash.hh" -#include "nix/util/archive.hh" #include "nix/util/configuration.hh" #include "nix/util/split.hh" #include "nix/util/base-n.hh" diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index 39dad9268bcb..9c96c5bcb768 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -1,14 +1,10 @@ #include "nix/util/linux-namespaces.hh" -#include "nix/util/current-process.hh" #include "nix/util/util.hh" -#include "nix/util/finally.hh" #include "nix/util/file-system.hh" #include "nix/util/processes.hh" -#include "nix/util/signals.hh" #include #include -#include "nix/util/cgroup.hh" #include diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index f1c8190f7db3..58c78df34986 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -4,7 +4,6 @@ #include "nix/util/terminal.hh" #include "nix/util/util.hh" #include "nix/util/config-global.hh" -#include "nix/util/source-path.hh" #include "nix/util/position.hh" #include "nix/util/sync.hh" #include "nix/util/unix-domain-socket.hh" @@ -12,7 +11,6 @@ #include #include #include -#include namespace nix { diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index f0f7a952296a..6657291bb041 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -1,5 +1,4 @@ #include "nix/util/memory-source-accessor.hh" -#include "nix/util/json-utils.hh" #include diff --git a/src/libutil/nar-accessor.cc b/src/libutil/nar-accessor.cc index 6404128e9d48..f27d1390f915 100644 --- a/src/libutil/nar-accessor.cc +++ b/src/libutil/nar-accessor.cc @@ -1,7 +1,6 @@ #include "nix/util/nar-accessor.hh" #include "nix/util/file-descriptor.hh" #include "nix/util/error.hh" -#include "nix/util/signals.hh" namespace nix { diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 45af4e88ffd6..609855281389 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -3,7 +3,6 @@ #include "nix/util/memory-source-accessor.hh" #include "nix/util/source-path.hh" #include "nix/util/signals.hh" -#include "nix/util/sync.hh" #include diff --git a/src/libutil/signature/signer.cc b/src/libutil/signature/signer.cc index fff03fc30db1..9294b2a1802d 100644 --- a/src/libutil/signature/signer.cc +++ b/src/libutil/signature/signer.cc @@ -1,5 +1,4 @@ #include "nix/util/signature/signer.hh" -#include "nix/util/error.hh" #include diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 91a0f73ec1db..83ccf8b3968a 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -1,9 +1,6 @@ -#include #include -#include #include "nix/util/strings-inline.hh" -#include "nix/util/os-string.hh" #include "nix/util/error.hh" #include "nix/util/util.hh" diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 2367a12bf693..778b17cf0b2d 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -3,7 +3,6 @@ #include "nix/util/terminal.hh" #include -#include namespace nix { diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index 48d692872712..ebdc783e4abb 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -13,7 +13,6 @@ #endif #include #include -#include #include // for ptsname and ptsname_r namespace { diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index 21339afe882b..317bc33fff74 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -1,14 +1,11 @@ #include "nix/util/file-system.hh" #include "nix/util/file-system-at.hh" #include "nix/util/signals.hh" -#include "nix/util/finally.hh" -#include "nix/util/serialise.hh" #include #include #include -#include "util-config-private.hh" #include "util-unix-config-private.hh" namespace nix { diff --git a/src/libutil/unix/file-path.cc b/src/libutil/unix/file-path.cc index 55ccddb45683..01e89e14372e 100644 --- a/src/libutil/unix/file-path.cc +++ b/src/libutil/unix/file-path.cc @@ -1,10 +1,4 @@ -#include -#include -#include -#include - #include "nix/util/file-path.hh" -#include "nix/util/util.hh" namespace nix { diff --git a/src/libutil/unix/muxable-pipe.cc b/src/libutil/unix/muxable-pipe.cc index 12c971f94d7c..3362fc34a675 100644 --- a/src/libutil/unix/muxable-pipe.cc +++ b/src/libutil/unix/muxable-pipe.cc @@ -1,6 +1,5 @@ #include -#include "nix/util/logging.hh" #include "nix/util/util.hh" #include "nix/util/muxable-pipe.hh" diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 9ae42498ded5..9f23da876aaa 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -13,7 +13,6 @@ #include #include #include -#include #include using namespace std::chrono_literals; @@ -31,7 +30,6 @@ using namespace std::chrono_literals; # include #endif -#include "util-config-private.hh" #include "util-unix-config-private.hh" namespace nix { diff --git a/src/libutil/unix/users.cc b/src/libutil/unix/users.cc index 5d05dc79f36f..24ef6151c84f 100644 --- a/src/libutil/unix/users.cc +++ b/src/libutil/unix/users.cc @@ -1,7 +1,7 @@ -#include "nix/util/util.hh" #include "nix/util/users.hh" #include "nix/util/environment-variables.hh" #include "nix/util/file-system.hh" +#include "nix/util/logging.hh" #include #include diff --git a/src/libutil/unix/xdg-dirs.cc b/src/libutil/unix/xdg-dirs.cc index cc66f59f3a93..4384583cad44 100644 --- a/src/libutil/unix/xdg-dirs.cc +++ b/src/libutil/unix/xdg-dirs.cc @@ -1,4 +1,4 @@ -#include "nix/util/util.hh" +#include "nix/util/strings.hh" #include "nix/util/users.hh" #include "nix/util/environment-variables.hh" diff --git a/src/libutil/users.cc b/src/libutil/users.cc index f05dfcf760e2..cc3fbb0e5440 100644 --- a/src/libutil/users.cc +++ b/src/libutil/users.cc @@ -1,7 +1,5 @@ -#include "nix/util/util.hh" #include "nix/util/users.hh" #include "nix/util/environment-variables.hh" -#include "nix/util/executable-path.hh" #include "nix/util/file-system.hh" #ifndef _WIN32 diff --git a/src/libutil/util.cc b/src/libutil/util.cc index d75aa4d67d94..8ed058d76112 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,12 +1,9 @@ #include "nix/util/util.hh" #include "nix/util/fmt.hh" -#include "nix/util/file-path.hh" #include "nix/util/signals.hh" #include #include -#include -#include #include #include diff --git a/src/libutil/windows/current-process.cc b/src/libutil/windows/current-process.cc index 3d252257e197..b6805417be9c 100644 --- a/src/libutil/windows/current-process.cc +++ b/src/libutil/windows/current-process.cc @@ -1,5 +1,4 @@ #include "nix/util/current-process.hh" -#include "nix/util/error.hh" #include #ifdef _WIN32 diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 134d31fac357..2a24c4199f43 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -1,8 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" #include "nix/store/store-api.hh" -#include "nix/util/archive.hh" -#include "nix/util/git.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/cmd/misc-store-flags.hh" diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index 332a3955d87a..6436466374bd 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -5,7 +5,6 @@ #include #include #include -#include #ifdef __APPLE__ # include #endif diff --git a/src/nix/build.cc b/src/nix/build.cc index 4d2adfd5a6c9..2de7fdf42362 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -2,7 +2,6 @@ #include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" -#include "nix/store/local-fs-store.hh" #include diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index a9266a11bde4..85c8cdb58759 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -1,6 +1,5 @@ #include "nix/cmd/installable-flake.hh" #include "nix/cmd/command-installable-value.hh" -#include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 09416b1f44e7..8a816f528815 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,7 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" #include "nix/util/archive.hh" -#include "nix/util/nar-accessor.hh" #include "nix/util/serialise.hh" #include "nix/util/source-accessor.hh" diff --git a/src/nix/config.cc b/src/nix/config.cc index c2a9fd8e2fe8..3fd0cde808f7 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -1,7 +1,5 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" -#include "nix/main/shared.hh" -#include "nix/store/store-api.hh" #include "nix/util/config-global.hh" #include diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc index 2dbc89d6f8e7..2046ecca3892 100644 --- a/src/nix/derivation-add.cc +++ b/src/nix/derivation-add.cc @@ -3,7 +3,6 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" #include "nix/store/store-api.hh" -#include "nix/util/archive.hh" #include "nix/store/derivations.hh" #include "nix/store/globals.hh" #include diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index ce2594ddcab6..1a6f00c8cdee 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -4,7 +4,6 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" #include "nix/store/store-api.hh" -#include "nix/util/archive.hh" #include "nix/store/derivations.hh" #include diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 29b44d780906..301e35b55ffb 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,7 +1,7 @@ +#include "nix/cmd/command.hh" #include "nix/util/config-global.hh" #include "nix/expr/eval.hh" #include "nix/cmd/installable-flake.hh" -#include "nix/cmd/command-installable-value.hh" #include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" @@ -14,7 +14,6 @@ # include "run.hh" #endif -#include #include #include #include diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index d36a21d746ff..ff37100651cf 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -1,7 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" -#include "nix/main/common-args.hh" #include "nix/store/names.hh" #include diff --git a/src/nix/formatter.cc b/src/nix/formatter.cc index 08f0b5f053ea..619dcf4fad0b 100644 --- a/src/nix/formatter.cc +++ b/src/nix/formatter.cc @@ -2,8 +2,6 @@ #include "nix/cmd/installable-flake.hh" #include "nix/cmd/installable-value.hh" #include "nix/expr/eval.hh" -#include "nix/store/local-fs-store.hh" -#include "nix/cmd/installable-derived-path.hh" #include "nix/util/environment-variables.hh" #include "nix/store/globals.hh" diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 43ace0d36cb3..bc1ce313b08d 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -1,10 +1,8 @@ #include "nix/cmd/command.hh" #include "nix/util/hash.hh" -#include "nix/store/content-address.hh" #include "nix/cmd/legacy.hh" #include "nix/main/shared.hh" #include "nix/store/references.hh" -#include "nix/util/archive.hh" #include "nix/util/git.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/cmd/misc-store-flags.hh" diff --git a/src/nix/log.cc b/src/nix/log.cc index 8f251ddab14d..acc03cbb6783 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -1,6 +1,5 @@ #include "nix/cmd/command.hh" #include "nix/cmd/get-build-log.hh" -#include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/globals.hh" diff --git a/src/nix/main.cc b/src/nix/main.cc index 297318ddcd7f..c19aa26925d2 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -22,7 +22,6 @@ #include "nix/expr/eval-cache.hh" #include "nix/flake/flake.hh" #include "nix/flake/settings.hh" -#include "nix/util/json-utils.hh" #include "self-exe.hh" #include "crash-handler.hh" diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index 6f691faeb48a..d9e2f2dc8d41 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -1,5 +1,4 @@ #include -#include #include #include #include diff --git a/src/nix/nix-channel/nix-channel.cc b/src/nix/nix-channel/nix-channel.cc index 05adc3d2cda8..911aabd43833 100644 --- a/src/nix/nix-channel/nix-channel.cc +++ b/src/nix/nix-channel/nix-channel.cc @@ -9,7 +9,6 @@ #include "nix/util/os-string.hh" #include "nix/util/users.hh" #include "nix/fetchers/tarball.hh" -#include "nix/fetchers/fetch-settings.hh" #include "self-exe.hh" #include "man-pages.hh" diff --git a/src/nix/nix-collect-garbage/nix-collect-garbage.cc b/src/nix/nix-collect-garbage/nix-collect-garbage.cc index 3d84ca13cd99..a9f8b7fb6ba1 100644 --- a/src/nix/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix/nix-collect-garbage/nix-collect-garbage.cc @@ -10,7 +10,6 @@ #include "nix/cmd/legacy.hh" #include "man-pages.hh" -#include #include using namespace nix; diff --git a/src/nix/nix-env/user-env.cc b/src/nix/nix-env/user-env.cc index f87e7f6103a9..cb74a36d0d4c 100644 --- a/src/nix/nix-env/user-env.cc +++ b/src/nix/nix-env/user-env.cc @@ -3,7 +3,6 @@ #include "nix/store/store-api.hh" #include "nix/store/path-with-outputs.hh" #include "nix/store/local-fs-store.hh" -#include "nix/store/globals.hh" #include "nix/main/shared.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-inline.hh" @@ -11,7 +10,6 @@ #include "nix/expr/print-ambiguous.hh" #include "nix/expr/static-string-data.hh" -#include #include namespace nix { diff --git a/src/nix/nix-instantiate/nix-instantiate.cc b/src/nix/nix-instantiate/nix-instantiate.cc index 82838e2ae8ae..b368346eea93 100644 --- a/src/nix/nix-instantiate/nix-instantiate.cc +++ b/src/nix/nix-instantiate/nix-instantiate.cc @@ -5,7 +5,6 @@ #include "nix/expr/eval-inline.hh" #include "nix/expr/get-drvs.hh" #include "nix/expr/attr-path.hh" -#include "nix/util/signals.hh" #include "nix/expr/value-to-xml.hh" #include "nix/expr/value-to-json.hh" #include "nix/store/store-open.hh" @@ -14,7 +13,6 @@ #include "nix/cmd/legacy.hh" #include "man-pages.hh" -#include #include using namespace nix; diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index e932690679e5..a534e19afb9f 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -35,7 +35,6 @@ #include #include -#include "nix/store/build-result.hh" #include "nix/util/exit.hh" #include "nix/store/serve-protocol-impl.hh" diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index e000026fcc64..5edbabc3bd10 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -2,8 +2,6 @@ #include "nix/main/shared.hh" #include "nix/store/store-api.hh" -#include - using namespace nix; struct CmdOptimiseStore : StoreCommand diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 3e36197b958d..9abe38bfdde6 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -5,7 +5,6 @@ #include "nix/store/nar-info.hh" #include -#include #include diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3a7e448cda24..1ace94a8b782 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -3,7 +3,6 @@ #include "nix/main/shared.hh" #include "nix/store/store-open.hh" #include "nix/store/filetransfer.hh" -#include "nix/util/finally.hh" #include "nix/main/loggers.hh" #include "nix/util/tarfile.hh" #include "nix/expr/attr-path.hh" @@ -11,7 +10,6 @@ #include "nix/cmd/legacy.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/cmd/misc-store-flags.hh" -#include "nix/util/terminal.hh" #include "nix/util/environment-variables.hh" #include "nix/util/url.hh" #include "nix/store/path.hh" diff --git a/src/nix/registry.cc b/src/nix/registry.cc index c943e80f9e1c..5f77700d65a8 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -1,8 +1,6 @@ #include "nix/cmd/command.hh" -#include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/expr/eval.hh" -#include "nix/flake/flake.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/registry.hh" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index ea1ecb305aa0..85192faa6490 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,8 +1,6 @@ #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" #include "nix/util/config-global.hh" -#include "nix/store/globals.hh" -#include "nix/store/store-open.hh" #include "nix/cmd/command.hh" #include "nix/cmd/installable-value.hh" #include "nix/cmd/repl.hh" diff --git a/src/nix/run.cc b/src/nix/run.cc index 38dbc42b6815..a87a7679d435 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,14 +1,12 @@ #include "nix/util/current-process.hh" #include "run.hh" #include "nix/cmd/command-installable-value.hh" -#include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/util/signals.hh" #include "nix/store/store-api.hh" #include "nix/store/derivations.hh" #include "nix/store/local-fs-store.hh" #include "nix/util/finally.hh" -#include "nix/util/source-accessor.hh" #include "nix/expr/eval.hh" #include "nix/util/util.hh" #include "nix/store/globals.hh" @@ -20,8 +18,6 @@ # include "nix/store/personality.hh" #endif -#include - extern char ** environ __attribute__((weak)); using namespace nix; diff --git a/src/nix/search.cc b/src/nix/search.cc index dac60ceba573..4f5ae8cafaa2 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -4,7 +4,6 @@ #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" #include "nix/store/names.hh" -#include "nix/expr/get-drvs.hh" #include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/expr/eval-cache.hh" @@ -13,7 +12,6 @@ #include "nix/util/strings-inline.hh" #include -#include #include #include "nix/util/strings.hh" diff --git a/src/nix/self-exe.cc b/src/nix/self-exe.cc index 36f6e17ec8b6..d7991810015e 100644 --- a/src/nix/self-exe.cc +++ b/src/nix/self-exe.cc @@ -1,6 +1,5 @@ #include "nix/util/current-process.hh" -#include "nix/util/file-system.hh" -#include "nix/store/globals.hh" +#include "nix/util/environment-variables.hh" #include "self-exe.hh" #include "cli-config-private.hh" diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index 6e442f3713c1..cbd4e04f6aa6 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -3,10 +3,6 @@ #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" #include "nix/store/log-store.hh" -#include "nix/util/sync.hh" -#include "nix/util/thread-pool.hh" - -#include using namespace nix; diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 3e2e0f2ba517..5d28979a493c 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -1,5 +1,4 @@ #include "nix/cmd/command.hh" -#include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index 971be26be5b3..f4624a40e8af 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -5,7 +5,6 @@ #include "nix/store/store-cast.hh" #include "nix/store/gc-store.hh" #include "nix/util/error.hh" -#include "nix/util/logging.hh" using namespace nix; diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index a8cfb089fc28..278eaef374d5 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -1,25 +1,22 @@ ///@file #include "nix/util/signals.hh" -#include "nix/util/unix-domain-socket.hh" #include "nix/cmd/command.hh" #include "nix/main/shared.hh" -#include "nix/store/local-fs-store.hh" #include "nix/store/local-store.hh" #include "nix/store/uds-remote-store.hh" #include "nix/store/remote-store.hh" #include "nix/store/remote-store-connection.hh" #include "nix/store/store-open.hh" #include "nix/util/serialise.hh" -#include "nix/util/archive.hh" #include "nix/store/globals.hh" #include "nix/util/config-global.hh" #include "nix/store/derivations.hh" -#include "nix/util/finally.hh" #include "nix/cmd/legacy.hh" #include "nix/cmd/unix-socket-server.hh" #include "nix/store/daemon.hh" #include "man-pages.hh" +#include "nix/util/socket.hh" #include #include From b7c97d2043471b0d447a809b4332e0d3396c5ee5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 9 Apr 2026 16:09:47 -0400 Subject: [PATCH 222/555] `linux::openat2` wrapper return `std::optional` not `std::optional` This is because it is returning a (newly opened) owned not borrowed descriptor. This is a follow up from 89dd96efbfe55d5e56f99b85cb4f67dbf3d548cc when we did the same thing, to the same reason, to other `open*` functions. --- src/libutil/include/nix/util/file-system-at.hh | 3 ++- src/libutil/unix/file-system-at.cc | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh index c78cafe6a559..75f4e6b2cff1 100644 --- a/src/libutil/include/nix/util/file-system-at.hh +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -120,7 +120,8 @@ namespace linux { * * @return nullopt if openat2 is not supported by the kernel. */ -std::optional openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve); +std::optional +openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve); } // namespace linux #endif diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index d8be6fdeb70b..01a4c664d31d 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -29,7 +29,7 @@ namespace nix { namespace linux { -std::optional openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve) +std::optional openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve) { # if HAVE_OPENAT2 /* Cache the result of whether openat2 is not supported. */ @@ -47,7 +47,7 @@ std::optional openat2(Descriptor dirFd, const char * path, uint64_t return std::nullopt; } - return res; + return AutoCloseFD{static_cast(res)}; } # endif return std::nullopt; @@ -202,9 +202,9 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & auto maybeFd = linux::openat2( dirFd, path.rel_c_str(), flags, static_cast(mode), RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS); if (maybeFd) { - if (*maybeFd < 0 && errno == ELOOP) + if (!*maybeFd && errno == ELOOP) throw SymlinkNotAllowed(path); - return AutoCloseFD{*maybeFd}; + return std::move(*maybeFd); } #endif return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode); From 5f1f60ff2b200c1dd2c9f4c6a38ec37a8d29dcd2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 10 Apr 2026 11:33:21 -0400 Subject: [PATCH 223/555] Fix Windows Build This header was accidentally removed in 303fc45c71cb7ac6f366ac8901dab92e428c9355. --- src/libutil/windows/current-process.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/windows/current-process.cc b/src/libutil/windows/current-process.cc index b6805417be9c..3d252257e197 100644 --- a/src/libutil/windows/current-process.cc +++ b/src/libutil/windows/current-process.cc @@ -1,4 +1,5 @@ #include "nix/util/current-process.hh" +#include "nix/util/error.hh" #include #ifdef _WIN32 From f7b59de6217fdc63a784ee6d0f337d45f59ffedf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 10 Apr 2026 11:33:21 -0400 Subject: [PATCH 224/555] Femove a bunch of redundant `#ifdef _WIN32` in Windows-only files This was always redundant, but was formally needed to run `clang-tidy`. Now we have a better `clang-tidy` setup that doesn't need this hack. --- src/libstore/windows/pathlocks.cc | 8 +++----- src/libutil/windows/current-process.cc | 6 ++---- src/libutil/windows/environment-variables.cc | 4 +--- .../windows/include/nix/util/windows-async-pipe.hh | 2 -- src/libutil/windows/muxable-pipe.cc | 10 ++++------ src/libutil/windows/os-string.cc | 4 ---- src/libutil/windows/processes.cc | 8 ++------ src/libutil/windows/users.cc | 6 ++---- src/libutil/windows/windows-async-pipe.cc | 7 +------ src/libutil/windows/windows-error.cc | 11 ++++------- 10 files changed, 19 insertions(+), 47 deletions(-) diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 8021bb072a22..f048986c75be 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -5,10 +5,9 @@ #include "nix/util/util.hh" #include "nix/util/windows-environment.hh" -#ifdef _WIN32 -# include -# include -# include +#include +#include +#include namespace nix { @@ -167,4 +166,3 @@ FdLock::FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view w } } // namespace nix -#endif diff --git a/src/libutil/windows/current-process.cc b/src/libutil/windows/current-process.cc index 3d252257e197..4d1ecdc1d7f0 100644 --- a/src/libutil/windows/current-process.cc +++ b/src/libutil/windows/current-process.cc @@ -2,9 +2,8 @@ #include "nix/util/error.hh" #include -#ifdef _WIN32 -# define WIN32_LEAN_AND_MEAN -# include +#define WIN32_LEAN_AND_MEAN +#include namespace nix { @@ -31,4 +30,3 @@ std::chrono::microseconds getCpuUserTime() } } // namespace nix -#endif // ifdef _WIN32 diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 34825793e433..767c5cd40ef8 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -1,7 +1,6 @@ #include "nix/util/environment-variables.hh" -#ifdef _WIN32 -# include +#include namespace nix { @@ -85,4 +84,3 @@ int setEnvOs(const OsString & name, const OsString & value) } } // namespace nix -#endif diff --git a/src/libutil/windows/include/nix/util/windows-async-pipe.hh b/src/libutil/windows/include/nix/util/windows-async-pipe.hh index e700aa4f63dc..7fd94313a723 100644 --- a/src/libutil/windows/include/nix/util/windows-async-pipe.hh +++ b/src/libutil/windows/include/nix/util/windows-async-pipe.hh @@ -2,7 +2,6 @@ ///@file #include "nix/util/file-descriptor.hh" -#ifdef _WIN32 namespace nix::windows { @@ -26,4 +25,3 @@ public: }; } // namespace nix::windows -#endif diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index 5c427d67aec1..1e89ccc5e364 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -1,9 +1,8 @@ -#ifdef _WIN32 -# include +#include -# include "nix/util/logging.hh" -# include "nix/util/util.hh" -# include "nix/util/muxable-pipe.hh" +#include "nix/util/logging.hh" +#include "nix/util/util.hh" +#include "nix/util/muxable-pipe.hh" namespace nix { @@ -70,4 +69,3 @@ void MuxablePipePollState::iterate( } } // namespace nix -#endif diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index 2fface419552..f0fdb0c02b18 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -5,8 +5,6 @@ #include "nix/util/os-string.hh" -#ifdef _WIN32 - namespace nix { std::string os_string_to_string(OsStringView s) @@ -32,5 +30,3 @@ OsString string_to_os_string(std::string s) } } // namespace nix - -#endif diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 9c7fa5d35ffb..c0b0fa6c10be 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -25,10 +25,8 @@ #include #include -#ifdef _WIN32 - -# define WIN32_LEAN_AND_MEAN -# include +#define WIN32_LEAN_AND_MEAN +#include namespace nix { @@ -404,5 +402,3 @@ int execvpe(const wchar_t * file0, const wchar_t * const argv[], const wchar_t * } } // namespace nix - -#endif diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index caab6745d4f4..e5acd8cbdcfd 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -3,9 +3,8 @@ #include "nix/util/environment-variables.hh" #include "nix/util/file-system.hh" -#ifdef _WIN32 -# define WIN32_LEAN_AND_MEAN -# include +#define WIN32_LEAN_AND_MEAN +#include namespace nix { @@ -50,4 +49,3 @@ bool isRootUser() } } // namespace nix -#endif diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc index 09b72277a918..960dd63f80f0 100644 --- a/src/libutil/windows/windows-async-pipe.cc +++ b/src/libutil/windows/windows-async-pipe.cc @@ -1,7 +1,4 @@ - - -#ifdef _WIN32 -# include "nix/util/windows-async-pipe.hh" +#include "nix/util/windows-async-pipe.hh" namespace nix::windows { @@ -49,5 +46,3 @@ void AsyncPipe::close() } } // namespace nix::windows - -#endif diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index f04b70a2161e..85fcc1c1e50b 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -1,10 +1,8 @@ -#ifdef _WIN32 +#include "nix/util/error.hh" -# include "nix/util/error.hh" - -# include -# define WIN32_LEAN_AND_MEAN -# include +#include +#define WIN32_LEAN_AND_MEAN +#include namespace nix::windows { @@ -33,4 +31,3 @@ std::string WinError::renderError(DWORD lastError) } } // namespace nix::windows -#endif From 00298039678721b4d11fb0796bf7c62fcb859389 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 10 Apr 2026 13:06:58 -0400 Subject: [PATCH 225/555] Replace top-level `using namespace nix...;` Instead of pulling in the `nix` namespace at file scope with `using namespace nix;`, wrap the code in `namespace nix { }` blocks. This is cleaner and avoids polluting the global namespace. Sub-namespace `using namespace nix::*;` declarations are either replaced with explicit qualification (e.g. `linux::getCgroupFS()`) or moved into the functions that need them. All remaining `using namespace` directives now use the fully qualified form (e.g. `using namespace nix::fetchers;` rather than `using namespace fetchers;`). Also remove `using std::cin;` and `using std::cout;` in `nix-store.cc`, qualifying them explicitly as `std::cin`/`std::cout` instead. Some POSIX functions (`read`, `write`) that could collide with `nix::` members now use `::` global scope qualification. (`pread` won't collide, but it doesn't hurt to use `::pread` too for symmetry.) --- src/libexpr/lexer.l | 2 + src/libflake/flake.cc | 2 - src/libflake/flakeref.cc | 2 +- .../unix/build/linux-derivation-builder.cc | 10 ++--- src/libstore/windows/pathlocks.cc | 3 +- src/libutil-test-support/hash.cc | 1 + .../include/nix/util/tests/hash.hh | 1 + src/libutil-tests/checked-arithmetic.cc | 1 + src/libutil-tests/git.cc | 37 ++++++++++------ src/libutil-tests/unix/file-system-at.cc | 6 ++- src/libutil-tests/unix/unix-domain-socket.cc | 6 ++- src/libutil/unix/file-descriptor.cc | 2 +- .../unix/include/nix/util/signals-impl.hh | 2 +- src/libutil/unix/signals.cc | 3 +- src/libutil/windows/file-descriptor.cc | 17 ++++--- src/libutil/windows/file-system-at.cc | 22 +++++----- src/libutil/windows/file-system.cc | 10 ++--- src/libutil/windows/known-folders.cc | 2 - src/libutil/windows/muxable-pipe.cc | 6 +-- src/libutil/windows/processes.cc | 20 ++++----- src/libutil/windows/users.cc | 6 +-- src/nix/add-to-store.cc | 4 +- src/nix/build-remote/build-remote.cc | 6 ++- src/nix/build.cc | 4 +- src/nix/bundle.cc | 4 +- src/nix/cat.cc | 4 +- src/nix/config-check.cc | 4 +- src/nix/config.cc | 4 +- src/nix/copy.cc | 4 +- src/nix/derivation-add.cc | 5 ++- src/nix/derivation-show.cc | 5 ++- src/nix/derivation.cc | 4 +- src/nix/develop.cc | 4 +- src/nix/diff-closures.cc | 6 +-- src/nix/dump-path.cc | 4 +- src/nix/edit.cc | 4 +- src/nix/env.cc | 4 +- src/nix/eval.cc | 4 +- src/nix/flake-command.hh | 4 +- src/nix/flake-prefetch-inputs.cc | 6 ++- src/nix/flake.cc | 26 +++++------ src/nix/formatter.cc | 4 +- src/nix/hash.cc | 4 +- src/nix/log.cc | 4 +- src/nix/ls.cc | 4 +- src/nix/main.cc | 8 ++-- src/nix/make-content-addressed.cc | 6 ++- src/nix/nar.cc | 4 +- src/nix/nix-build/nix-build.cc | 5 ++- src/nix/nix-channel/nix-channel.cc | 4 +- .../nix-collect-garbage.cc | 4 +- src/nix/nix-copy-closure/nix-copy-closure.cc | 4 +- src/nix/nix-env/nix-env.cc | 11 ++--- src/nix/nix-instantiate/nix-instantiate.cc | 4 +- src/nix/nix-store/nix-store.cc | 44 +++++++++---------- src/nix/optimise-store.cc | 4 +- src/nix/path-from-hash-part.cc | 4 +- src/nix/path-info.cc | 5 ++- src/nix/prefetch.cc | 4 +- src/nix/profile.cc | 4 +- src/nix/realisation.cc | 4 +- src/nix/registry.cc | 5 ++- src/nix/run.cc | 8 ++-- src/nix/search.cc | 5 ++- src/nix/sigs.cc | 4 +- src/nix/store-copy-log.cc | 4 +- src/nix/store-delete.cc | 4 +- src/nix/store-gc.cc | 4 +- src/nix/store-info.cc | 4 +- src/nix/store-repair.cc | 4 +- src/nix/store.cc | 4 +- src/nix/unix/daemon.cc | 13 +++--- src/nix/unix/store-roots-daemon.cc | 4 +- src/nix/upgrade-nix.cc | 4 +- src/nix/verify.cc | 4 +- src/nix/why-depends.cc | 4 +- tests/functional/plugins/plugintest.cc | 4 +- .../functional/test-libstoreconsumer/main.cc | 4 +- 78 files changed, 293 insertions(+), 196 deletions(-) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 5bdb5335b841..477eee1df9e6 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -33,6 +33,8 @@ namespace nix { struct LexerState; } +// OK because we keep this in a separate compilation unit even for unity +// builds. using namespace nix; using namespace nix::lexer::internal; diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 277adc092f2a..3d2351be08e9 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -63,8 +63,6 @@ namespace nix { struct SourceAccessor; -using namespace flake; - namespace flake { static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index 2960a158e37b..01b5be1a82d2 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -270,7 +270,7 @@ std::pair parseFlakeRefWithFragment( bool isFlake, bool preserveRelativePaths) { - using namespace fetchers; + using namespace nix::fetchers; if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { return *res; diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index c71d23e15e02..0bc32a91e648 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -37,8 +37,6 @@ namespace nix { -using namespace nix::linux; - static void setupSeccomp(const LocalSettings & localSettings) { if (!localSettings.filterSyscalls) @@ -337,10 +335,10 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu /* If we're running from the daemon, then this will return the root cgroup of the service. Otherwise, it will return the current cgroup. */ - auto cgroupFS = getCgroupFS(); + auto cgroupFS = linux::getCgroupFS(); if (!cgroupFS) throw Error("cannot determine the cgroups file system"); - auto rootCgroupPath = *cgroupFS / getRootCgroup().rel(); + auto rootCgroupPath = *cgroupFS / linux::getRootCgroup().rel(); if (!pathExists(rootCgroupPath)) throw Error("expected cgroup directory %s", PathFmt(rootCgroupPath)); @@ -363,7 +361,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu if (pathExists(cgroupFile)) { auto prevCgroup = readFile(cgroupFile); - destroyCgroup(prevCgroup); + linux::destroyCgroup(prevCgroup); } writeFile(cgroupFile, cgroup->native()); @@ -825,7 +823,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu void killSandbox(bool getStats) override { if (cgroup) { - auto stats = destroyCgroup(*cgroup); + auto stats = linux::destroyCgroup(*cgroup); if (getStats) { buildResult.cpuUser = stats.cpuUser; buildResult.cpuSystem = stats.cpuSystem; diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index f048986c75be..fe0e056d7c73 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -11,8 +11,6 @@ namespace nix { -using namespace nix::windows; - void deleteLockFile(const std::filesystem::path & path, Descriptor desc) { @@ -59,6 +57,7 @@ AutoCloseFD openLockFile(const std::filesystem::path & path, bool create) template static bool warnOrThrowWine(DWORD lastError, const std::string & fs, const Args &... args) { + using namespace nix::windows; if (isWine()) { warn(fs + ": %s (ignored under Wine)", args..., lastError); return true; diff --git a/src/libutil-test-support/hash.cc b/src/libutil-test-support/hash.cc index fdc95a6f8123..d9c7a0f74798 100644 --- a/src/libutil-test-support/hash.cc +++ b/src/libutil-test-support/hash.cc @@ -8,6 +8,7 @@ #include "nix/util/tests/hash.hh" namespace rc { + using namespace nix; Gen Arbitrary::arbitrary() diff --git a/src/libutil-test-support/include/nix/util/tests/hash.hh b/src/libutil-test-support/include/nix/util/tests/hash.hh index 633f7bbf76d7..ee676f3eb210 100644 --- a/src/libutil-test-support/include/nix/util/tests/hash.hh +++ b/src/libutil-test-support/include/nix/util/tests/hash.hh @@ -6,6 +6,7 @@ #include "nix/util/hash.hh" namespace rc { + using namespace nix; template<> diff --git a/src/libutil-tests/checked-arithmetic.cc b/src/libutil-tests/checked-arithmetic.cc index 2b5970fb64b8..cc88d353932c 100644 --- a/src/libutil-tests/checked-arithmetic.cc +++ b/src/libutil-tests/checked-arithmetic.cc @@ -10,6 +10,7 @@ #include "nix/util/tests/gtest-with-params.hh" namespace rc { + using namespace nix; template diff --git a/src/libutil-tests/git.cc b/src/libutil-tests/git.cc index dc7bd8fed4e3..81e34a065cac 100644 --- a/src/libutil-tests/git.cc +++ b/src/libutil-tests/git.cc @@ -7,8 +7,6 @@ namespace nix { -using namespace git; - class GitTest : public CharacterizationTest { std::filesystem::path unitTestData = getUnitTestData() / "git"; @@ -36,6 +34,7 @@ class GitTest : public CharacterizationTest TEST(GitMode, gitMode_directory) { + using namespace git; Mode m = Mode::Directory; RawMode r = 0040000; ASSERT_EQ(static_cast(m), r); @@ -44,6 +43,7 @@ TEST(GitMode, gitMode_directory) TEST(GitMode, gitMode_executable) { + using namespace git; Mode m = Mode::Executable; RawMode r = 0100755; ASSERT_EQ(static_cast(m), r); @@ -52,6 +52,7 @@ TEST(GitMode, gitMode_executable) TEST(GitMode, gitMode_regular) { + using namespace git; Mode m = Mode::Regular; RawMode r = 0100644; ASSERT_EQ(static_cast(m), r); @@ -60,6 +61,7 @@ TEST(GitMode, gitMode_regular) TEST(GitMode, gitMode_symlink) { + using namespace git; Mode m = Mode::Symlink; RawMode r = 0120000; ASSERT_EQ(static_cast(m), r); @@ -68,6 +70,7 @@ TEST(GitMode, gitMode_symlink) TEST_F(GitTest, blob_read) { + using namespace git; readTest("hello-world-blob.bin", [&](const auto & encoded) { StringSource in{encoded}; StringSink out; @@ -83,6 +86,7 @@ TEST_F(GitTest, blob_read) TEST_F(GitTest, blob_write) { + using namespace git; writeTest("hello-world-blob.bin", [&]() { auto decoded = readFile(goldenMaster("hello-world.bin")); StringSink s; @@ -97,11 +101,11 @@ TEST_F(GitTest, blob_write) * so that we can check our test data in a small shell script test test * (`src/libutil-tests/data/git/check-data.sh`). */ -const static Tree treeSha1 = { +const static git::Tree treeSha1 = { { "Foo", { - .mode = Mode::Regular, + .mode = git::Mode::Regular, // hello world with special chars from above .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), }, @@ -109,7 +113,7 @@ const static Tree treeSha1 = { { "bAr", { - .mode = Mode::Executable, + .mode = git::Mode::Executable, // ditto .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), }, @@ -117,7 +121,7 @@ const static Tree treeSha1 = { { "baZ/", { - .mode = Mode::Directory, + .mode = git::Mode::Directory, // Empty directory hash .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", HashAlgorithm::SHA1), }, @@ -125,7 +129,7 @@ const static Tree treeSha1 = { { "quuX", { - .mode = Mode::Symlink, + .mode = git::Mode::Symlink, // hello world with special chars from above (symlink target // can be anything) .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), @@ -137,11 +141,11 @@ const static Tree treeSha1 = { * Same conceptual object as `treeSha1`, just different hash algorithm. * See that one for details. */ -const static Tree treeSha256 = { +const static git::Tree treeSha256 = { { "Foo", { - .mode = Mode::Regular, + .mode = git::Mode::Regular, .hash = Hash::parseAny( "ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1", HashAlgorithm::SHA256), }, @@ -149,7 +153,7 @@ const static Tree treeSha256 = { { "bAr", { - .mode = Mode::Executable, + .mode = git::Mode::Executable, .hash = Hash::parseAny( "ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1", HashAlgorithm::SHA256), }, @@ -157,7 +161,7 @@ const static Tree treeSha256 = { { "baZ/", { - .mode = Mode::Directory, + .mode = git::Mode::Directory, .hash = Hash::parseAny( "6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321", HashAlgorithm::SHA256), }, @@ -165,15 +169,16 @@ const static Tree treeSha256 = { { "quuX", { - .mode = Mode::Symlink, + .mode = git::Mode::Symlink, .hash = Hash::parseAny( "ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1", HashAlgorithm::SHA256), }, }, }; -static auto mkTreeReadTest(HashAlgorithm hashAlgo, Tree tree, const ExperimentalFeatureSettings & mockXpSettings) +static auto mkTreeReadTest(HashAlgorithm hashAlgo, git::Tree tree, const ExperimentalFeatureSettings & mockXpSettings) { + using namespace git; return [hashAlgo, tree, mockXpSettings](const auto & encoded) { StringSource in{encoded}; NullFileSystemObjectSink out; @@ -208,6 +213,7 @@ TEST_F(GitTest, tree_sha256_read) TEST_F(GitTest, tree_sha1_write) { + using namespace git; writeTest("tree-sha1.bin", [&]() { StringSink s; dumpTree(treeSha1, s, mockXpSettings); @@ -217,6 +223,7 @@ TEST_F(GitTest, tree_sha1_write) TEST_F(GitTest, tree_sha256_write) { + using namespace git; writeTest("tree-sha256.bin", [&]() { StringSink s; dumpTree(treeSha256, s, mockXpSettings); @@ -232,6 +239,7 @@ extern ref exampleComplex(); TEST_F(GitTest, both_roundrip) { + using namespace git; auto files = memory_source_accessor::exampleComplex(); for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) { @@ -285,6 +293,7 @@ TEST_F(GitTest, both_roundrip) TEST(GitLsRemote, parseSymrefLineWithReference) { + using namespace git; auto line = "ref: refs/head/main HEAD"; auto res = parseLsRemoteLine(line); ASSERT_TRUE(res.has_value()); @@ -295,6 +304,7 @@ TEST(GitLsRemote, parseSymrefLineWithReference) TEST(GitLsRemote, parseSymrefLineWithNoReference) { + using namespace git; auto line = "ref: refs/head/main"; auto res = parseLsRemoteLine(line); ASSERT_TRUE(res.has_value()); @@ -305,6 +315,7 @@ TEST(GitLsRemote, parseSymrefLineWithNoReference) TEST(GitLsRemote, parseObjectRefLine) { + using namespace git; auto line = "abc123 refs/head/main"; auto res = parseLsRemoteLine(line); ASSERT_TRUE(res.has_value()); diff --git a/src/libutil-tests/unix/file-system-at.cc b/src/libutil-tests/unix/file-system-at.cc index 5fb0d26b9ced..09e427a26faf 100644 --- a/src/libutil-tests/unix/file-system-at.cc +++ b/src/libutil-tests/unix/file-system-at.cc @@ -15,14 +15,14 @@ namespace nix { -using namespace nix::unix; - /* ---------------------------------------------------------------------------- * fchmodatTryNoFollow * --------------------------------------------------------------------------*/ TEST(fchmodatTryNoFollow, works) { + using namespace nix::unix; + std::filesystem::path tmpDir = nix::createTempDir(); nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true); @@ -106,6 +106,8 @@ TEST(fchmodatTryNoFollow, works) TEST(fchmodatTryNoFollow, fallbackWithoutProc) { + using namespace nix::unix; + if (!userNamespacesSupported()) GTEST_SKIP() << "User namespaces not supported"; diff --git a/src/libutil-tests/unix/unix-domain-socket.cc b/src/libutil-tests/unix/unix-domain-socket.cc index 6c50eab9b2da..439d3188ac2d 100644 --- a/src/libutil-tests/unix/unix-domain-socket.cc +++ b/src/libutil-tests/unix/unix-domain-socket.cc @@ -9,14 +9,14 @@ namespace nix { -using namespace nix::unix; - /* ---------------------------------------------------------------------------- * sendMessageWithFds / receiveMessageWithFds * --------------------------------------------------------------------------*/ TEST(MessageWithFds, streamWithData) { + using namespace nix::unix; + int sockets[2]; ASSERT_EQ(socketpair(AF_UNIX, SOCK_STREAM, 0, sockets), 0); AutoCloseFD sender(sockets[0]); @@ -77,6 +77,8 @@ TEST(MessageWithFds, streamWithData) TEST(MessageWithFds, datagramEmptyData) { + using namespace nix::unix; + int sockets[2]; ASSERT_EQ(socketpair(AF_UNIX, SOCK_DGRAM, 0, sockets), 0); AutoCloseFD sender(sockets[0]); diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index 317bc33fff74..545f4f2b7040 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -33,7 +33,7 @@ size_t readOffset(Descriptor fd, off_t offset, std::span buffer) ssize_t n; do { checkInterrupt(); - n = pread(fd, buffer.data(), buffer.size(), offset); + n = ::pread(fd, buffer.data(), buffer.size(), offset); } while (n == -1 && errno == EINTR); if (n == -1) throw SysError("pread of %1% bytes at offset %2%", buffer.size(), offset); diff --git a/src/libutil/unix/include/nix/util/signals-impl.hh b/src/libutil/unix/include/nix/util/signals-impl.hh index dab0b09ed2dc..e22268214478 100644 --- a/src/libutil/unix/include/nix/util/signals-impl.hh +++ b/src/libutil/unix/include/nix/util/signals-impl.hh @@ -77,7 +77,7 @@ static inline bool getInterrupted() static inline bool isInterrupted() { - using namespace unix; + using namespace nix::unix; return _isInterrupted || (interruptCheck && interruptCheck()); } diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index e236a3cc2abf..7e425ab09018 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -9,8 +9,6 @@ namespace nix { -using namespace unix; - std::atomic unix::_isInterrupted = false; thread_local std::function unix::interruptCheck; @@ -56,6 +54,7 @@ static Sync & getInterruptCallbacks() static void signalHandlerThread(sigset_t set) { + using namespace nix::unix; while (true) { int signal = 0; sigwait(&set, &signal); diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 0020623e31a5..c571d0a3e9f7 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -14,13 +14,11 @@ namespace nix { -using namespace nix::windows; - std::make_unsigned_t getFileSize(Descriptor fd) { LARGE_INTEGER li; if (!GetFileSizeEx(fd, &li)) { - throw WinError([&] { return HintFmt("getting size of file %s", PathFmt(descriptorToPath(fd))); }); + throw windows::WinError([&] { return HintFmt("getting size of file %s", PathFmt(descriptorToPath(fd))); }); } return li.QuadPart; } @@ -34,7 +32,8 @@ size_t read(Descriptor fd, std::span buffer) if (lastError == ERROR_BROKEN_PIPE) n = 0; // Treat as EOF else - throw WinError(lastError, "reading %1% bytes from %2%", buffer.size(), PathFmt(descriptorToPath(fd))); + throw windows::WinError( + lastError, "reading %1% bytes from %2%", buffer.size(), PathFmt(descriptorToPath(fd))); } return static_cast(n); } @@ -48,7 +47,7 @@ size_t readOffset(Descriptor fd, off_t offset, std::span buffer) ov.OffsetHigh = static_cast(offset >> 32); DWORD n; if (!ReadFile(fd, buffer.data(), static_cast(buffer.size()), &n, &ov)) { - throw WinError([&] { + throw windows::WinError([&] { return HintFmt( "reading %1% bytes at offset %2% from %3%", buffer.size(), offset, PathFmt(descriptorToPath(fd))); }); @@ -62,7 +61,7 @@ size_t write(Descriptor fd, std::span buffer, bool allowInterru checkInterrupt(); // For consistency with unix DWORD n; if (!WriteFile(fd, buffer.data(), static_cast(buffer.size()), &n, NULL)) { - throw WinError( + throw windows::WinError( [&] { return HintFmt("writing %1% bytes to %2%", buffer.size(), PathFmt(descriptorToPath(fd))); }); } return static_cast(n); @@ -72,7 +71,7 @@ AutoCloseFD dupDescriptor(Descriptor fd) { HANDLE newHandle; if (!DuplicateHandle(GetCurrentProcess(), fd, GetCurrentProcess(), &newHandle, 0, FALSE, DUPLICATE_SAME_ACCESS)) { - throw WinError("duplicating handle"); + throw windows::WinError("duplicating handle"); } return AutoCloseFD{newHandle}; } @@ -88,7 +87,7 @@ void Pipe::create() HANDLE hReadPipe, hWritePipe; if (!CreatePipe(&hReadPipe, &hWritePipe, &saAttr, 0)) - throw WinError("CreatePipe"); + throw windows::WinError("CreatePipe"); readSide = hReadPipe; writeSide = hWritePipe; @@ -130,7 +129,7 @@ off_t lseek(HANDLE h, off_t offset, int whence) void syncDescriptor(Descriptor fd) { if (!::FlushFileBuffers(fd)) { - throw WinError([&] { return HintFmt("flushing file %s", PathFmt(descriptorToPath(fd))); }); + throw windows::WinError([&] { return HintFmt("flushing file %s", PathFmt(descriptorToPath(fd))); }); } } diff --git a/src/libutil/windows/file-system-at.cc b/src/libutil/windows/file-system-at.cc index a10dedbb886f..668bd7eb402f 100644 --- a/src/libutil/windows/file-system-at.cc +++ b/src/libutil/windows/file-system-at.cc @@ -13,8 +13,6 @@ namespace nix { -using namespace nix::windows; - namespace windows { namespace { @@ -214,7 +212,7 @@ PosixStat fstat(Descriptor fd) { BY_HANDLE_FILE_INFORMATION info; if (!GetFileInformationByHandle(fd, &info)) - throw WinError("getting file information for %s", PathFmt(descriptorToPath(fd))); + throw windows::WinError("getting file information for %s", PathFmt(descriptorToPath(fd))); PosixStat st; windows::statFromFileInfo( @@ -253,9 +251,9 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( /* Helper to check if a component is a symlink and throw SymlinkNotAllowed if so */ auto throwIfSymlink = [&](std::wstring_view component, const CanonPath & pathForError) { try { - auto testHandle = - ntOpenAt(getParentFd(), component, FILE_READ_ATTRIBUTES | SYNCHRONIZE, FILE_OPEN_REPARSE_POINT); - if (isReparsePoint(testHandle.get())) + auto testHandle = windows::ntOpenAt( + getParentFd(), component, FILE_READ_ATTRIBUTES | SYNCHRONIZE, FILE_OPEN_REPARSE_POINT); + if (windows::isReparsePoint(testHandle.get())) throw SymlinkNotAllowed(pathForError); } catch (SymlinkNotAllowed &) { throw; @@ -272,13 +270,13 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( /* Open directory without following symlinks */ AutoCloseFD parentFd2; try { - parentFd2 = ntOpenAt( + parentFd2 = windows::ntOpenAt( getParentFd(), wcomponent, FILE_TRAVERSE | SYNCHRONIZE, // Just need traversal rights FILE_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT // Open directory, don't follow symlinks ); - } catch (WinError & e) { + } catch (windows::WinError & e) { /* Check if this is because it's a symlink */ if (e.lastError == ERROR_CANT_ACCESS_FILE || e.lastError == ERROR_ACCESS_DENIED) { throwIfSymlink(wcomponent, pathUpTo(std::next(it))); @@ -287,7 +285,7 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( } /* Check if what we opened is actually a symlink */ - if (isReparsePoint(parentFd2.get())) { + if (windows::isReparsePoint(parentFd2.get())) { throw SymlinkNotAllowed(pathUpTo(std::next(it))); } @@ -299,13 +297,13 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( AutoCloseFD finalHandle; try { - finalHandle = ntOpenAt( + finalHandle = windows::ntOpenAt( getParentFd(), finalComponent, desiredAccess, createOptions | FILE_OPEN_REPARSE_POINT, // Don't follow symlinks on final component either createDisposition); - } catch (WinError & e) { + } catch (windows::WinError & e) { /* Check if final component is a symlink when we requested to not follow it */ if (e.lastError == ERROR_CANT_ACCESS_FILE) { throwIfSymlink(finalComponent, path); @@ -314,7 +312,7 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( } /* Final check: did we accidentally open a symlink? */ - if (isReparsePoint(finalHandle.get())) + if (windows::isReparsePoint(finalHandle.get())) throw SymlinkNotAllowed(path); return finalHandle; diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 9dab6f2b7a04..e644f5258029 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -16,8 +16,6 @@ static_assert(S_IFLNK != S_IFCHR, "S_IFLNK must not equal S_IFCHR"); namespace nix { -using namespace nix::windows; - void setWriteTime( const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) { @@ -72,7 +70,7 @@ std::filesystem::path defaultTempDir() wchar_t buf[MAX_PATH + 1]; DWORD len = GetTempPathW(MAX_PATH + 1, buf); if (len == 0 || len > MAX_PATH) - throw WinError("getting default temporary directory"); + throw windows::WinError("getting default temporary directory"); return std::filesystem::path(buf); } @@ -106,7 +104,7 @@ std::filesystem::path descriptorToPath(Descriptor handle) if (dw > buf.size()) { buf.resize(dw); if (GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED) != dw - 1) - throw WinError("GetFinalPathNameByHandleW"); + throw windows::WinError("GetFinalPathNameByHandleW"); dw -= 1; } return std::filesystem::path{std::wstring{buf.data(), dw}}; @@ -179,7 +177,7 @@ PosixStat lstat(const std::filesystem::path & path) { WIN32_FILE_ATTRIBUTE_DATA attrData; if (!GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &attrData)) - throw WinError("getting status of %s", PathFmt(path)); + throw windows::WinError("getting status of %s", PathFmt(path)); return statFromFileInfo(attrData); } @@ -190,7 +188,7 @@ std::optional maybeLstat(const std::filesystem::path & path) auto lastError = GetLastError(); if (lastError == ERROR_FILE_NOT_FOUND || lastError == ERROR_PATH_NOT_FOUND) return std::nullopt; - throw WinError(lastError, "getting status of %s", PathFmt(path)); + throw windows::WinError(lastError, "getting status of %s", PathFmt(path)); } return statFromFileInfo(attrData); } diff --git a/src/libutil/windows/known-folders.cc b/src/libutil/windows/known-folders.cc index 1d2867428459..5e0966c8491f 100644 --- a/src/libutil/windows/known-folders.cc +++ b/src/libutil/windows/known-folders.cc @@ -8,8 +8,6 @@ namespace nix::windows::known_folders { -using namespace nix::windows; - static std::filesystem::path getKnownFolder(REFKNOWNFOLDERID rfid) { PWSTR str = nullptr; diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index 1e89ccc5e364..b1f98b63afcf 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -6,8 +6,6 @@ namespace nix { -using namespace nix::windows; - void MuxablePipePollState::poll(HANDLE ioport, std::optional timeout) { /* We are on at least Windows Vista / Server 2008 and can get many @@ -16,7 +14,7 @@ void MuxablePipePollState::poll(HANDLE ioport, std::optional timeo ioport, oentries, sizeof(oentries) / sizeof(*oentries), &removed, timeout ? *timeout : INFINITE, false)) { auto lastError = GetLastError(); if (lastError != WAIT_TIMEOUT) - throw WinError(lastError, "GetQueuedCompletionStatusEx"); + throw windows::WinError(lastError, "GetQueuedCompletionStatusEx"); assert(removed == 0); } else { assert(0 < removed && removed <= sizeof(oentries) / sizeof(*oentries)); @@ -58,7 +56,7 @@ void MuxablePipePollState::iterate( handleEOF((*p)->readSide.get()); nextp = channels.erase(p); // no need to maintain `channels` ? } else if (lastError != ERROR_IO_PENDING) - throw WinError(lastError, "ReadFile(%s, ..)", (*p)->readSide.get()); + throw windows::WinError(lastError, "ReadFile(%s, ..)", (*p)->readSide.get()); } } break; diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index c0b0fa6c10be..46a149603fa6 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -30,8 +30,6 @@ namespace nix { -using namespace nix::windows; - Pid::Pid() {} Pid::Pid(Pid && other) noexcept @@ -64,7 +62,7 @@ int Pid::kill(bool allowInterrupts) debug("killing process %1%", pid.get()); if (!TerminateProcess(pid.get(), 1)) - logError(WinError("terminating process %1%", pid.get()).info()); + logError(windows::WinError("terminating process %1%", pid.get()).info()); return wait(allowInterrupts); } @@ -76,11 +74,11 @@ int Pid::wait(bool allowInterrupts) assert(pid.get() != INVALID_DESCRIPTOR); DWORD status = WaitForSingleObject(pid.get(), INFINITE); if (status != WAIT_OBJECT_0) - throw WinError("waiting for process %1%", pid.get()); + throw windows::WinError("waiting for process %1%", pid.get()); DWORD exitCode = 0; if (GetExitCodeProcess(pid.get(), &exitCode) == FALSE) - throw WinError("getting exit code of process %1%", pid.get()); + throw windows::WinError("getting exit code of process %1%", pid.get()); pid.close(); return exitCode; @@ -126,7 +124,7 @@ void setFDInheritable(AutoCloseFD & fd, bool inherit) { if (fd.get() != INVALID_DESCRIPTOR) { if (!SetHandleInformation(fd.get(), HANDLE_FLAG_INHERIT, inherit ? HANDLE_FLAG_INHERIT : 0)) { - throw WinError("Couldn't disable inheriting of handle"); + throw windows::WinError("Couldn't disable inheriting of handle"); } } } @@ -146,7 +144,7 @@ AutoCloseFD nullFD() 0, NULL); if (!nul.get()) { - throw WinError("Couldn't open NUL device"); + throw windows::WinError("Couldn't open NUL device"); } // Let this handle be inheritable by child processes setFDInheritable(nul, true); @@ -258,7 +256,7 @@ Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & o &startInfo, &procInfo) == 0) { - throw WinError("CreateProcessW failed (%1%)", os_string_to_string(cmdline)); + throw windows::WinError("CreateProcessW failed (%1%)", os_string_to_string(cmdline)); } // Convert these to use RAII @@ -271,15 +269,15 @@ Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & o Descriptor job = CreateJobObjectW(NULL, NULL); if (job == NULL) { TerminateProcess(procInfo.hProcess, 0); - throw WinError("Couldn't create job object for child process"); + throw windows::WinError("Couldn't create job object for child process"); } if (AssignProcessToJobObject(job, procInfo.hProcess) == FALSE) { TerminateProcess(procInfo.hProcess, 0); - throw WinError("Couldn't assign child process to job object"); + throw windows::WinError("Couldn't assign child process to job object"); } if (ResumeThread(procInfo.hThread) == (DWORD) -1) { TerminateProcess(procInfo.hProcess, 0); - throw WinError("Couldn't resume child process thread"); + throw windows::WinError("Couldn't resume child process thread"); } return process; diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index e5acd8cbdcfd..eb82a827d6a5 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -8,8 +8,6 @@ namespace nix { -using namespace nix::windows; - std::string getUserName() { // Get the required buffer size @@ -17,7 +15,7 @@ std::string getUserName() if (!GetUserNameA(nullptr, &size)) { auto lastError = GetLastError(); if (lastError != ERROR_INSUFFICIENT_BUFFER) - throw WinError(lastError, "cannot figure out size of user name"); + throw windows::WinError(lastError, "cannot figure out size of user name"); } std::string name; @@ -28,7 +26,7 @@ std::string getUserName() // Retrieve the username if (!GetUserNameA(&name[0], &size)) - throw WinError("cannot figure out user name"); + throw windows::WinError("cannot figure out user name"); return name; } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 2a24c4199f43..4f0fe722f197 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -4,7 +4,7 @@ #include "nix/util/posix-source-accessor.hh" #include "nix/cmd/misc-store-flags.hh" -using namespace nix; +namespace nix { struct CmdAddToStore : MixDryRun, StoreCommand { @@ -84,3 +84,5 @@ struct CmdAddPath : CmdAddToStore static auto rCmdAddFile = registerCommand2({"store", "add-file"}); static auto rCmdAddPath = registerCommand2({"store", "add-path"}); static auto rCmdAdd = registerCommand2({"store", "add"}); + +} // namespace nix diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index 6436466374bd..e5ebf948d7e2 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -5,6 +5,7 @@ #include #include #include + #ifdef __APPLE__ # include #endif @@ -24,8 +25,7 @@ #include "nix/util/experimental-features.hh" #include "nix/store/globals.hh" -using namespace nix; -using std::cin; +namespace nix { static void handleAlarm(int sig) {} @@ -410,3 +410,5 @@ static int main_build_remote(int argc, char ** argv) } static RegisterLegacyCommand r_build_remote("build-remote", main_build_remote); + +} // namespace nix diff --git a/src/nix/build.cc b/src/nix/build.cc index 2de7fdf42362..4677f325fbf9 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -5,7 +5,7 @@ #include -using namespace nix; +namespace nix { /* This serialization code is diferent from the canonical (single) derived path serialization because: @@ -186,3 +186,5 @@ struct CmdBuild : InstallablesCommand, MixOutLinkByDefault, MixDryRun, MixJSON, }; static auto rCmdBuild = registerCommand("build"); + +} // namespace nix diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 85c8cdb58759..4808a2fc716f 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -6,7 +6,7 @@ #include "nix/expr/eval-inline.hh" #include "nix/store/globals.hh" -using namespace nix; +namespace nix { struct CmdBundle : InstallableValueCommand { @@ -131,3 +131,5 @@ struct CmdBundle : InstallableValueCommand }; static auto r2 = registerCommand("bundle"); + +} // namespace nix diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 8a816f528815..a68285efba4b 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -6,7 +6,7 @@ #include -using namespace nix; +namespace nix { struct MixCat : virtual Args { @@ -120,3 +120,5 @@ struct CmdCatNar : StoreCommand, MixCat static auto rCmdCatStore = registerCommand2({"store", "cat"}); static auto rCmdCatNar = registerCommand2({"nar", "cat"}); + +} // namespace nix diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 76c2eaa74ec5..8bd923aabf0e 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -11,7 +11,7 @@ #include "nix/util/executable-path.hh" #include "nix/store/globals.hh" -using namespace nix; +namespace nix { namespace { @@ -178,3 +178,5 @@ struct CmdConfigCheck : StoreCommand }; static auto rCmdConfigCheck = registerCommand2({"config", "check"}); + +} // namespace nix diff --git a/src/nix/config.cc b/src/nix/config.cc index 3fd0cde808f7..6230fd103650 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -4,7 +4,7 @@ #include -using namespace nix; +namespace nix { struct CmdConfig : NixMultiCommand { @@ -79,3 +79,5 @@ struct CmdConfigShow : Command, MixJSON static auto rCmdConfig = registerCommand("config"); static auto rShowConfig = registerCommand2({"config", "show"}); + +} // namespace nix diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 86306e7fdb8e..11a0cbf6236a 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -3,7 +3,7 @@ #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" -using namespace nix; +namespace nix { struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile, MixNoCheckSigs { @@ -75,3 +75,5 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile, Mix }; static auto rCmdCopy = registerCommand("copy"); + +} // namespace nix diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc index 2046ecca3892..def1d7158c92 100644 --- a/src/nix/derivation-add.cc +++ b/src/nix/derivation-add.cc @@ -7,9 +7,10 @@ #include "nix/store/globals.hh" #include -using namespace nix; using json = nlohmann::json; +namespace nix { + struct CmdAddDerivation : MixDryRun, StoreCommand { std::string description() override @@ -43,3 +44,5 @@ struct CmdAddDerivation : MixDryRun, StoreCommand }; static auto rCmdAddDerivation = registerCommand2({"derivation", "add"}); + +} // namespace nix diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index 1a6f00c8cdee..5880c6c8f13b 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -7,9 +7,10 @@ #include "nix/store/derivations.hh" #include -using namespace nix; using json = nlohmann::json; +namespace nix { + struct CmdShowDerivation : InstallablesCommand, MixPrintJSON { bool recursive = false; @@ -68,3 +69,5 @@ struct CmdShowDerivation : InstallablesCommand, MixPrintJSON }; static auto rCmdShowDerivation = registerCommand2({"derivation", "show"}); + +} // namespace nix diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc index 2634048ac246..98f41d77361f 100644 --- a/src/nix/derivation.cc +++ b/src/nix/derivation.cc @@ -1,6 +1,6 @@ #include "nix/cmd/command.hh" -using namespace nix; +namespace nix { struct CmdDerivation : NixMultiCommand { @@ -21,3 +21,5 @@ struct CmdDerivation : NixMultiCommand }; static auto rCmdDerivation = registerCommand("derivation"); + +} // namespace nix diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 301e35b55ffb..50b248bcb884 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -21,7 +21,7 @@ #include "nix/util/strings.hh" -using namespace nix; +namespace nix { struct DevelopSettings : Config { @@ -752,3 +752,5 @@ struct CmdPrintDevEnv : Common, MixJSON static auto rCmdPrintDevEnv = registerCommand("print-dev-env"); static auto rCmdDevelop = registerCommand("develop"); + +} // namespace nix diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index ff37100651cf..72108399d12f 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -112,10 +112,6 @@ void printClosureDiff( } } -} // namespace nix - -using namespace nix; - struct CmdDiffClosures : SourceExprCommand, MixOperateOnOptions { std::string _before, _after; @@ -149,3 +145,5 @@ struct CmdDiffClosures : SourceExprCommand, MixOperateOnOptions }; static auto rCmdDiffClosures = registerCommand2({"store", "diff-closures"}); + +} // namespace nix diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 2a6253b026fb..25129487786b 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -3,7 +3,7 @@ #include "nix/util/archive.hh" #include "nix/util/terminal.hh" -using namespace nix; +namespace nix { static FdSink getNarSink() { @@ -77,3 +77,5 @@ struct CmdNarDumpPath : CmdDumpPath2 static auto rCmdNarPack = registerCommand2({"nar", "pack"}); static auto rCmdNarDumpPath = registerCommand2({"nar", "dump-path"}); + +} // namespace nix diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 0657301f36be..aeb26915ba0e 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -7,7 +7,7 @@ #include -using namespace nix; +namespace nix { struct CmdEdit : InstallableValueCommand { @@ -58,3 +58,5 @@ struct CmdEdit : InstallableValueCommand }; static auto rCmdEdit = registerCommand("edit"); + +} // namespace nix diff --git a/src/nix/env.cc b/src/nix/env.cc index 79d187d85e59..4022c3effa81 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -10,7 +10,7 @@ #include "nix/util/environment-variables.hh" #include "nix/util/mounted-source-accessor.hh" -using namespace nix; +namespace nix { struct CmdEnv : NixMultiCommand { @@ -121,3 +121,5 @@ struct CmdShell : InstallablesCommand, MixEnvironment }; static auto rCmdShell = registerCommand2({"env", "shell"}); + +} // namespace nix diff --git a/src/nix/eval.cc b/src/nix/eval.cc index f41d98f1a8d0..ed6fe41be655 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -8,7 +8,7 @@ #include -using namespace nix; +namespace nix { struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption { @@ -126,3 +126,5 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption }; static auto rCmdEval = registerCommand("eval"); + +} // namespace nix diff --git a/src/nix/flake-command.hh b/src/nix/flake-command.hh index 3636bd525109..ae52bcabb912 100644 --- a/src/nix/flake-command.hh +++ b/src/nix/flake-command.hh @@ -6,8 +6,6 @@ namespace nix { -using namespace nix::flake; - class FlakeCommand : virtual Args, public MixFlakeOptions { protected: @@ -19,7 +17,7 @@ public: FlakeRef getFlakeRef(); - LockedFlake lockFlake(); + flake::LockedFlake lockFlake(); std::vector getFlakeRefsForCompletion() override; }; diff --git a/src/nix/flake-prefetch-inputs.cc b/src/nix/flake-prefetch-inputs.cc index 4ea6342c3695..eb73c6c915a4 100644 --- a/src/nix/flake-prefetch-inputs.cc +++ b/src/nix/flake-prefetch-inputs.cc @@ -6,8 +6,7 @@ #include -using namespace nix; -using namespace nix::flake; +namespace nix { struct CmdFlakePrefetchInputs : FlakeCommand { @@ -25,6 +24,7 @@ struct CmdFlakePrefetchInputs : FlakeCommand void run(nix::ref store) override { + using namespace nix::flake; auto flake = lockFlake(); ThreadPool pool{fileTransferSettings.httpConnections}; @@ -69,3 +69,5 @@ struct CmdFlakePrefetchInputs : FlakeCommand }; static auto rCmdFlakePrefetchInputs = registerCommand2({"flake", "prefetch-inputs"}); + +} // namespace nix diff --git a/src/nix/flake.cc b/src/nix/flake.cc index df2a3a9114a0..42de78453451 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -30,9 +30,7 @@ // FIXME is this supposed to be private or not? #include "flake-command.hh" -using namespace nix; -using namespace nix::flake; -using json = nlohmann::json; +namespace nix { struct CmdFlakeUpdate; @@ -52,7 +50,7 @@ FlakeRef FlakeCommand::getFlakeRef() return parseFlakeRef(fetchSettings, flakeUrl, std::filesystem::current_path().string()); // FIXME } -LockedFlake FlakeCommand::lockFlake() +flake::LockedFlake FlakeCommand::lockFlake() { return flake::lockFlake(flakeSettings, *getEvalState(), getFlakeRef(), lockFlags); } @@ -89,7 +87,7 @@ struct CmdFlakeUpdate : FlakeCommand .optional = true, .handler = {[&](std::vector inputsToUpdate) { for (const auto & inputToUpdate : inputsToUpdate) { - std::optional inputAttrPath; + std::optional inputAttrPath; try { inputAttrPath = flake::NonEmptyInputAttrPath::parse(inputToUpdate); if (!inputAttrPath) @@ -269,9 +267,9 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON if (!lockedFlake.lockFile.root->inputs.empty()) logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); - std::set> visited{lockedFlake.lockFile.root}; + std::set> visited{lockedFlake.lockFile.root}; - [&](this const auto & recurse, const Node & node, const std::string & prefix) -> void { + [&](this const auto & recurse, const flake::Node & node, const std::string & prefix) -> void { for (const auto & [i, input] : enumerate(node.inputs)) { bool last = i + 1 == node.inputs.size(); @@ -295,7 +293,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON "%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'", prefix + (last ? treeLast : treeConn), input.first, - printInputAttrPath(*follows)); + flake::printInputAttrPath(*follows)); } } }(*lockedFlake.lockFile.root, ""); @@ -856,7 +854,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand std::string templateUrl = "templates"; std::filesystem::path destDir; - const LockFlags lockFlags{.writeLockFile = false}; + const flake::LockFlags lockFlags{.writeLockFile = false}; CmdFlakeInitCommon() { @@ -1097,9 +1095,9 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs sources.insert(storePath); // FIXME: use graph output, handle cycles. - std::function traverse; - traverse = [&](const Node & node) { - nlohmann::json jsonObj2 = json ? json::object() : nlohmann::json(nullptr); + std::function traverse; + traverse = [&](const flake::Node & node) { + nlohmann::json jsonObj2 = json ? nlohmann::json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { std::optional storePath; @@ -1174,7 +1172,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON evalSettings.enableImportFromDerivation.setDefault(false); auto state = getEvalState(); - auto flake = make_ref(lockFlake()); + auto flake = make_ref(lockFlake()); auto localSystem = std::string(settings.thisSystem.get()); std::function @@ -1567,3 +1565,5 @@ static auto rCmdFlakeNew = registerCommand2({"flake", "new"}); static auto rCmdFlakePrefetch = registerCommand2({"flake", "prefetch"}); static auto rCmdFlakeShow = registerCommand2({"flake", "show"}); static auto rCmdFlakeUpdate = registerCommand2({"flake", "update"}); + +} // namespace nix diff --git a/src/nix/formatter.cc b/src/nix/formatter.cc index 619dcf4fad0b..19d473863570 100644 --- a/src/nix/formatter.cc +++ b/src/nix/formatter.cc @@ -7,7 +7,7 @@ #include "run.hh" -using namespace nix; +namespace nix { struct CmdFormatter : NixMultiCommand { @@ -158,3 +158,5 @@ struct CmdFmt : CmdFormatterRun }; static auto rFmt = registerCommand("fmt"); + +} // namespace nix diff --git a/src/nix/hash.cc b/src/nix/hash.cc index bc1ce313b08d..90f44fdecd26 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -9,7 +9,7 @@ #include "man-pages.hh" #include "nix/util/fun.hh" -using namespace nix; +namespace nix { /** * Base for `nix hash path`, `nix hash file` (deprecated), and `nix-hash` (legacy). @@ -366,3 +366,5 @@ static int compatNixHash(int argc, char ** argv) } static RegisterLegacyCommand r_nix_hash("nix-hash", compatNixHash); + +} // namespace nix diff --git a/src/nix/log.cc b/src/nix/log.cc index acc03cbb6783..1ee3af669906 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -3,7 +3,7 @@ #include "nix/main/shared.hh" #include "nix/store/globals.hh" -using namespace nix; +namespace nix { struct CmdLog : InstallableCommand { @@ -47,3 +47,5 @@ struct CmdLog : InstallableCommand }; static auto rCmdLog = registerCommand("log"); + +} // namespace nix diff --git a/src/nix/ls.cc b/src/nix/ls.cc index a9d082fb3782..410f4f420e05 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -4,7 +4,7 @@ #include "nix/main/common-args.hh" #include -using namespace nix; +namespace nix { struct MixLs : virtual Args, MixJSON { @@ -160,3 +160,5 @@ struct CmdLsNar : Command, MixLs static auto rCmdLsStore = registerCommand2({"store", "ls"}); static auto rCmdLsNar = registerCommand2({"nar", "ls"}); + +} // namespace nix diff --git a/src/nix/main.cc b/src/nix/main.cc index c19aa26925d2..98423b3b1014 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -41,16 +41,16 @@ # include "nix/util/linux-namespaces.hh" #endif +#include "nix/util/strings.hh" + +namespace nix { + #ifndef _WIN32 extern std::string chrootHelperName; void chrootHelper(int argc, char ** argv); #endif -#include "nix/util/strings.hh" - -namespace nix { - /* Check if we have a non-loopback/link-local network interface. */ static bool haveInternet() { diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index cd0f3f59f610..1268163967a9 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -5,10 +5,10 @@ #include -using namespace nix; - using nlohmann::json; +namespace nix { + struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON { CmdMakeContentAddressed() @@ -56,3 +56,5 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, }; static auto rCmdMakeContentAddressed = registerCommand2({"store", "make-content-addressed"}); + +} // namespace nix diff --git a/src/nix/nar.cc b/src/nix/nar.cc index bae77b6cc10f..fe6c89ec12d3 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -1,6 +1,6 @@ #include "nix/cmd/command.hh" -using namespace nix; +namespace nix { struct CmdNar : NixMultiCommand { @@ -28,3 +28,5 @@ struct CmdNar : NixMultiCommand }; static auto rCmdNar = registerCommand("nar"); + +} // namespace nix diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index d9e2f2dc8d41..e68e775e024f 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -31,11 +31,12 @@ #include "nix/util/fun.hh" #include "man-pages.hh" -using namespace nix; using namespace std::string_literals; extern char ** environ __attribute__((weak)); +namespace nix { + /* Recreate the effect of the perl shellwords function, breaking up a * string into arguments like a shell word, including escapes */ @@ -747,3 +748,5 @@ static void main_nix_build(int argc, char ** argv) static RegisterLegacyCommand r_nix_build("nix-build", main_nix_build); static RegisterLegacyCommand r_nix_shell("nix-shell", main_nix_build); + +} // namespace nix diff --git a/src/nix/nix-channel/nix-channel.cc b/src/nix/nix-channel/nix-channel.cc index 911aabd43833..aa58beb2436f 100644 --- a/src/nix/nix-channel/nix-channel.cc +++ b/src/nix/nix-channel/nix-channel.cc @@ -16,7 +16,7 @@ #include #include -using namespace nix; +namespace nix { typedef StringMap Channels; @@ -306,3 +306,5 @@ static int main_nix_channel(int argc, char ** argv) } static RegisterLegacyCommand r_nix_channel("nix-channel", main_nix_channel); + +} // namespace nix diff --git a/src/nix/nix-collect-garbage/nix-collect-garbage.cc b/src/nix/nix-collect-garbage/nix-collect-garbage.cc index a9f8b7fb6ba1..c267e95c9ff5 100644 --- a/src/nix/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix/nix-collect-garbage/nix-collect-garbage.cc @@ -12,7 +12,7 @@ #include -using namespace nix; +namespace nix { std::string deleteOlderThan; bool dryRun = false; @@ -109,3 +109,5 @@ static int main_nix_collect_garbage(int argc, char ** argv) } static RegisterLegacyCommand r_nix_collect_garbage("nix-collect-garbage", main_nix_collect_garbage); + +} // namespace nix diff --git a/src/nix/nix-copy-closure/nix-copy-closure.cc b/src/nix/nix-copy-closure/nix-copy-closure.cc index 33aec6c3e659..0c939df659ae 100644 --- a/src/nix/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix/nix-copy-closure/nix-copy-closure.cc @@ -5,7 +5,7 @@ #include "nix/cmd/legacy.hh" #include "man-pages.hh" -using namespace nix; +namespace nix { static int main_nix_copy_closure(int argc, char ** argv) { @@ -69,3 +69,5 @@ static int main_nix_copy_closure(int argc, char ** argv) } static RegisterLegacyCommand r_nix_copy_closure("nix-copy-closure", main_nix_copy_closure); + +} // namespace nix diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index b93a51ac533b..abe04d96341c 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -33,8 +33,7 @@ #include #include -using namespace nix; -using std::cout; +namespace nix { /** * Settings related to Nix user environments. @@ -1073,7 +1072,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* Print the desired columns, or XML output. */ if (jsonOutput) { queryJSON(globals, elems, printOutPath, printDrvPath, printMeta); - cout << '\n'; + std::cout << '\n'; return; } @@ -1082,7 +1081,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) Table table; std::ostringstream dummy; - XMLWriter xml(true, *(xmlOutput ? &cout : &dummy)); + XMLWriter xml(true, *(xmlOutput ? &std::cout : &dummy)); XMLOpenElement xmlRoot(xml, "items"); for (auto & i : elems) { @@ -1274,7 +1273,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } else table.push_back(columns); - cout.flush(); + std::cout.flush(); } catch (AssertionError & e) { printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName()); @@ -1536,3 +1535,5 @@ static int main_nix_env(int argc, char ** argv) } static RegisterLegacyCommand r_nix_env("nix-env", main_nix_env); + +} // namespace nix diff --git a/src/nix/nix-instantiate/nix-instantiate.cc b/src/nix/nix-instantiate/nix-instantiate.cc index b368346eea93..27a11767b004 100644 --- a/src/nix/nix-instantiate/nix-instantiate.cc +++ b/src/nix/nix-instantiate/nix-instantiate.cc @@ -15,7 +15,7 @@ #include -using namespace nix; +namespace nix { std::filesystem::path gcRoot; static int rootNr = 0; @@ -207,3 +207,5 @@ static int main_nix_instantiate(int argc, char ** argv) } static RegisterLegacyCommand r_nix_instantiate("nix-instantiate", main_nix_instantiate); + +} // namespace nix diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index a534e19afb9f..2c344d4cd9df 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -41,8 +41,6 @@ namespace nix_store { using namespace nix; -using std::cin; -using std::cout; typedef void (*Operation)(Strings opFlags, Strings opArgs); @@ -182,7 +180,7 @@ static void opRealise(Strings opFlags, Strings opArgs) auto paths2 = realisePath(i, false); if (!noOutput) for (auto & j : paths2) - cout << fmt("%s\n", j.string()); + std::cout << fmt("%s\n", j.string()); } } @@ -194,7 +192,7 @@ static void opAdd(Strings opFlags, Strings opArgs) for (auto & i : opArgs) { auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), sourcePath))); + std::cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), sourcePath))); } } @@ -242,7 +240,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string hash = *i++; std::string name = *i++; - cout << fmt( + std::cout << fmt( "%s\n", store->printStorePath(store->makeFixedOutputPath( name, @@ -280,11 +278,11 @@ static void printTree(const StorePath & path, const std::string & firstPad, const std::string & tailPad, StorePathSet & done) { if (!done.insert(path).second) { - cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path)); + std::cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path)); return; } - cout << fmt("%s%s\n", firstPad, store->printStorePath(path)); + std::cout << fmt("%s%s\n", firstPad, store->printStorePath(path)); auto info = store->queryPathInfo(path); @@ -387,7 +385,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & i : opArgs) { auto outputs = maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise); for (auto & outputPath : outputs) - cout << fmt("%1%\n", store->printStorePath(outputPath)); + std::cout << fmt("%1%\n", store->printStorePath(outputPath)); } break; } @@ -416,14 +414,14 @@ static void opQuery(Strings opFlags, Strings opArgs) } auto sorted = store->topoSortPaths(paths); for (StorePaths::reverse_iterator i = sorted.rbegin(); i != sorted.rend(); ++i) - cout << fmt("%s\n", store->printStorePath(*i)); + std::cout << fmt("%s\n", store->printStorePath(*i)); break; } case qDeriver: for (auto & i : opArgs) { auto info = store->queryPathInfo(store->followLinksToStorePath(i)); - cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver"); + std::cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver"); } break; @@ -437,7 +435,7 @@ static void opQuery(Strings opFlags, Strings opArgs) } auto sorted = store->topoSortPaths(result); for (StorePaths::reverse_iterator i = sorted.rbegin(); i != sorted.rend(); ++i) - cout << fmt("%s\n", store->printStorePath(*i)); + std::cout << fmt("%s\n", store->printStorePath(*i)); break; } @@ -449,7 +447,7 @@ static void opQuery(Strings opFlags, Strings opArgs) if (j == drv.env.end()) throw Error( "derivation '%s' has no environment binding named '%s'", store->printStorePath(path), bindingName); - cout << fmt("%s\n", j->second); + std::cout << fmt("%s\n", j->second); } break; @@ -460,9 +458,9 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.algo == HashAlgorithm::SHA256); - cout << fmt("%s\n", info->narHash.to_string(HashFormat::Nix32, true)); + std::cout << fmt("%s\n", info->narHash.to_string(HashFormat::Nix32, true)); } else if (query == qSize) - cout << fmt("%d\n", info->narSize); + std::cout << fmt("%d\n", info->narSize); } } break; @@ -494,7 +492,7 @@ static void opQuery(Strings opFlags, Strings opArgs) case qResolve: { for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->followLinksToStorePath(i))); + std::cout << fmt("%s\n", store->printStorePath(store->followLinksToStorePath(i))); break; } @@ -513,7 +511,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & [target, links] : roots) if (referrers.find(target) != referrers.end()) for (auto & link : links) - cout << fmt("%1% -> %2%\n", link, gcStore.printStorePath(target)); + std::cout << fmt("%1% -> %2%\n", link, gcStore.printStorePath(target)); break; } @@ -539,7 +537,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) /* Also output the arguments. */ std::string argsStr = concatStringsSep(" ", drv.args); - cout << "export _args; _args=" << escapeShellArgAlways(argsStr) << "\n"; + std::cout << "export _args; _args=" << escapeShellArgAlways(argsStr) << "\n"; } static void opReadLog(Strings opFlags, Strings opArgs) @@ -566,10 +564,10 @@ static void opDumpDB(Strings opFlags, Strings opArgs) throw UsageError("unknown flag"); if (!opArgs.empty()) { for (auto & i : opArgs) - cout << store->makeValidityRegistration({store->followLinksToStorePath(i)}, true, true); + std::cout << store->makeValidityRegistration({store->followLinksToStorePath(i)}, true, true); } else { for (auto & i : store->queryAllValidPaths()) - cout << store->makeValidityRegistration({i}, true, true); + std::cout << store->makeValidityRegistration({i}, true, true); } } @@ -585,7 +583,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) std::numeric_limits::max(), }} : std::nullopt; - auto info = decodeValidPathInfo(*store, cin, hashResultOpt); + auto info = decodeValidPathInfo(*store, std::cin, hashResultOpt); if (!info) break; if (!store->isValidPath(info->path) || reregister) { @@ -651,7 +649,7 @@ static void opCheckValidity(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); if (!store->isValidPath(path)) { if (printInvalid) - cout << fmt("%s\n", store->printStorePath(path)); + std::cout << fmt("%s\n", store->printStorePath(path)); else throw Error("path '%s' is not valid", store->printStorePath(path)); } @@ -703,7 +701,7 @@ static void opGC(Strings opFlags, Strings opArgs) Finally printer([&] { if (options.action != GCOptions::gcDeleteDead) for (auto & i : results.paths) - cout << i << std::endl; + std::cout << i << std::endl; else printFreed(false, results); }); @@ -788,7 +786,7 @@ static void opImport(Strings opFlags, Strings opArgs) auto paths = importPaths(*store, source, NoCheckSigs); for (auto & i : paths) - cout << fmt("%s\n", store->printStorePath(i)) << std::flush; + std::cout << fmt("%s\n", store->printStorePath(i)) << std::flush; } /* Initialise the Nix databases. */ diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index 5edbabc3bd10..2b2f3be48fc4 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -2,7 +2,7 @@ #include "nix/main/shared.hh" #include "nix/store/store-api.hh" -using namespace nix; +namespace nix { struct CmdOptimiseStore : StoreCommand { @@ -25,3 +25,5 @@ struct CmdOptimiseStore : StoreCommand }; static auto rCmdOptimiseStore = registerCommand2({"store", "optimise"}); + +} // namespace nix diff --git a/src/nix/path-from-hash-part.cc b/src/nix/path-from-hash-part.cc index 7e6c6ec280b1..a2f16393be12 100644 --- a/src/nix/path-from-hash-part.cc +++ b/src/nix/path-from-hash-part.cc @@ -1,7 +1,7 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" -using namespace nix; +namespace nix { struct CmdPathFromHashPart : StoreCommand { @@ -37,3 +37,5 @@ struct CmdPathFromHashPart : StoreCommand }; static auto rCmdPathFromHashPart = registerCommand2({"store", "path-from-hash-part"}); + +} // namespace nix diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 9abe38bfdde6..5be20f66da7f 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -10,7 +10,8 @@ #include "nix/util/strings.hh" -using namespace nix; +namespace nix { + using nlohmann::json; /** @@ -237,3 +238,5 @@ struct CmdPathInfo : StorePathsCommand, MixJSON }; static auto rCmdPathInfo = registerCommand("path-info"); + +} // namespace nix diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 1ace94a8b782..7457931d6fc9 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -18,7 +18,7 @@ #include -using namespace nix; +namespace nix { /* If ‘url’ starts with ‘mirror://’, then resolve it using the list of mirrors defined in Nixpkgs. */ @@ -349,3 +349,5 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON }; static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); + +} // namespace nix diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 9819004c1447..c85c406aa518 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -20,7 +20,7 @@ #include "nix/util/strings.hh" -using namespace nix; +namespace nix { struct ProfileElementSource { @@ -1029,3 +1029,5 @@ struct CmdProfile : NixMultiCommand }; static auto rCmdProfile = registerCommand("profile"); + +} // namespace nix diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index 8dd608d23b08..129f06f6c5de 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -3,7 +3,7 @@ #include -using namespace nix; +namespace nix { struct CmdRealisation : NixMultiCommand { @@ -78,3 +78,5 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON }; static auto rCmdRealisationInfo = registerCommand2({"realisation", "info"}); + +} // namespace nix diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 5f77700d65a8..713c0d5d1795 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -5,8 +5,7 @@ #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/registry.hh" -using namespace nix; -using namespace nix::flake; +namespace nix { class RegistryCommand : virtual Args { @@ -268,3 +267,5 @@ struct CmdRegistry : NixMultiCommand }; static auto rCmdRegistry = registerCommand("registry"); + +} // namespace nix diff --git a/src/nix/run.cc b/src/nix/run.cc index a87a7679d435..d9056ba91df1 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -20,12 +20,10 @@ extern char ** environ __attribute__((weak)); -using namespace nix; +namespace nix { std::string chrootHelperName = "__run_in_chroot"; -namespace nix { - /* Convert `env` to a list of strings suitable for `execve`'s `envp` argument. */ Strings toEnvp(StringMap env) { @@ -108,8 +106,6 @@ void execProgramInStore( throw SysError("unable to execute '%s'", program); } -} // namespace nix - struct CmdRun : InstallableValueCommand, MixEnvironment { using InstallableCommand::run; @@ -264,3 +260,5 @@ void chrootHelper(int argc, char ** argv) throw Error("mounting the Nix store on '%s' is not supported on this platform", storeDir); #endif } + +} // namespace nix diff --git a/src/nix/search.cc b/src/nix/search.cc index 4f5ae8cafaa2..c4b37b8a7149 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -16,9 +16,10 @@ #include "nix/util/strings.hh" -using namespace nix; using json = nlohmann::json; +namespace nix { + std::string wrap(std::string prefix, std::string s) { return concatStrings(prefix, s, ANSI_NORMAL); @@ -196,3 +197,5 @@ struct CmdSearch : InstallableValueCommand, MixJSON }; static auto rCmdSearch = registerCommand("search"); + +} // namespace nix diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index c72204cea3d4..bcc577317983 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -7,7 +7,7 @@ #include -using namespace nix; +namespace nix { struct CmdCopySigs : StorePathsCommand { @@ -226,3 +226,5 @@ struct CmdKey : NixMultiCommand }; static auto rCmdKey = registerCommand("key"); + +} // namespace nix diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index cbd4e04f6aa6..bcdbee75f211 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -4,7 +4,7 @@ #include "nix/store/store-cast.hh" #include "nix/store/log-store.hh" -using namespace nix; +namespace nix { struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand { @@ -37,3 +37,5 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand }; static auto rCmdCopyLog = registerCommand2({"store", "copy-log"}); + +} // namespace nix diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 5d28979a493c..c18b5187764d 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -4,7 +4,7 @@ #include "nix/store/store-cast.hh" #include "nix/store/gc-store.hh" -using namespace nix; +namespace nix { struct CmdStoreDelete : StorePathsCommand { @@ -45,3 +45,5 @@ struct CmdStoreDelete : StorePathsCommand }; static auto rCmdStoreDelete = registerCommand2({"store", "delete"}); + +} // namespace nix diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index f4624a40e8af..c2923fdc5645 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -6,7 +6,7 @@ #include "nix/store/gc-store.hh" #include "nix/util/error.hh" -using namespace nix; +namespace nix { struct CmdStoreGC : StoreCommand, MixDryRun { @@ -49,3 +49,5 @@ struct CmdStoreGC : StoreCommand, MixDryRun }; static auto rCmdStoreGC = registerCommand2({"store", "gc"}); + +} // namespace nix diff --git a/src/nix/store-info.cc b/src/nix/store-info.cc index 4526d9cda11d..ade9230c98ca 100644 --- a/src/nix/store-info.cc +++ b/src/nix/store-info.cc @@ -5,7 +5,7 @@ #include -using namespace nix; +namespace nix { struct CmdInfoStore : StoreCommand, MixJSON { @@ -45,3 +45,5 @@ struct CmdInfoStore : StoreCommand, MixJSON }; static auto rCmdInfoStore = registerCommand2({"store", "info"}); + +} // namespace nix diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc index cd243691c539..a6eee6dfbf2c 100644 --- a/src/nix/store-repair.cc +++ b/src/nix/store-repair.cc @@ -1,7 +1,7 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" -using namespace nix; +namespace nix { struct CmdStoreRepair : StorePathsCommand { @@ -25,3 +25,5 @@ struct CmdStoreRepair : StorePathsCommand }; static auto rStoreRepair = registerCommand2({"store", "repair"}); + +} // namespace nix diff --git a/src/nix/store.cc b/src/nix/store.cc index 45e505d06984..ee3c74875295 100644 --- a/src/nix/store.cc +++ b/src/nix/store.cc @@ -1,6 +1,6 @@ #include "nix/cmd/command.hh" -using namespace nix; +namespace nix { struct CmdStore : NixMultiCommand { @@ -24,3 +24,5 @@ struct CmdStore : NixMultiCommand }; static auto rCmdStore = registerCommand("store"); + +} // namespace nix diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 278eaef374d5..05e47f79c36b 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -38,8 +38,7 @@ # include "nix/util/cgroup.hh" #endif -using namespace nix; -using namespace nix::daemon; +namespace nix { /** * Settings related to authenticating clients for the Nix daemon. @@ -106,12 +105,12 @@ static ssize_t splice(int fd_in, void * off_in, int fd_out, void * off_out, size { // We ignore most parameters, we just have them for conformance with the linux syscall std::vector buf(8192); - auto read_count = read(fd_in, buf.data(), buf.size()); + auto read_count = ::read(fd_in, buf.data(), buf.size()); if (read_count == -1) return read_count; auto write_count = decltype(read_count)(0); while (write_count < read_count) { - auto res = write(fd_out, buf.data() + write_count, read_count - write_count); + auto res = ::write(fd_out, buf.data() + write_count, read_count - write_count); if (res == -1) return res; write_count += res; @@ -250,6 +249,8 @@ static void daemonLoop( std::optional forceTrustClientOpt, std::filesystem::path socketPath) { + using namespace nix::daemon; + if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -404,7 +405,7 @@ static void forwardStdioConnection(RemoteStore & store) */ static void processStdioConnection(ref store, TrustedFlag trustClient) { - processConnection(store, FdSource(STDIN_FILENO), FdSink(STDOUT_FILENO), trustClient, NotRecursive); + processConnection(store, FdSource(STDIN_FILENO), FdSink(STDOUT_FILENO), trustClient, daemon::NotRecursive); } /** @@ -622,3 +623,5 @@ struct CmdDaemon : StoreConfigCommand }; static auto rCmdDaemon = registerCommand2({"daemon"}); + +} // namespace nix diff --git a/src/nix/unix/store-roots-daemon.cc b/src/nix/unix/store-roots-daemon.cc index c9728a2efe98..85b0a67a9156 100644 --- a/src/nix/unix/store-roots-daemon.cc +++ b/src/nix/unix/store-roots-daemon.cc @@ -7,7 +7,7 @@ #include -using namespace nix; +namespace nix { struct CmdRootsDaemon : StoreConfigCommand { @@ -64,3 +64,5 @@ struct CmdRootsDaemon : StoreConfigCommand }; static auto rCmdStoreRootsDaemon = registerCommand2({"store", "roots-daemon"}); + +} // namespace nix diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17b410b32549..3136f4d5fd15 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -13,7 +13,7 @@ #include "nix/util/config-global.hh" #include "self-exe.hh" -using namespace nix; +namespace nix { /** * Check whether a path has a "profiles" component. @@ -210,3 +210,5 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand }; static auto rCmdUpgradeNix = registerCommand("upgrade-nix"); + +} // namespace nix diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 6fb10bf3e897..27a349062e41 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -9,7 +9,7 @@ #include "nix/util/exit.hh" -using namespace nix; +namespace nix { struct CmdVerify : StorePathsCommand { @@ -186,3 +186,5 @@ struct CmdVerify : StorePathsCommand }; static auto rCmdVerify = registerCommand2({"store", "verify"}); + +} // namespace nix diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index e7224f6cc999..d953b022f39f 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -7,7 +7,7 @@ #include -using namespace nix; +namespace nix { static std::string hilite(const std::string & s, size_t pos, size_t len, const std::string & colour = ANSI_RED) { @@ -295,3 +295,5 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions }; static auto rCmdWhyDepends = registerCommand("why-depends"); + +} // namespace nix diff --git a/tests/functional/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc index f562bab56af0..dae7a34c9531 100644 --- a/tests/functional/plugins/plugintest.cc +++ b/tests/functional/plugins/plugintest.cc @@ -1,7 +1,7 @@ #include "nix/util/config-global.hh" #include "nix/expr/primops.hh" -using namespace nix; +namespace nix { struct MySettings : Config { @@ -25,3 +25,5 @@ static RegisterPrimOp rp({ .arity = 0, .impl = prim_anotherNull, }); + +} // namespace nix diff --git a/tests/functional/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc index 6cfe50047af6..cab02d799d79 100644 --- a/tests/functional/test-libstoreconsumer/main.cc +++ b/tests/functional/test-libstoreconsumer/main.cc @@ -3,10 +3,10 @@ #include "nix/store/build-result.hh" #include -using namespace nix; - int main(int argc, char ** argv) { + using namespace nix; + try { if (argc != 2) { std::cerr << "Usage: " << argv[0] << " store/path/to/something.drv\n"; From 693a14190aabf379ef1ed67d219fa665bfc788a8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 10 Apr 2026 17:15:48 -0400 Subject: [PATCH 226/555] libutil-tests: Run tests under `enosys` to exercise syscall fallbacks Add a second test run that uses `enosys --syscall openat2 --syscall fchmodat2` to make both syscalls return `ENOSYS` via seccomp. This forces the iterative `openat`-based fallback in `openFileEnsureBeneathNoSymlinks` and the `/proc/self/fd` fallback in `fchmodatTryNoFollow`, giving CI coverage for code paths that are otherwise only reachable on older kernels, or other Unix OSes. - `meson.build`: new `nix-util-tests-without-new-syscalls` test target, gated on Linux and `enosys` being available. - `package.nix`: new `run-without-new-syscalls` passthru test, gated on `hostPlatform.isLinux && buildPlatform.canExecute hostPlatform` (enosys uses seccomp and can't work through an emulator). - `everything.nix`: include the new test in `checkInputs` under the same platform guard so Hydra builds it on Linux. Idea-by: Sergei Zimmerman --- packaging/everything.nix | 5 +++++ src/libutil-tests/meson.build | 24 ++++++++++++++++++++++++ src/libutil-tests/package.nix | 28 +++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/packaging/everything.nix b/packaging/everything.nix index 4d8f94f4b9ab..751d861c9e80 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -140,6 +140,11 @@ stdenv.mkDerivation (finalAttrs: { # Make sure the functional tests have passed nix-functional-tests ] + ++ + lib.optionals (stdenv.hostPlatform.isLinux && stdenv.buildPlatform.canExecute stdenv.hostPlatform) + [ + nix-util-tests.tests.run-without-new-syscalls + ] ++ lib.optionals (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index a76d8d97092b..ff6e539c8a2e 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -126,3 +126,27 @@ test( }, protocol : 'gtest', ) + +# Run the same tests again under `enosys -d openat2` to exercise the +# iterative (non-openat2) fallback path on Linux. `enosys` uses +# seccomp to make `openat2` return `ENOSYS`, which the wrapper in +# file-system-at.cc caches and falls back to the iterative impl. +if host_machine.system() == 'linux' + enosys = find_program('enosys', required : false) + if enosys.found() + test( + meson.project_name() + '-without-new-syscalls', + enosys, + args : [ + '--syscall', 'openat2', + '--syscall', 'fchmodat2', + '--', + this_exe, + ], + env : { + '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + }, + protocol : 'gtest', + ) + endif +endif diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index f24d69243b86..03c1c2d0efaf 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -11,6 +11,7 @@ rapidcheck, gtest, runCommand, + util-linux, # Configuration Options @@ -44,6 +45,9 @@ mkMesonExecutable (finalAttrs: { nix-util-test-support rapidcheck gtest + ] + ++ lib.optionals stdenv.hostPlatform.isLinux [ + util-linux ]; mesonFlags = [ @@ -67,7 +71,29 @@ mkMesonExecutable (finalAttrs: { touch $out '' ); - }; + } + // + lib.optionalAttrs + (stdenv.hostPlatform.isLinux && stdenv.buildPlatform.canExecute stdenv.hostPlatform) + { + # Run the same tests with newer syscalls disabled via seccomp, + # to exercise fallback paths (iterative openat for openat2, + # /proc/self/fd for fchmodat2). + run-without-new-syscalls = + runCommand "${finalAttrs.pname}-run-without-new-syscalls" + { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + nativeBuildInputs = [ util-linux ]; + } + '' + export _NIX_TEST_UNIT_DATA=${./data} + enosys \ + --syscall openat2 \ + --syscall fchmodat2 \ + -- ${lib.getExe finalAttrs.finalPackage} + touch $out + ''; + }; }; meta = { From 4d470c30091375501a2a794683a5b73b562cee91 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Apr 2026 18:08:50 -0400 Subject: [PATCH 227/555] libutil: Ensure `openFileEnsureBeneathNoSymlinks` throws enough, simplify interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We had some issues where `openFileEnsureBeneathNoSymlinks` was not always throwing `SymlinkNotAllowed` when expected. This wasn't necessarily a problem, but opened the door to a bad situation where callers might handle different errors differently in unexpected and unintentional ways. When I was investigating this, I realized the presence or absence of `O_NOFOLLOW` in the caller-provided flags was doubling the number of cases that we needed to handle, and this was becoming quite burdensome. The function doesn't need to work with arbitrary, e.g. user-provided flags. All its callers (at least eventually, if you trace back far enough) in Nix should always have statically-known flags. So we can use our own invariants to cut down that state space. Specifically, callers of `openFileEnsureBeneathNoSymlinks` should never pass `O_NOFOLLOW` — the function owns symlink policy (it's in the name). Add an assert enforcing this and drop `O_NOFOLLOW` from the one caller that had it (`fs-sink.cc`'s directory open). Each implementation then handles trailing-symlink detection internally: - The `openat2` path uses `RESOLVE_NO_SYMLINKS`, which gives a uniform `ELOOP` for every trailing-symlink case. - The iterative fallback adds its own defensive `O_NOFOLLOW` on the final `::openat`, with post-checks for two kernel oddities where `O_NOFOLLOW` produces unexpected results (`O_DIRECTORY` gives `ENOTDIR` instead of `ELOOP`; `O_PATH` silently succeeds with a symlink fd). With `O_PATH` (without `O_DIRECTORY`), a trailing symlink is intentionally allowed — the caller gets a path fd to the symlink itself. Interior symlinks are always rejected regardless of flags. Also add comprehensive tests covering every flag combination (`O_RDONLY`, `O_DIRECTORY`, `O_PATH | O_DIRECTORY`, `O_PATH`), including symlink rejection, trailing-symlink-fd success for `O_PATH`, non-symlink `ENOTDIR` propagation, happy paths, and death tests for the `O_NOFOLLOW` assert. Reviewed-by: Sergei Zimmerman --- src/libstore/unix/build/derivation-builder.cc | 2 +- src/libutil-tests/file-system-at.cc | 173 ++++++++++++++---- src/libutil/fs-sink.cc | 2 +- .../include/nix/util/file-system-at.hh | 48 +++-- src/libutil/unix/file-system-at.cc | 70 ++++++- 5 files changed, 237 insertions(+), 58 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 8288a4a31851..85aa98c7ae87 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1273,7 +1273,7 @@ void DerivationBuilderImpl::writeBuilderFile(const std::string & name, std::stri a single path component without any `..`, `.` components. */ auto relPath = CanonPath::fromFilename(name); AutoCloseFD fd = openFileEnsureBeneathNoSymlinks( - tmpDirFd.get(), relPath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL | O_NOFOLLOW, 0666); + tmpDirFd.get(), relPath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL, 0666); auto path = tmpDir / relPath.rel(); /* This is used only for error messages. */ if (!fd) throw SysError("creating file %s", PathFmt(path)); diff --git a/src/libutil-tests/file-system-at.cc b/src/libutil-tests/file-system-at.cc index fa646820d7c2..82b73d7af0d9 100644 --- a/src/libutil-tests/file-system-at.cc +++ b/src/libutil-tests/file-system-at.cc @@ -4,6 +4,7 @@ #include "nix/util/file-system-at.hh" #include "nix/util/file-system.hh" #include "nix/util/fs-sink.hh" +#include "nix/util/tests/gmock-matchers.hh" namespace nix { @@ -120,51 +121,80 @@ TEST(openFileEnsureBeneathNoSymlinks, works) auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); - // Helper to open files with platform-specific arguments + /* Helpers that wrap `openFileEnsureBeneathNoSymlinks` to throw + `SysError` on null-fd failure. This way every failure is an + exception and tests can use plain `EXPECT_THROW`/`EXPECT_TRUE` + without juggling errno-vs-throw state. Callers that want to + check a specific errno can catch `SysError` and inspect + `.errNo`. */ + auto check = [](std::string_view what, AutoCloseFD fd) { + if (!fd) + throw SysError("openFileEnsureBeneathNoSymlinks: %s", what); + return fd; + }; + auto openRead = [&](std::string_view path) -> AutoCloseFD { - return openFileEnsureBeneathNoSymlinks( - dirFd.get(), - CanonPath(path), + return check( + path, + openFileEnsureBeneathNoSymlinks( + dirFd.get(), + CanonPath(path), #ifdef _WIN32 - FILE_READ_DATA | FILE_READ_ATTRIBUTES | SYNCHRONIZE, - 0 + FILE_READ_DATA | FILE_READ_ATTRIBUTES | SYNCHRONIZE, + 0 #else - O_RDONLY, - 0 + O_RDONLY, + 0 #endif - ); + )); }; auto openReadDir = [&](std::string_view path) -> AutoCloseFD { - return openFileEnsureBeneathNoSymlinks( - dirFd.get(), - CanonPath(path), + return check( + path, + openFileEnsureBeneathNoSymlinks( + dirFd.get(), + CanonPath(path), #ifdef _WIN32 - FILE_READ_ATTRIBUTES | SYNCHRONIZE, - FILE_DIRECTORY_FILE + FILE_READ_ATTRIBUTES | SYNCHRONIZE, + FILE_DIRECTORY_FILE #else - O_RDONLY | O_DIRECTORY, - 0 + O_RDONLY | O_DIRECTORY, + 0 #endif - ); + )); + }; + +#if defined(__linux__) + auto openReadDirPath = [&](std::string_view path) -> AutoCloseFD { + return check( + path, openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath(path), O_PATH | O_DIRECTORY | O_CLOEXEC, 0)); }; + auto openPath = [&](std::string_view path) -> AutoCloseFD { + return check(path, openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath(path), O_PATH | O_CLOEXEC, 0)); + }; +#endif auto openCreateExclusive = [&](std::string_view path) -> AutoCloseFD { - return openFileEnsureBeneathNoSymlinks( - dirFd.get(), - CanonPath(path), + return check( + path, + openFileEnsureBeneathNoSymlinks( + dirFd.get(), + CanonPath(path), #ifdef _WIN32 - FILE_WRITE_DATA | SYNCHRONIZE, - 0, - FILE_CREATE // Create new file, fail if exists (equivalent to O_CREAT | O_EXCL) + FILE_WRITE_DATA | SYNCHRONIZE, + 0, + FILE_CREATE // Create new file, fail if exists (equivalent to O_CREAT | O_EXCL) #else - O_CREAT | O_WRONLY | O_EXCL, - 0666 + O_CREAT | O_WRONLY | O_EXCL, + 0666 #endif - ); + )); }; - // Test that symlinks are detected and rejected + using nix::testing::ThrowsSysError; + + // Symlinks in the path (any position) are rejected. EXPECT_THROW(openRead("a/absolute_symlink"), SymlinkNotAllowed); EXPECT_THROW(openRead("a/relative_symlink"), SymlinkNotAllowed); EXPECT_THROW(openRead("a/absolute_symlink/a"), SymlinkNotAllowed); @@ -173,21 +203,94 @@ TEST(openFileEnsureBeneathNoSymlinks, works) EXPECT_THROW(openRead("a/b/c/d"), SymlinkNotAllowed); EXPECT_THROW(openRead("a/broken_symlink"), SymlinkNotAllowed); + /* Trailing symlink with `O_DIRECTORY` (no `O_NOFOLLOW`): the + syscall reports `ELOOP`, handled by the direct + `SymlinkNotAllowed` throw in `openFileEnsureBeneathNoSymlinks`. */ + EXPECT_THROW(openReadDir("a/absolute_symlink"), SymlinkNotAllowed); + EXPECT_THROW(openReadDir("a/relative_symlink"), SymlinkNotAllowed); + EXPECT_THROW(openReadDir("a/b/c"), SymlinkNotAllowed); + EXPECT_THROW(openReadDir("a/broken_symlink"), SymlinkNotAllowed); + +#if defined(__linux__) + // Same thing for `O_PATH | O_DIRECTORY` (Linux-only flag). + EXPECT_THROW(openReadDirPath("a/absolute_symlink"), SymlinkNotAllowed); + EXPECT_THROW(openReadDirPath("a/relative_symlink"), SymlinkNotAllowed); + EXPECT_THROW(openReadDirPath("a/b/c"), SymlinkNotAllowed); + EXPECT_THROW(openReadDirPath("a/broken_symlink"), SymlinkNotAllowed); + + /* `O_PATH` on a trailing symlink is allowed: the caller gets a + path fd referring to the symlink itself. Interior symlinks are + still rejected. */ + EXPECT_TRUE(openPath("a/absolute_symlink")); + EXPECT_TRUE(openPath("a/relative_symlink")); + EXPECT_TRUE(openPath("a/b/c")); + EXPECT_TRUE(openPath("a/broken_symlink")); + EXPECT_THROW(openPath("a/absolute_symlink/a"), SymlinkNotAllowed); + EXPECT_THROW(openPath("a/absolute_symlink/c/d"), SymlinkNotAllowed); + EXPECT_THROW(openPath("a/relative_symlink/c"), SymlinkNotAllowed); + EXPECT_THROW(openPath("a/b/c/d"), SymlinkNotAllowed); +#endif + #if !defined(_WIN32) && !defined(__CYGWIN__) - // This returns ELOOP on cygwin when O_NOFOLLOW is used - EXPECT_FALSE(openCreateExclusive("a/broken_symlink")); - /* Sanity check, no symlink shenanigans and behaves the same as regular openat with O_EXCL | O_CREAT. */ - EXPECT_EQ(errno, EEXIST); + /* Sanity check: behaves the same as plain openat with + `O_EXCL | O_CREAT` on an existing broken-symlink path — + the symlink counts as "already there". */ + EXPECT_THAT([&] { openCreateExclusive("a/broken_symlink"); }, ThrowsSysError(EEXIST)); #endif EXPECT_THROW(openCreateExclusive("a/absolute_symlink/broken_symlink"), SymlinkNotAllowed); - // Test invalid paths - EXPECT_FALSE(openRead("c/d/regular/a")); - EXPECT_FALSE(openReadDir("c/d/regular")); + // Non-symlink failure modes: errno must survive. + EXPECT_THAT([&] { openRead("c/d/regular/a"); }, ThrowsSysError(ENOTDIR)); + EXPECT_THAT([&] { openReadDir("c/d/regular"); }, ThrowsSysError(ENOTDIR)); + EXPECT_THAT([&] { openRead("c/d/nonexistent"); }, ThrowsSysError(ENOENT)); - // Test valid paths work + // Happy paths. EXPECT_TRUE(openRead("c/d/regular")); + EXPECT_TRUE(openReadDir("c")); + EXPECT_TRUE(openReadDir("c/d")); EXPECT_TRUE(openCreateExclusive("a/regular")); +#if defined(__linux__) + EXPECT_TRUE(openReadDirPath("c")); + EXPECT_TRUE(openReadDirPath("c/d")); + EXPECT_TRUE(openPath("c/d/regular")); + EXPECT_TRUE(openPath("c")); +#endif +} + +/* ---------------------------------------------------------------------------- + * openFileEnsureBeneathNoSymlinks — O_NOFOLLOW is forbidden + * --------------------------------------------------------------------------*/ + +#if !defined(_WIN32) +TEST(openFileEnsureBeneathNoSymlinksDeathTest, rejectsONofollow) +{ + std::filesystem::path tmpDir = nix::createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true); + + { + RestoreSink sink(/*startFsync=*/false); + sink.dstPath = tmpDir; + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink & crf) {}); + } + + auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); + + /* The function asserts that callers don't pass `O_NOFOLLOW` — it + owns symlink policy. Verify the assert fires for every flag + combination that includes `O_NOFOLLOW`. */ + EXPECT_DEATH( + openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath("file"), O_RDONLY | O_NOFOLLOW, 0), "O_NOFOLLOW"); + EXPECT_DEATH( + openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath("file"), O_RDONLY | O_DIRECTORY | O_NOFOLLOW, 0), + "O_NOFOLLOW"); +# if defined(__linux__) + EXPECT_DEATH(openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath("file"), O_PATH | O_NOFOLLOW, 0), "O_NOFOLLOW"); + EXPECT_DEATH( + openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath("file"), O_PATH | O_DIRECTORY | O_NOFOLLOW, 0), + "O_NOFOLLOW"); +# endif } +#endif } // namespace nix diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 6aebb3f5ea94..b0bb20d99fc4 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -90,7 +90,7 @@ void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallba FILE_READ_ATTRIBUTES | SYNCHRONIZE, FILE_DIRECTORY_FILE #else - O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC, + O_RDONLY | O_DIRECTORY | O_CLOEXEC, 0 #endif ); diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh index 75f4e6b2cff1..0f4678461ca0 100644 --- a/src/libutil/include/nix/util/file-system-at.hh +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -72,25 +72,45 @@ PosixStat fstatat(Descriptor dirFd, const std::filesystem::path & path); OsString readLinkAt(Descriptor dirFd, const CanonPath & path); /** - * Safe(r) function to open a file relative to dirFd, while - * disallowing escaping from a directory and any symlinks in the process. + * Open a file relative to @p dirFd, ensuring the path stays beneath + * @p dirFd and that no path component is a symlink (with the + * exception that `O_PATH` (without `O_DIRECTORY`) on Unix permits a + * trailing symlink). * - * @note On Windows, implemented via NtCreateFile single path component traversal - * with FILE_OPEN_REPARSE_POINT. On Unix, uses RESOLVE_BENEATH with openat2 when - * available, or falls back to openat single path component traversal. + * Callers must not pass `O_NOFOLLOW` on Unix (enforced by assert); + * this function owns symlink policy and handles the flag internally. * * @param dirFd Directory handle to open relative to - * @param path Relative path (no .. or . components) - * @param desiredAccess (Windows) Windows ACCESS_MASK (e.g., GENERIC_READ, FILE_WRITE_DATA) - * @param createOptions (Windows) Windows create options (e.g., FILE_NON_DIRECTORY_FILE) - * @param createDisposition (Windows) FILE_OPEN, FILE_CREATE, etc. - * @param flags (Unix) O_* flags - * @param mode (Unix) Mode for O_{CREAT,TMPFILE} + * @param path Relative path (with no `..` or `.` components) * - * @pre path.isRoot() is false + * @param desiredAccess (Windows) Windows `ACCESS_MASK` + * @param createOptions (Windows) Windows create options + * @param createDisposition (Windows) `FILE_OPEN`, `FILE_CREATE`, etc. + * + * @param flags (Unix) `O_*` flags (must not include `O_NOFOLLOW`) + * @param mode (Unix) Mode for `O_{CREAT,TMPFILE}` + * + * @pre `path.isRoot()` is false + * + * @throws SymlinkNotAllowed if an interior path component is a + * symlink, or if the final component is a symlink and `O_PATH` + * (without `O_DIRECTORY`) was *not* passed. With `O_PATH` + * (without `O_DIRECTORY`) on Unix, a trailing symlink is + * permitted and the caller receives a "path fd" to the symlink + * itself. + * + * @note With `O_CREAT | O_EXCL`, a pre-existing symlink at the + * final component causes the OS to return `EEXIST` rather + * than `ELOOP`, so `SymlinkNotAllowed` is *not* thrown — the + * caller sees a failed descriptor with `errno == EEXIST`. + * + * @return A valid descriptor on success, or an invalid descriptor + * on non-symlink errors (Unix: `errno` set, e.g. `ENOENT`, + * `ENOTDIR`, `EACCES`; Windows: last error set). The caller is + * responsible for checking the return value. * - * @throws SymlinkNotAllowed if any path components are symlinks - * @throws SystemError on other errors + * `errno` will never be `ELOOP` because that case is translated + * to a `SymlinkNotAllowed` throw instead. */ AutoCloseFD openFileEnsureBeneathNoSymlinks( Descriptor dirFd, diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index 01a4c664d31d..762a245ded04 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -175,6 +175,9 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat }); if (errno == ENOTDIR) /* Path component might be a symlink. */ { + /* Does not follow final symlink. We know `component` is a + single component so we don't have to worry about intermediate + symlinks either. */ if (auto st = maybeFstatat(getParentFd(), component); st && S_ISLNK(st->st_mode)) throw SymlinkNotAllowed(path2); errno = ENOTDIR; /* Restore the errno. */ @@ -188,25 +191,78 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat parentFd = std::move(parentFd2); } - AutoCloseFD res = ::openat(getParentFd(), std::string(path.baseName().value()).c_str(), flags | O_NOFOLLOW, mode); - if (!res && errno == ELOOP) - throw SymlinkNotAllowed(path); + auto lastComponent = std::string(path.baseName().value()); + AutoCloseFD res = ::openat(getParentFd(), lastComponent.c_str(), flags | O_NOFOLLOW, mode); + + if (!res) { + if (errno == ELOOP) + throw SymlinkNotAllowed(path); + /* `O_DIRECTORY | O_NOFOLLOW` on a trailing symlink returns + `ENOTDIR` rather than `ELOOP`. Post-check via `fstatat` to + disambiguate — only on the error path, so the common + successful-directory-open case pays no extra syscall. */ + if (errno == ENOTDIR) { + if (auto st = maybeFstatat(getParentFd(), lastComponent); st && S_ISLNK(st->st_mode)) + throw SymlinkNotAllowed(path); + /* Put back errno so the caller will get the original + error. */ + errno = ENOTDIR; + } + return res; + } + + /* For `O_PATH`, the defensive `| O_NOFOLLOW` we added above + means a trailing symlink silently succeeds with an fd to the + symlink itself (`O_PATH | O_NOFOLLOW` is the idiomatic way to + obtain a symlink fd). This is intentional — `O_PATH` callers + are asking for a path reference, and interior symlinks are + already guarded by the component-by-component walk above. */ + return res; } AutoCloseFD openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) { - assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */ + /* Just in case the invariant is somehow broken. */ + assert(!path.rel().starts_with('/')); assert(!path.isRoot()); + + /* We don't want callers of this function to think about the presence or + absence of `O_NOFOLLOW`. "ensure beneath no symlinks" is in the name, so + we want them to trust us to handle it instead. */ + assert(!(flags & O_NOFOLLOW)); + + /* See doxygen in `file-system-at.hh` for why we reject this. */ + #if HAVE_OPENAT2 - auto maybeFd = linux::openat2( - dirFd, path.rel_c_str(), flags, static_cast(mode), RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS); - if (maybeFd) { + /* Two things are being fixed here: + + 1. For `O_PATH` (without `O_DIRECTORY`), add `O_NOFOLLOW` so + that a trailing symlink returns an fd to the symlink itself + rather than `ELOOP`. `O_PATH | O_NOFOLLOW` is the idiomatic + way to obtain a symlink fd, and `RESOLVE_NO_SYMLINKS` does + not refuse it. + + 2. We must not add `O_NOFOLLOW` when `O_DIRECTORY` is set, + because `O_DIRECTORY | O_NOFOLLOW` on a trailing symlink + returns `ENOTDIR` instead of `ELOOP`. Interior symlinks are + still caught by `RESOLVE_NO_SYMLINKS` regardless, but the + non-inclusion of `O_NOFOLLOW` is needed for + `RESOLVE_NO_SYMLINKS` to make the final symlink an `ELOOP` + rather than `O_DIRECTORY` making it an `ENOTDIR`. + + For other cases, `O_NOFOLLOW` doesn't really matter, but we + default to not including it. */ + auto flagsAdj = (flags & O_PATH) && !(flags & O_DIRECTORY) ? flags | O_NOFOLLOW : flags; + + if (auto maybeFd = linux::openat2( + dirFd, path.rel_c_str(), flagsAdj, static_cast(mode), RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS)) { if (!*maybeFd && errno == ELOOP) throw SymlinkNotAllowed(path); return std::move(*maybeFd); } #endif + return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode); } From dc83e83e73cbbcd07bd7fb45d85474bb798648cc Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 18 Mar 2026 15:43:50 -0400 Subject: [PATCH 228/555] Add option `--skip-alive` to `nix store delete` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows garbage collecting a closure. `nix store delete --recursive --skip-alive` is morally equivalent to ```bash for validPath in $(nix path-info --recursive foo); do nix store delete "$validPath" || true done ``` Resolves NixOS#7239 Co-authored-by: Théophane Hufschmitt Co-authored-by: Alexander Bantyev Signed-off-by: Lisanna Dettwyler --- doc/manual/rl-next/closure-gc.md | 9 ++ src/libstore/daemon.cc | 17 ++- src/libstore/gc.cc | 115 +++++++++++------- src/libstore/include/nix/store/gc-store.hh | 8 +- .../include/nix/store/worker-protocol.hh | 8 ++ src/libstore/remote-store.cc | 26 +++- src/libstore/worker-protocol.cc | 32 +++++ .../nix-collect-garbage.cc | 1 + src/nix/nix-store/nix-store.cc | 5 +- src/nix/store-delete.cc | 11 +- src/nix/store-gc.cc | 1 + tests/functional/gc-closure.sh | 29 +++++ tests/functional/meson.build | 1 + 13 files changed, 213 insertions(+), 50 deletions(-) create mode 100644 doc/manual/rl-next/closure-gc.md create mode 100755 tests/functional/gc-closure.sh diff --git a/doc/manual/rl-next/closure-gc.md b/doc/manual/rl-next/closure-gc.md new file mode 100644 index 000000000000..fc24d5e08b9b --- /dev/null +++ b/doc/manual/rl-next/closure-gc.md @@ -0,0 +1,9 @@ +--- +synopsis: "Added `--skip-alive` option to `nix store delete` for collecting garbage within a closure" +issues: 7239 +prs: 15236 +--- + +`nix store delete --recursive --skip-alive` can be used to collect garbage +within a closure, in which case it will only collect the dead paths that are +part of the closure of its arguments. diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 31d30097fceb..52e1c121c3e7 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -17,6 +17,7 @@ #include "nix/util/args.hh" #include "nix/util/logging.hh" #include "nix/store/globals.hh" +#include #ifndef _WIN32 // TODO need graceful async exit support on Windows? # include "nix/util/monitor-fd.hh" @@ -745,13 +746,27 @@ static void performOp( case WorkerProto::Op::CollectGarbage: { GCOptions options; options.action = WorkerProto::Serialise::read(*store, rconn); - options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); + if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); + } else { + auto paths = WorkerProto::Serialise::read(*store, rconn); + if (options.action != GCAction::gcDeleteSpecific && paths.empty()) + options.pathsToDelete = GCOptions::WholeStore{}; + else + options.pathsToDelete = paths; + } conn.from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(conn.from); readInt(conn.from); readInt(conn.from); + if (options.action == GCAction::gcDeleteDead && std::holds_alternative(options.pathsToDelete) + && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + throw Error( + "Garbage collecting specific paths requested but it is not supported by the negotiated protocol"); + } + GCResults results; logger->startWork(); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f3bded97e4fc..835d06864671 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,3 +1,4 @@ +#include "nix/store/gc-store.hh" #include "nix/store/local-gc.hh" #include "nix/store/local-settings.hh" #include "nix/store/local-store.hh" @@ -22,6 +23,7 @@ #include #include #include +#include #if HAVE_STATVFS # include #endif @@ -360,6 +362,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) boost::unordered_flat_set> roots, dead, alive; + /* Return early if nothing to delete */ + if (std::holds_alternative(options.pathsToDelete) + && std::get(options.pathsToDelete).empty()) + return; + struct Shared { // The temp roots only store the hash part to make it easier to @@ -577,7 +584,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) via the referrers edges and optionally derivers and derivation output edges. If none of those paths are roots, then all visited paths are garbage and are deleted. */ - auto deleteReferrersClosure = [&](const StorePath & start) { + auto maybeDeleteReferrersClosure = [&](const StorePath & start) { StorePathSet visited; std::queue todo; @@ -634,7 +641,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) return markAlive(); } - if (options.action == GCOptions::gcDeleteSpecific && !options.pathsToDelete.count(*path)) + if (std::holds_alternative(options.pathsToDelete) + && !std::get(options.pathsToDelete).contains(*path)) return; { @@ -694,50 +702,73 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - /* Either delete all garbage paths, or just the specified - paths (for gcDeleteSpecific). */ - if (options.action == GCOptions::gcDeleteSpecific) { - - for (auto & i : options.pathsToDelete) { - deleteReferrersClosure(i); - if (!dead.count(i)) - throw Error( - "Cannot delete path '%1%' since it is still alive. " - "To find out why, use: " - "nix-store --query --roots and nix-store --query --referrers", - printStorePath(i)); - } + try { + /* Either delete all garbage paths, or just the specified paths. */ + std::visit( + overloaded{ + [&](const StorePathSet & paths) { + for (auto & i : paths) { + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage within specified paths..."); + break; + case GCOptions::gcDeleteSpecific: + printInfo("deleting specified paths..."); + break; + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } - } else if (options.maxFreed > 0) { + maybeDeleteReferrersClosure(i); - if (shouldDelete) - printInfo("deleting garbage..."); - else - printInfo("determining live/dead paths..."); + if (options.action == GCOptions::gcDeleteSpecific && !dead.count(i)) + throw Error( + "Cannot delete path '%1%' since it is still alive. " + "To find out why, use: " + "nix-store --query --roots and nix-store --query --referrers", + printStorePath(i)); + } + }, + [&](const GCOptions::WholeStore & _) { + if (options.maxFreed == 0) + return; - try { - AutoCloseDir dir(opendir(config->realStoreDir.get().string().c_str())); - if (!dir) - throw SysError("opening directory %1%", PathFmt(config->realStoreDir.get())); - - /* Read the store and delete all paths that are invalid or - unreachable. We don't use readDirectory() here so that - GCing can start faster. */ - auto linksName = linksDir.filename(); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == ".." || name == linksName) - continue; + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage..."); + break; + case GCOptions::gcDeleteSpecific: + throw Error("Cannot delete the entire store"); + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } - if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) - deleteReferrersClosure(*storePath); - else - deleteFromStore(name, false); - } - } catch (GCLimitReached & e) { - } + AutoCloseDir dir(opendir(config->realStoreDir.get().string().c_str())); + if (!dir) + throw SysError("opening directory %1%", PathFmt(config->realStoreDir.get())); + + /* Read the store and delete all paths that are invalid or + unreachable. We don't use readDirectory() here so that + GCing can start faster. */ + auto linksName = linksDir.filename(); + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == ".." || name == linksName) + continue; + + if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) + maybeDeleteReferrersClosure(*storePath); + else + deleteFromStore(name, false); + } + }, + }, + options.pathsToDelete); + } catch (GCLimitReached & e) { } if (options.action == GCOptions::gcReturnLive) { diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index de016e241de4..5e23f2052472 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -39,6 +39,9 @@ struct GCOptions using GCAction = nix::GCAction; using enum GCAction; + struct WholeStore + {}; + GCAction action{gcDeleteDead}; /** @@ -50,9 +53,10 @@ struct GCOptions bool ignoreLiveness{false}; /** - * For `gcDeleteSpecific`, the paths to delete. + * The paths from which to delete. */ - StorePathSet pathsToDelete; + using GCPaths = std::variant; + GCPaths pathsToDelete; /** * Stop after at least `maxFreed` bytes have been freed. diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 8e20d3c992cf..ea1cbb502d7a 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -6,6 +6,7 @@ #include #include "nix/store/common-protocol.hh" +#include "nix/store/gc-store.hh" namespace nix { @@ -124,6 +125,11 @@ struct WorkerProto */ static constexpr std::string_view featureRealisationWithPath = "realisation-with-path-not-hash"; + /** + * Feature for garbage collecting a specific set of paths. + */ + static constexpr std::string_view featureDeleteDeadSpecific = "delete-dead-specific"; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. @@ -339,6 +345,8 @@ template<> DECLARE_WORKER_SERIALISER(std::optional); template<> DECLARE_WORKER_SERIALISER(WorkerProto::ClientHandshakeInfo); +template<> +DECLARE_WORKER_SERIALISER(GCOptions::GCPaths); template DECLARE_WORKER_SERIALISER(std::vector); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 3bb5b0755779..2def1bf87aa7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,3 +1,5 @@ +#include "nix/store/path.hh" +#include "nix/store/store-api.hh" #include "nix/util/serialise.hh" #include "nix/util/util.hh" #include "nix/store/path-with-outputs.hh" @@ -18,6 +20,7 @@ #include "nix/store/filetransfer.hh" #include "nix/util/signals.hh" #include "nix/util/socket.hh" +#include #ifndef _WIN32 # include @@ -679,9 +682,26 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { auto conn(getConnection()); - conn->to << WorkerProto::Op::CollectGarbage; - WorkerProto::write(*this, *conn, options.action); - WorkerProto::write(*this, *conn, options.pathsToDelete); + if (conn->protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + conn->to << WorkerProto::Op::CollectGarbage; + WorkerProto::write(*this, *conn, options.action); + WorkerProto::write(*this, *conn, options.pathsToDelete); + } else { + auto paths = std::visit( + overloaded{ + [&](const StorePathSet & paths) { + if (options.action != GCOptions::gcDeleteSpecific) + throw Error( + "Your daemon version is too old to support garbage collecting a specific set of paths"); + return paths; + }, + [](const GCOptions::WholeStore & _) { return StorePathSet{}; }, + }, + options.pathsToDelete); + conn->to << WorkerProto::Op::CollectGarbage; + WorkerProto::write(*this, *conn, options.action); + WorkerProto::write(*this, *conn, paths); + } conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 7f9bd163d91f..31373a128f96 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -1,3 +1,4 @@ +#include "nix/store/store-dir-config.hh" #include "nix/util/serialise.hh" #include "nix/store/path-with-outputs.hh" #include "nix/store/store-api.hh" @@ -8,8 +9,10 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/store/path-info.hh" #include "nix/util/json-utils.hh" +#include "nix/util/util.hh" #include +#include #include namespace nix { @@ -25,6 +28,7 @@ const WorkerProto::Version WorkerProto::latest = { std::string{ WorkerProto::featureRealisationWithPath, }, + std::string{WorkerProto::featureDeleteDeadSpecific}, }, }; @@ -529,4 +533,32 @@ void WorkerProto::Serialise::write(const StoreDirConfig & store, Wr WorkerProto::write(store, conn, static_cast(info)); } +GCOptions::GCPaths WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + uint8_t wholeStore; + conn.from >> wholeStore; + switch (wholeStore) { + case 0: + return WorkerProto::Serialise::read(store, conn); + case 1: + return GCOptions::WholeStore{}; + default: + throw Error("Invalid whole store indicator from remote"); + } +} + +void WorkerProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const GCOptions::GCPaths & gcPaths) +{ + std::visit( + overloaded{ + [&](const StorePathSet paths) { + conn.to << uint8_t{0}; + WorkerProto::write(store, conn, paths); + }, + [&](const GCOptions::WholeStore & _) { conn.to << uint8_t{1}; }, + }, + gcPaths); +} + } // namespace nix diff --git a/src/nix/nix-collect-garbage/nix-collect-garbage.cc b/src/nix/nix-collect-garbage/nix-collect-garbage.cc index a9f8b7fb6ba1..abab2ce4d2d2 100644 --- a/src/nix/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix/nix-collect-garbage/nix-collect-garbage.cc @@ -100,6 +100,7 @@ static int main_nix_collect_garbage(int argc, char ** argv) auto store = openStore(); auto & gcStore = require(*store); options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + options.pathsToDelete = GCOptions::WholeStore{}; GCResults results; Finally printer([&] { printFreed(dryRun, results); }); gcStore.collectGarbage(options, results); diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index a534e19afb9f..9daf15aa9441 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -663,6 +663,7 @@ static void opGC(Strings opFlags, Strings opArgs) bool printRoots = false; GCOptions options; options.action = GCOptions::gcDeleteDead; + options.pathsToDelete = GCOptions::WholeStore{}; GCResults results; @@ -725,8 +726,10 @@ static void opDelete(Strings opFlags, Strings opArgs) else throw UsageError("unknown flag '%1%'", i); + StorePathSet paths; for (auto & i : opArgs) - options.pathsToDelete.insert(store->followLinksToStorePath(i)); + paths.insert(store->followLinksToStorePath(i)); + options.pathsToDelete = std::move(paths); auto & gcStore = require(*store); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 5d28979a493c..12b03f870a88 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -17,6 +17,13 @@ struct CmdStoreDelete : StorePathsCommand .description = "Do not check whether the paths are reachable from a root.", .handler = {&options.ignoreLiveness, true}, }); + + addFlag({ + .longName = "skip-alive", + .description = + "Do not emit errors when attempting to delete something that is still alive, useful with --recursive.", + .handler = {&options.action, GCOptions::gcDeleteDead}, + }); } std::string description() override @@ -35,8 +42,10 @@ struct CmdStoreDelete : StorePathsCommand { auto & gcStore = require(*store); + StorePathSet paths; for (auto & path : storePaths) - options.pathsToDelete.insert(path); + paths.insert(path); + options.pathsToDelete = std::move(paths); GCResults results; Finally printer([&] { printFreed(false, results); }); diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index f4624a40e8af..02df4ec0602a 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -42,6 +42,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun auto & gcStore = require(*store); options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + options.pathsToDelete = GCOptions::WholeStore{}; GCResults results; Finally printer([&] { printFreed(dryRun, results); }); gcStore.collectGarbage(options, results); diff --git a/tests/functional/gc-closure.sh b/tests/functional/gc-closure.sh new file mode 100755 index 000000000000..03e2a0fc25d6 --- /dev/null +++ b/tests/functional/gc-closure.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +nix_gc_closure() { + clearStore + nix build -f dependencies.nix input0_drv --out-link "$TEST_ROOT/gc-root" + input0=$(realpath "$TEST_ROOT/gc-root") + input1=$(nix build -f dependencies.nix input1_drv --no-link --print-out-paths) + input2=$(nix build -f dependencies.nix input2_drv --no-link --print-out-paths) + top=$(nix build -f dependencies.nix --no-link --print-out-paths) + somthing_else=$(nix store add-path ./dependencies.nix) + + if isDaemonNewer "2.35pre"; then + # Check that nix store delete --recursive --skip-alive is best-effort (doesn't fail when some paths in the closure are alive) + nix store delete --recursive --skip-alive "$top" + [[ ! -e "$top" ]] || fail "top should have been deleted" + [[ -e "$input0" ]] || fail "input0 is a gc root, shouldn't have been deleted" + [[ ! -e "$input2" ]] || fail "input2 is not a gc root and is part of top's closure, it should have been deleted" + [[ -e "$input1" ]] || fail "input1 is not in the closure of top, it shouldn't have been deleted" + [[ -e "$somthing_else" ]] || fail "somthing_else is not in the closure of top, it shouldn't have been deleted" + else + expectStderr 1 nix store delete --recursive --skip-alive "$top" | grepQuiet "Your daemon version is too old to support garbage collecting a specific set of paths" + fi +} + +nix_gc_closure diff --git a/tests/functional/meson.build b/tests/functional/meson.build index bc4d2643e265..e2668c716ed8 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -89,6 +89,7 @@ suites = [ 'hash-convert.sh', 'hash-path.sh', 'gc-non-blocking.sh', + 'gc-closure.sh', 'check.sh', 'nix-shell.sh', 'check-refs.sh', From 26bb1f12c8030454f324d71769f509a6f23bf4d4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 11 Apr 2026 17:13:33 +0300 Subject: [PATCH 229/555] libmain: Don't raise the RLIMIT_NOFILE to RLIM_INFINITY, cap at 1048576 used by the daemon See the comment. Should fix https://github.com/NixOS/nix/issues/15619. See: https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 --- src/libmain/shared.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f3293060f16a..6ddabf12c3bd 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -133,7 +133,14 @@ void bumpFileLimit() return; if (limit.rlim_cur < limit.rlim_max) { - limit.rlim_cur = limit.rlim_max; + // Some software misbehaves really bad when we try to raise the + // limit to RLIM_INFINITY, so cap the limit at the 1048576 limit used + // by the daemon. + // + // GNU patch < 2.8 crashes with **** out of memory, which breaks in nixpkgs darwin bootstrap tools. + // This was fixed in: + // https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 + limit.rlim_cur = std::min(limit.rlim_max, rlim_t(1048576)); // Ignore errors, this is best effort. setrlimit(RLIMIT_NOFILE, &limit); } From 9418535d877b043327c1c16ac45c0be71e6d49cd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 5 Apr 2026 12:19:58 +0200 Subject: [PATCH 230/555] Remove shallow.lock before running git fetch An interrupted `git fetch` can leave behind a `shallow.lock` file which causes subsequent fetches to fail. Since Nix already has a PathLock on the repo, it should be safe to just delete this file. --- src/libfetchers/git-utils.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index fcffdcfa8aef..0bf984652e9d 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -638,6 +638,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this // then use code that was removed in this commit (see blame) auto dir = this->path; + + // Remove shallow.lock left behind by a previously interrupted `git fetch`, as it would prevent `git fetch` + // from running. Note that we already have a repository-wide `PathLock` (see git.cc), so this is safe. + tryUnlink(dir / "shallow.lock"); + OsStrings gitArgs = { OS_STR("-C"), dir.native(), From 729819d21f5d098706fc4195d3eb624b3abcd08f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 11 Apr 2026 12:38:12 -0400 Subject: [PATCH 231/555] tests/functional/nars: test edge-case restore destinations Add expected-failure tests for `nix-store --restore` with destinations that can't work: - Trailing `/.` and `/..` require the named directory to already exist, which conflicts with `--restore` creating something new. Tested for file, symlink, and directory NARs. - Empty string destination. All currently fail with "No such file or directory". These establish a baseline so we notice if the behaviour changes during refactoring. Also replace `(! ...)` with `expectStderr 1` for the case-hack collision dump test, matching the rest of the file's convention. Also add a comment on a symlink test I found interesting. --- tests/functional/nars.sh | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index ecae66c14151..9b8fe67716f6 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -35,6 +35,11 @@ rm -rf "$TEST_ROOT/out" mkdir -p "$TEST_ROOT/out" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" +# Note that the target of the symlink doesn't exist, but the parent does. +# +# A more lenient implementation could allow this, but it would hard/impossible +# to implement without TOCTOU (to the extent avoiding TOCTOU is even +# well-defined with symlinks, though). rm -rf "$TEST_ROOT/out" ln -s "$TEST_ROOT/out2" "$TEST_ROOT/out" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" @@ -47,6 +52,19 @@ rm -rf "$TEST_ROOT/out" (cd "$TEST_ROOT" && nix-store --restore out < tmp.nar) [[ -f "$TEST_ROOT/out" ]] +# Trailing `/.` and `/..` refer to entries within a directory that +# must already exist, which conflicts with --restore creating +# something new. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" + +# Destination with trailing `/..` should fail. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/.." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" + +# Empty destination should fail. +expectStderr 1 nix-store --restore "" < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" + # The same, but for a symlink. ln -sfn foo "$TEST_ROOT/symlink" nix-store --dump "$TEST_ROOT/symlink" > "$TEST_ROOT/tmp.nar" @@ -73,6 +91,10 @@ rm -rf "$TEST_ROOT/out" [[ -L "$TEST_ROOT/out" ]] [[ $(readlink "$TEST_ROOT/out") = foo ]] +# Trailing `/.` — same baseline as above (currently fails). +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" + # Check whether restoring and dumping a NAR that contains case # collisions is round-tripping, even on a case-insensitive system. rm -rf "$TEST_ROOT/case" @@ -96,7 +118,11 @@ rm -rf "$TEST_ROOT/case" # Check whether we detect true collisions (e.g. those remaining after # removal of the suffix). touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" -(! nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > /dev/null) +expectStderr 1 nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > /dev/null + +# Trailing `/.` — same baseline as above (currently fails). +rm -rf "$TEST_ROOT/case" +expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case/." < case.nar | grepQuiet "No such file or directory" # Detect NARs that have a directory entry that after case-hacking # collides with another entry (e.g. a directory containing 'Test', From 95e4446b4eaf5b44dccc320aa76246bfdfb71cdf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Apr 2026 16:53:50 -0400 Subject: [PATCH 232/555] libutil: Use `*at` operations for root file/symlink in `RestoreSink` Previously, when the root entry was a file or symlink (rather than a directory), `RestoreSink` fell back to path-based operations without the race-free symlink safety of `*at` calls. Now, all three methods (`createDirectory`, `createRegularFile`, `createSymlink`) use a shared static `getParentFdAndName` helper that always returns a valid parent fd and single-component name, ensuring consistent use of `openFileEnsureBeneathNoSymlinks`, `mkdirat`, and `symlinkat` across all cases. Multi-component paths also now go through `openFileEnsureBeneathNoSymlinks`, fixing a case where `createDirectory` would follow symlinks in intermediate path components. Update the `openFileEnsureBeneathNoSymlinks` unit test to expect `SymlinkNotAllowed` on Unix too (previously a FIXME that noted `createDirectory` incorrectly succeeded). This is now caught because `getParentFdAndName` routes multi-component paths through `openFileEnsureBeneathNoSymlinks` rather than passing them directly to `mkdirat`. This doesn't help the caller so much because they cannot provide the parent FD, but that will be fixed next. Co-authored-by: Sergei Zimmerman --- src/libutil-tests/file-system-at.cc | 5 -- src/libutil/fs-sink.cc | 111 +++++++++++++++++++--------- tests/functional/nars.sh | 20 ++--- 3 files changed, 87 insertions(+), 49 deletions(-) diff --git a/src/libutil-tests/file-system-at.cc b/src/libutil-tests/file-system-at.cc index 82b73d7af0d9..527e6fc26106 100644 --- a/src/libutil-tests/file-system-at.cc +++ b/src/libutil-tests/file-system-at.cc @@ -102,12 +102,7 @@ TEST(openFileEnsureBeneathNoSymlinks, works) dirSink.createDirectory(CanonPath("d")); dirSink.createSymlink(CanonPath("c"), "./d"); }); -#ifdef _WIN32 EXPECT_THROW(sink.createDirectory(CanonPath("a/b/c/e")), SymlinkNotAllowed); -#else - // FIXME: This still follows symlinks on Unix (incorrectly succeeds) - sink.createDirectory(CanonPath("a/b/c/e")); -#endif // Test that symlinks in intermediate path are detected during nested operations EXPECT_THROW( sink.createDirectory( diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index b0bb20d99fc4..e41d153e92db 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -70,6 +70,61 @@ static std::filesystem::path append(const std::filesystem::path & src, const Can return dst; } +#ifndef _WIN32 +/** + * Return a descriptor and single-component name suitable for + * `*at` operations. The returned `CanonPath` is always a pure + * name (no slashes). The `Descriptor` is the fd to use. The + * `AutoCloseFD` keeps it alive when it was opened temporarily + * (for multi-component paths); otherwise it is empty and the + * `Descriptor` borrows from `dirFd`. When `dirFd` is not set, + * temporarily opens the parent of `dstPath`. + */ +static std::tuple +getParentFdAndName(Descriptor dirFd, const std::filesystem::path & dstPath, const CanonPath & path) +{ + if (dirFd != INVALID_DESCRIPTOR) { + /* dirFd is the root of the restore tree, which means we already created + a root directory, which means that path must be relative (i.e. not + root) within it. */ + assert(!path.isRoot()); + auto parent = path.parent(); + if (parent->isRoot()) + return {AutoCloseFD{}, dirFd, CanonPath::fromFilename(*path.baseName())}; + auto parentFd = openFileEnsureBeneathNoSymlinks(dirFd, *parent, O_RDONLY | O_DIRECTORY | O_CLOEXEC, 0); + if (!parentFd) + throw SysError("opening parent directory of %s", PathFmt(append(dstPath, path))); + auto fd = parentFd.get(); + return {std::move(parentFd), fd, CanonPath::fromFilename(*path.baseName())}; + } + + /* Without dirFd, we're creating the root entry itself, so path + must be root. If it's not, someone forgot to create the root + directory first. */ + auto p = append(dstPath, path); + if (!path.isRoot()) + throw Error("cannot create non-root path %s without a root directory", PathFmt(p)); + if (p.empty()) + throw Error("restore destination path is empty"); + auto filename = p.filename(); + if (filename == "." || filename == "..") + throw Error( + "restore destination '%s' ends in '%s', which is not a valid filename", p.native(), filename.native()); + auto parentPath = p.parent_path(); + /* Relative path with no directory component (e.g. "out") — + the parent is the current working directory. Open it so we + hold a stable reference in case something else in the process + changes the working directory mid-unpack. */ + if (parentPath.empty()) + parentPath = "."; + AutoCloseFD parentFd{::open(parentPath.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC)}; + if (!parentFd) + throw SysError("opening parent directory of %s", PathFmt(p)); + auto fd = parentFd.get(); + return {std::move(parentFd), fd, CanonPath::fromFilename(p.filename().native())}; +} +#endif + void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallback callback) { if (path.isRoot()) { @@ -103,34 +158,27 @@ void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallba void RestoreSink::createDirectory(const CanonPath & path) { - auto p = append(dstPath, path); - #ifndef _WIN32 - if (dirFd) { - if (path.isRoot()) - /* Trying to create a directory that we already have a file descriptor for. */ - throw Error("path %s already exists", PathFmt(p)); + if (dirFd && path.isRoot()) + /* Trying to create a directory that we already have a file descriptor for. */ + throw Error("path %s already exists", PathFmt(append(dstPath, path))); - if (::mkdirat(dirFd.get(), path.rel_c_str(), 0777) == -1) - throw SysError("creating directory %s", PathFmt(p)); + auto [_parentFd, fd, name] = getParentFdAndName(dirFd.get(), dstPath, path); - return; - } -#endif - - if (!std::filesystem::create_directory(p)) - throw Error("path '%s' already exists", p.string()); - -#ifndef _WIN32 - if (path.isRoot()) { - assert(!dirFd); // Handled above + if (::mkdirat(fd, name.rel_c_str(), 0777) == -1) + throw SysError("creating directory %s", PathFmt(append(dstPath, path))); + if (!dirFd) { /* Open directory for further *at operations relative to the sink root directory. */ - dirFd = open(p.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + dirFd = openFileEnsureBeneathNoSymlinks(fd, name, O_RDONLY | O_DIRECTORY | O_CLOEXEC, 0); if (!dirFd) - throw SysError("creating directory %1%", PathFmt(p)); + throw SysError("opening directory %s", PathFmt(append(dstPath, path))); } +#else + auto p = append(dstPath, path); + if (!std::filesystem::create_directory(p)) + throw Error("path '%s' already exists", p.string()); #endif }; @@ -172,13 +220,11 @@ struct RestoreRegularFile : CreateRegularFileSink, FdSink void RestoreSink::createRegularFile(const CanonPath & path, fun func) { - auto p = append(dstPath, path); - auto crf = RestoreRegularFile( startFsync, #ifdef _WIN32 CreateFileW( - p.c_str(), + append(dstPath, path).c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, @@ -190,14 +236,13 @@ void RestoreSink::createRegularFile(const CanonPath & path, fun '%2%'", PathFmt(p), target); - return; - } + auto [_parentFd, fd, name] = getParentFdAndName(dirFd.get(), dstPath, path); + if (::symlinkat(requireCString(target), fd, name.rel_c_str()) == -1) + throw SysError("creating symlink from %1% -> '%2%'", PathFmt(append(dstPath, path)), target); +#else + nix::createSymlink(target, append(dstPath, path).string()); #endif - nix::createSymlink(target, p.string()); } void RegularFileSink::createRegularFile(const CanonPath & path, fun func) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index 9b8fe67716f6..02ee66a14de2 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -11,18 +11,18 @@ rm -rf "$TEST_ROOT/out" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "NAR directory is not sorted" # Check that nix-store --restore fails if the output already exists. -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out' already exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet 'creating directory ".*/out": File exists' rm -rf "$TEST_ROOT/out" echo foo > "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "File exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet 'creating directory ".*/out": File exists' rm -rf "$TEST_ROOT/out" ln -s "$TEST_ROOT/out2" "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "File exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet 'creating directory ".*/out": File exists' mkdir -p "$TEST_ROOT/out2" -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out' already exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet 'creating directory ".*/out": File exists' # The same, but for a regular file. nix-store --dump ./nars.sh > "$TEST_ROOT/tmp.nar" @@ -56,14 +56,14 @@ rm -rf "$TEST_ROOT/out" # must already exist, which conflicts with --restore creating # something new. rm -rf "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "ends in '\.'.*not a valid filename" # Destination with trailing `/..` should fail. rm -rf "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out/.." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/.." < "$TEST_ROOT/tmp.nar" | grepQuiet "ends in '\.\.'.*not a valid filename" # Empty destination should fail. -expectStderr 1 nix-store --restore "" < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" +expectStderr 1 nix-store --restore "" < "$TEST_ROOT/tmp.nar" | grepQuiet "destination path is empty" # The same, but for a symlink. ln -sfn foo "$TEST_ROOT/symlink" @@ -93,7 +93,7 @@ rm -rf "$TEST_ROOT/out" # Trailing `/.` — same baseline as above (currently fails). rm -rf "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "ends in '\.'.*not a valid filename" # Check whether restoring and dumping a NAR that contains case # collisions is round-tripping, even on a case-insensitive system. @@ -122,7 +122,7 @@ expectStderr 1 nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > /dev/null # Trailing `/.` — same baseline as above (currently fails). rm -rf "$TEST_ROOT/case" -expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case/." < case.nar | grepQuiet "No such file or directory" +expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case/." < case.nar | grepQuiet "ends in '\.'.*not a valid filename" # Detect NARs that have a directory entry that after case-hacking # collides with another entry (e.g. a directory containing 'Test', @@ -156,7 +156,7 @@ if (( unicodeTestCode == 1 )); then # If the command failed (MacOS or ZFS + normalization), checks that it failed # with the expected "already exists" error, and that this is the same # behavior as `touch` - echo "$unicodeTestOut" | grepQuiet "creating directory \".*/out/â\": File exists" + echo "$unicodeTestOut" | grepQuiet 'creating directory ".*/out/â": File exists' (( touchFilesCount == 1 )) elif (( unicodeTestCode == 0 )); then From a4064a93f779c939450dbdcb4a145fde4ab34708 Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Sun, 12 Apr 2026 14:52:18 +1000 Subject: [PATCH 233/555] flake: remove x86_64-unknown-freebsd13 seems this was a no op, freebsd13 was changed to freebsd a while ago --- flake.nix | 3 --- 1 file changed, 3 deletions(-) diff --git a/flake.nix b/flake.nix index 97db5eef9365..89cbe93cff7f 100644 --- a/flake.nix +++ b/flake.nix @@ -113,9 +113,6 @@ { config = crossSystem; } - // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") { - useLLVM = true; - } // lib.optionalAttrs (crossSystem == "x86_64-w64-mingw32") { emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable_11}/bin/wine"; }; From 9fae7a90f1699fcc823c6d2853d8cfc023c60852 Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Mon, 13 Apr 2026 12:49:41 +1000 Subject: [PATCH 234/555] libstore: enable sandbox by default on freebsd --- src/libstore/include/nix/store/local-settings.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index 962e831a3ed2..4fe28818d2b8 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -342,7 +342,7 @@ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllo Setting sandboxMode{ this, -#ifdef __linux__ +#if defined(__linux__) || defined(__FreeBSD__) smEnabled #else smDisabled From 72fb36eeff3428d3742bb7630039672f8b6f0991 Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Mon, 13 Apr 2026 12:50:03 +1000 Subject: [PATCH 235/555] packaging: add x86_64-freebsd to installer script --- packaging/hydra.nix | 1 + packaging/installer/install.in | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index fa41b070cc57..8f7e13c1f2ef 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -286,6 +286,7 @@ rec { self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + self.hydraJobs.binaryTarballCross."x86_64-linux"."x86_64-unknown-freebsd" ]; installerScriptForGHA = forAllSystems ( diff --git a/packaging/installer/install.in b/packaging/installer/install.in index b4e808d8e941..af61a433ddf5 100755 --- a/packaging/installer/install.in +++ b/packaging/installer/install.in @@ -65,6 +65,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_aarch64-darwin@ system=aarch64-darwin ;; + FreeBSD.amd64|FreeBSD.x86_64) + hash=@tarballHash_x86_64-freebsd@ + path=@tarballPath_x86_64-freebsd@ + system=x86_64-freebsd + ;; *) oops "sorry, there is no binary distribution of Nix for your platform";; esac From 9edc035b7ee3a112d8531ef7430dcac05114f458 Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Mon, 13 Apr 2026 12:50:34 +1000 Subject: [PATCH 236/555] maintainers/upload-release.pl: add x86_64-freebsd to nix-fallback-paths --- maintainers/upload-release.pl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index f7678b7d1c0a..06678553e712 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -279,6 +279,10 @@ sub downloadFile { downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); }; warn "$@" if $@; + eval { + downloadFile("binaryTarballCross.x86_64-linux.x86_64-unknown-freebsd", "1"); + }; + warn "$@" if $@; downloadFile("installerScript", "1"); # Upload nix-fallback-paths.nix. @@ -290,6 +294,7 @@ sub downloadFile { " riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . " x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" . " aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" . + " x86_64-freebsd = \"" . getStorePath("buildCross.nix-everything.x86_64-unknown-freebsd.x86_64-linux") . "\";\n" . "}\n"); for my $fn (glob "$tmpDir/*") { From 65ff1e6ebf294684d6f6c4ec24b45897b6bd6f60 Mon Sep 17 00:00:00 2001 From: espes Date: Wed, 25 Mar 2026 10:44:34 +0000 Subject: [PATCH 237/555] libutil: emit multi-frame zstd for parallel-decodable output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Libarchive's zstd filter always produces a single frame, so decompression of large NARs is stuck on one core. Replace the libarchive zstd compression path with a direct-libzstd sink that cuts a new frame every 16 MiB of uncompressed input, each with an exact pledged size so Frame_Content_Size lands in the header. Frame concatenation is mandatory in RFC 8878 §3.1, so existing nix binaries, libarchive and the zstd CLI decode the result unchanged. Nix still decompresses serially today, but the independent sized frames let a future parallel decoder exploit data already on disk. Per-frame compression uses up to 4 zstd workers (bounded by getMaxCPU()/hardware_concurrency). parallel-compression now defaults to true for zstd; xz keeps its false default. As a side effect peak RSS during compression drops substantially (~600 -> ~100 MiB for a 1 GiB store path) with effectively unchanged ratio. --- doc/manual/rl-next/zstd-multiframe.md | 18 +++ src/libstore/binary-cache-store.cc | 6 +- .../include/nix/store/binary-cache-store.hh | 6 +- src/libutil-tests/compression.cc | 58 ++++++++ src/libutil-tests/meson.build | 3 + src/libutil-tests/package.nix | 2 + src/libutil/compression.cc | 134 +++++++++++++++++- src/libutil/meson.build | 4 + src/libutil/package.nix | 2 + 9 files changed, 227 insertions(+), 6 deletions(-) create mode 100644 doc/manual/rl-next/zstd-multiframe.md diff --git a/doc/manual/rl-next/zstd-multiframe.md b/doc/manual/rl-next/zstd-multiframe.md new file mode 100644 index 000000000000..6f5d9875df74 --- /dev/null +++ b/doc/manual/rl-next/zstd-multiframe.md @@ -0,0 +1,18 @@ +--- +synopsis: zstd compression now emits multi-frame output and uses less memory +prs: [15550] +--- + +zstd-compressed NARs are now written as a sequence of independent 16 MiB +frames instead of a single large frame. This lays the groundwork for +parallel decompression in a future release without requiring caches to be +repopulated, and significantly lowers peak memory use during compression +(e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). + +The output remains standard zstd and is decoded unchanged by existing Nix +binaries and the `zstd` CLI; compression ratio is effectively unchanged. + +Per-frame compression now uses up to 4 worker threads. For zstd this is the +new default: the `parallel-compression` store setting defaults to `true` when +`compression=zstd` (it remains `false` for `xz`). Set +`?parallel-compression=false` to opt out. diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 624bbd6496df..64fe33536bbd 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -150,8 +150,10 @@ ref BinaryCacheStore::addToStoreCommon( { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed{fileSink, fileHashSink}; - auto compressionSink = makeCompressionSink( - config.compression, teeSinkCompressed, config.parallelCompression, config.compressionLevel); + bool parallel = config.parallelCompression.overridden ? config.parallelCompression.get() + : config.compression.get() == CompressionAlgo::zstd; + auto compressionSink = + makeCompressionSink(config.compression, teeSinkCompressed, parallel, config.compressionLevel); TeeSink teeSinkUncompressed{*compressionSink, narHashSink}; TeeSource teeSource{narSource, teeSinkUncompressed}; narAccessor = makeNarAccessor(parseNarListing(teeSource)); diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index ad38ddaa5b9c..7871ad03c884 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -58,7 +58,11 @@ struct BinaryCacheStoreConfig : virtual StoreConfig this, false, "parallel-compression", - "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + R"( + Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`. + + If not set explicitly, defaults to `true` when `compression` is `zstd` and `false` otherwise. + )"}; Setting compressionLevel{ this, diff --git a/src/libutil-tests/compression.cc b/src/libutil-tests/compression.cc index 53d476fa8593..f1575a67c54a 100644 --- a/src/libutil-tests/compression.cc +++ b/src/libutil-tests/compression.cc @@ -1,5 +1,6 @@ #include "nix/util/compression.hh" #include +#include namespace nix { @@ -61,6 +62,63 @@ TEST(decompress, decompressBrCompressed) ASSERT_EQ(o, str); } +TEST(decompress, decompressZstdCompressed) +{ + auto method = "zstd"; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + auto o = decompress(method, compress(CompressionAlgo::zstd, str)); + + ASSERT_EQ(o, str); +} + +TEST(decompress, decompressZstdCompressedParallel) +{ + auto method = "zstd"; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + auto o = decompress(method, compress(CompressionAlgo::zstd, str, true)); + + ASSERT_EQ(o, str); +} + +TEST(decompress, decompressZstdMultiFrameLargeInput) +{ + // Create input larger than the 16 MiB frame boundary to exercise + // multi-frame emission. + std::string str(20 * 1024 * 1024, 'x'); + for (size_t i = 0; i < str.size(); i += 997) + str[i] = 'y'; // add some variation + auto compressed = compress(CompressionAlgo::zstd, str); + auto o = decompress("zstd", compressed); + + ASSERT_EQ(o, str); +} + +TEST(compress, zstdEmptyInput) +{ + auto compressed = compress(CompressionAlgo::zstd, ""); + // Empty input should still emit a valid (empty-content) zstd + // frame so it round-trips through the decompressor. + ASSERT_FALSE(compressed.empty()); + auto o = decompress("zstd", compressed); + ASSERT_EQ(o, ""); +} + +TEST(compress, zstdExactFrameBoundary) +{ + // Input exactly equal to the 16 MiB frame boundary should not + // emit a trailing zero-content frame. + std::string str(16 * 1024 * 1024, 'z'); + auto compressed = compress(CompressionAlgo::zstd, str); + auto o = decompress("zstd", compressed); + ASSERT_EQ(o, str); + + // Verify there is exactly one frame by checking that + // the first frame's compressed size equals the total size. + size_t frameSize = ZSTD_findFrameCompressedSize(compressed.data(), compressed.size()); + ASSERT_FALSE(ZSTD_isError(frameSize)); + ASSERT_EQ(frameSize, compressed.size()); +} + TEST(decompress, decompressInvalidInputThrowsCompressionError) { auto method = "bzip2"; diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index ff6e539c8a2e..6a86504ded42 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -36,6 +36,9 @@ deps_private += gtest gmock = dependency('gmock') deps_private += gmock +zstd = dependency('libzstd', version : '>= 1.4.0') +deps_private += zstd + if host_machine.system() == 'windows' # Boost.ASIO needs this unconditionally. socket = cxx.find_library('ws2_32') diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index 03c1c2d0efaf..265e58e312f4 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -10,6 +10,7 @@ rapidcheck, gtest, + zstd, runCommand, util-linux, @@ -45,6 +46,7 @@ mkMesonExecutable (finalAttrs: { nix-util-test-support rapidcheck gtest + zstd ] ++ lib.optionals stdenv.hostPlatform.isLinux [ util-linux diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index f07b22dcab2e..24539a46335e 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -2,6 +2,7 @@ #include "nix/util/signals.hh" #include "nix/util/tarfile.hh" #include "nix/util/logging.hh" +#include "nix/util/current-process.hh" #include #include @@ -11,6 +12,9 @@ #include #include +#include +#include + namespace nix { static const int COMPRESSION_LEVEL_DEFAULT = -1; @@ -68,7 +72,11 @@ struct ArchiveDecompressionSource : Source } }; -/* Happens to match enum names. */ +/* Algorithms whose *compression* is handled by libarchive. zstd is + intentionally absent: ZstdMultiFrameCompressionSink compresses it + directly so the output is split into independent frames; zstd + *decompression* is still handled by libarchive via + ArchiveDecompressionSource. */ #define NIX_FOR_EACH_LA_ALGO(MACRO) \ MACRO(bzip2) \ MACRO(compress) \ @@ -79,8 +87,7 @@ struct ArchiveDecompressionSource : Source MACRO(lzip) \ MACRO(lzma) \ MACRO(lzop) \ - MACRO(xz) \ - MACRO(zstd) + MACRO(xz) struct ArchiveCompressionSink : CompressionSink { @@ -99,6 +106,7 @@ struct ArchiveCompressionSink : CompressionSink switch (method) { case CompressionAlgo::none: case CompressionAlgo::brotli: + case CompressionAlgo::zstd: unreachable(); #define NIX_DEF_LA_ALGO_CASE(algo) \ case CompressionAlgo::algo: \ @@ -317,6 +325,124 @@ struct BrotliCompressionSink : ChunkedCompressionSink } }; +/** + * Zstd compression that cuts a new frame every `bytesPerFrame` of + * uncompressed input. The result is a concatenation of independent + * frames, which any conformant zstd decoder (RFC 8878 §3.1) handles + * transparently — including libarchive's, which is what the nix + * substituter path uses for decompression. Because each frame is + * independent and carries its decompressed size, a parallel decoder + * can split work across them. + * + * Frame size is fixed at 16 MiB of input. zstd's window size is + * level-dependent (~2 MiB at the default level 3, up to 8 MiB at + * higher levels), so the ratio loss from not being able to reference + * across a frame boundary is small. 16 MiB gives ~700 frames for the + * biggest NARs, which is ample parallelism and lets a decoder start + * work before the whole blob is downloaded. + */ +struct ZstdMultiFrameCompressionSink : CompressionSink +{ + Sink & nextSink; + std::unique_ptr cctx{nullptr, ZSTD_freeCCtx}; + std::vector outbuf; + /** + * Input buffer for the current frame. We accumulate a full + * frame's worth before compressing so we can set an exact + * `ZSTD_CCtx_setPledgedSrcSize` — that writes `Frame_Content_Size` + * into the frame header, allowing a parallel decoder to compute + * each frame's output offset up front. + */ + std::vector inbuf; + bool emittedAnyFrame = false; + static constexpr uint64_t bytesPerFrame = 16 * 1024 * 1024; + + ZstdMultiFrameCompressionSink(Sink & nextSink, bool parallel, int level) + : nextSink(nextSink) + , outbuf(ZSTD_CStreamOutSize()) + { + inbuf.reserve(bytesPerFrame); + cctx.reset(ZSTD_createCCtx()); + if (!cctx) + throw CompressionError("unable to initialise zstd encoder"); + if (level != COMPRESSION_LEVEL_DEFAULT) + checkZstd(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, level)); + if (parallel) { + unsigned ncpu = getMaxCPU(); + if (ncpu == 0) + ncpu = std::thread::hardware_concurrency(); + /* Cap nbWorkers: zstd's MT engine splits each frame into + per-worker jobs. With 16 MiB frames, more than ~4 + workers yields diminishing returns (< 4 MiB per worker) + and the thread synchronisation overhead can make + compression slower than single-threaded. */ + if (ncpu > 4) + ncpu = 4; + if (ncpu > 1) + /* Don't checkZstd(): if libzstd was built without + ZSTD_MULTITHREAD this returns an error, but per the + zstd docs the parameter is simply ignored and + compression falls back to single-threaded. */ + ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_nbWorkers, ncpu); + } + } + + void checkZstd(size_t ret) + { + if (ZSTD_isError(ret)) + throw CompressionError("zstd error: %s", ZSTD_getErrorName(ret)); + } + + /** + * Compress all of `inbuf` as one complete frame, pledged at its + * exact size so `Frame_Content_Size` lands in the header. + */ + void emitFrame() + { + checkZstd(ZSTD_CCtx_reset(cctx.get(), ZSTD_reset_session_only)); + checkZstd(ZSTD_CCtx_setPledgedSrcSize(cctx.get(), inbuf.size())); + + ZSTD_inBuffer in = {inbuf.data(), inbuf.size(), 0}; + for (;;) { + checkInterrupt(); + ZSTD_outBuffer out = {outbuf.data(), outbuf.size(), 0}; + size_t remaining = ZSTD_compressStream2(cctx.get(), &out, &in, ZSTD_e_end); + checkZstd(remaining); + if (out.pos > 0) + nextSink({outbuf.data(), out.pos}); + if (remaining == 0) + break; + } + inbuf.clear(); + emittedAnyFrame = true; + } + + void writeUnbuffered(std::string_view data) override + { + while (!data.empty()) { + uint64_t room = bytesPerFrame - inbuf.size(); + size_t n = (room < data.size()) ? room : data.size(); + + inbuf.insert(inbuf.end(), data.data(), data.data() + n); + data.remove_prefix(n); + + if (inbuf.size() >= bytesPerFrame) + emitFrame(); + } + } + + void finish() override + { + flush(); + /* Emit the trailing partial frame, or an empty frame if we + never wrote anything — the output must contain at least one + frame header to be valid zstd (otherwise the libarchive + decoder chokes on round-tripped empty input). */ + if (!inbuf.empty() || !emittedAnyFrame) + emitFrame(); + } +}; + ref makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel, int level) { switch (method) { @@ -324,6 +450,8 @@ ref makeCompressionSink(CompressionAlgo method, Sink & nextSink return make_ref(nextSink); case CompressionAlgo::brotli: return make_ref(nextSink); + case CompressionAlgo::zstd: + return make_ref(nextSink, parallel, level); /* Everything else is supported via libarchive. */ #define NIX_DEF_LA_ALGO_CASE(algo) case CompressionAlgo::algo: NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE) diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 7e996b8c3dee..50ade6688726 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -101,6 +101,10 @@ brotli = [ ] deps_private += brotli +# Direct libzstd linkage for ZstdMultiFrameCompressionSink. +zstd = dependency('libzstd', version : '>= 1.4.0') +deps_private += zstd + cpuid_required = get_option('cpuid') if host_machine.cpu_family() != 'x86_64' and cpuid_required.enabled() warning('Force-enabling seccomp on non-x86_64 does not make sense') diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 3deb7ba3ae3c..049c75c5805d 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -11,6 +11,7 @@ libsodium, nlohmann_json, openssl, + zstd, # Configuration Options @@ -52,6 +53,7 @@ mkMesonLibrary (finalAttrs: { libblake3 libsodium openssl + zstd ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; From 0edf06155d96ba8d98aa422607f239ac05d72445 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 13 Apr 2026 20:25:24 +0300 Subject: [PATCH 238/555] Only bump the RLIMIT_STACK limit in EvalState constructor For regular commands like `nix store add` we have no reason to bump the stack size. I've seen pthread_create fail in nix-ninja with ENOMEM. This is a cross-port of https://gerrit.lix.systems/c/lix/+/5053. --- src/libexpr/eval.cc | 12 ++++++++++++ src/libutil/current-process.cc | 2 +- src/libutil/include/nix/util/current-process.hh | 5 +++-- src/nix/main.cc | 8 -------- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 42436f5ea951..1d66b8ead1c9 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -314,6 +315,17 @@ EvalState::EvalState( #endif , staticBaseEnv{std::make_shared(nullptr, nullptr)} { +#ifndef _WIN32 + static std::once_flag stackSizeBumped; + std::call_once(stackSizeBumped, []() { + // Increase the default stack size for the evaluator and for + // libstdc++'s std::regex. + // This used to be 64 MiB, but macOS as deployed on GitHub Actions has a + // hard limit slightly under that, so we round it down a bit. + nix::ensureStackSizeAtLeast(60 * 1024 * 1024); + }); +#endif + corepkgsFS->setPathDisplay(""); internalFS->setPathDisplay("«nix-internal»", ""); diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 177728a63343..acb6e52dd733 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -58,7 +58,7 @@ unsigned int getMaxCPU() #ifndef _WIN32 size_t savedStackSize = 0; -void setStackSize(size_t stackSize) +void ensureStackSizeAtLeast(size_t stackSize) { struct rlimit limit; if (getrlimit(RLIMIT_STACK, &limit) == 0 && static_cast(limit.rlim_cur) < stackSize) { diff --git a/src/libutil/include/nix/util/current-process.hh b/src/libutil/include/nix/util/current-process.hh index 657ff0b44e9c..cdd5a8689c88 100644 --- a/src/libutil/include/nix/util/current-process.hh +++ b/src/libutil/include/nix/util/current-process.hh @@ -27,9 +27,10 @@ unsigned int getMaxCPU(); // It does not seem possible to dynamically change stack size on Windows. #ifndef _WIN32 /** - * Change the stack size. + * Increase the RLIMIT_STACK rlimit if it is currently smaller than `stackSize`. + * @note Not thread safe. Calls to this should be wrapped in a std::call_once. */ -void setStackSize(size_t stackSize); +void ensureStackSizeAtLeast(size_t stackSize); #endif /** diff --git a/src/nix/main.cc b/src/nix/main.cc index 98423b3b1014..541f79622e5c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -597,13 +597,5 @@ int main(int argc, char ** argv) { // The CLI has a more detailed version than the libraries; see nixVersion. nix::nixVersion = NIX_CLI_VERSION; -#ifndef _WIN32 - // Increase the default stack size for the evaluator and for - // libstdc++'s std::regex. - // This used to be 64 MiB, but macOS as deployed on GitHub Actions has a - // hard limit slightly under that, so we round it down a bit. - nix::setStackSize(60 * 1024 * 1024); -#endif - return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); }); } From e4c4f920d32d9b6fd48ef1151f705e9d87d01182 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 11 Mar 2026 20:57:37 -0400 Subject: [PATCH 239/555] Make post-build-hook asynchronous Register the post-build-hook as a child to allow waiting on child events to service the hook (output and PID waiting) *without* blocking the main event loop. Because the servicing is handled as part of the goal, the current behavior is maintained wherein waiting goals will not start until the `post-build-hook` for their dependencies have completed. However, multiple `post-build-hook`s can now run in parallel rather than a single one blocking the entire event loop. Resolves #15406 Signed-off-by: Lisanna Dettwyler --- doc/manual/rl-next/async-post-build-hook.md | 9 + .../build/derivation-building-goal.cc | 179 +++++++++++++----- 2 files changed, 136 insertions(+), 52 deletions(-) create mode 100644 doc/manual/rl-next/async-post-build-hook.md diff --git a/doc/manual/rl-next/async-post-build-hook.md b/doc/manual/rl-next/async-post-build-hook.md new file mode 100644 index 000000000000..ea061f14ae9d --- /dev/null +++ b/doc/manual/rl-next/async-post-build-hook.md @@ -0,0 +1,9 @@ +--- +synopsis: Make post-build-hook asynchronous +prs: [15451] +issues: [15406] +--- + +This change makes the `post-build-hook` run asynchronously but still as part of the goal. +This retains the current behavior that a waiting goal will not start until the `post-build-hook` of the goal it is waiting on completes. +However, multiple `post-build-hook`s can now run concurrently just as multiple goals can run concurrently. diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 2235cf872b63..4b012c8a0bec 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -16,6 +16,7 @@ #include "nix/store/local-store.hh" // TODO remove, along with remaining downcasts #include "nix/store/outputs-query.hh" #include "nix/store/globals.hh" +#include "nix/util/current-process.hh" #include #include @@ -66,7 +67,72 @@ std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & dr return msg; } -static void runPostBuildHook( +struct LogSink : Sink +{ + Activity & act; + std::string currentLine; + + LogSink(Activity & act) + : act(act) + { + } + + void operator()(std::string_view data) override + { + for (auto c : data) { + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() + { + act.result(resPostBuildLogLine, currentLine); + currentLine.clear(); + } + + ~LogSink() + { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } +}; + +struct PostBuildHookState +{ + const std::string hook; + Activity act; + std::unique_ptr sink; + std::unique_ptr out; + Pid pid; + + PostBuildHookState(Logger & logger, const std::string hook, const std::string drvPath) + : hook(hook) + , act(logger, + lvlTalkative, + actPostBuildHook, + fmt("running post-build-hook '%s'", hook), + Logger::Fields{drvPath}) + , out(std::make_unique()) + { + out->create(); + sink = std::make_unique(act); + } + + void complete() + { + if (int ret = pid.wait()) { + throw Error("program \"%s\" %s", hook, statusToString(ret)); + } + } +}; + +static std::unique_ptr runPostBuildHook( const WorkerSettings & workerSettings, const StoreDirConfig & store, Logger & logger, @@ -762,7 +828,21 @@ Goal::Co DerivationBuildingGoal::buildWithHook( StorePathSet outputPaths; for (auto & [_, output] : builtOutputs) outputPaths.insert(output.outPath); - runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + + if (worker.settings.postBuildHook.get() != "") { + auto hookState = runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + worker.childStarted(shared_from_this(), {hookState->out->readSide.get()}, false, false); + while (true) { + auto event = co_await WaitForChildEvent{}; + if (auto * output = std::get_if(&event)) { + (*hookState->sink)(output->data); + } else if (std::get_if(&event)) { + hookState->complete(); + worker.childTerminated(this); + break; + } + } + } /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will @@ -992,7 +1072,21 @@ Goal::Co DerivationBuildingGoal::buildLocally( worker.markContentsGood(output.outPath); outputPaths.insert(output.outPath); } - runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + + if (worker.settings.postBuildHook.get() != "") { + auto hookState = runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + worker.childStarted(shared_from_this(), {hookState->out->readSide.get()}, false, false); + while (true) { + auto event = co_await WaitForChildEvent{}; + if (auto * output = std::get_if(&event)) { + (*hookState->sink)(output->data); + } else if (std::get_if(&event)) { + hookState->complete(); + worker.childTerminated(this); + break; + } + } + } /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will @@ -1005,24 +1099,21 @@ Goal::Co DerivationBuildingGoal::buildLocally( #endif } -static void runPostBuildHook( +static std::unique_ptr runPostBuildHook( const WorkerSettings & workerSettings, const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths) { - auto hook = workerSettings.postBuildHook; - if (hook == "") - return; +#ifdef _WIN32 + throw UnimplementedError("post-build-hook is not implemented on Windows"); +#else + auto state = + std::make_unique(logger, workerSettings.postBuildHook.get(), store.printStorePath(drvPath)); + + auto hook = workerSettings.postBuildHook.get(); - Activity act( - logger, - lvlTalkative, - actPostBuildHook, - fmt("running post-build-hook '%s'", workerSettings.postBuildHook), - Logger::Fields{store.printStorePath(drvPath)}); - PushActivity pact(act.id); OsStringMap hookEnvironment = getEnvOs(); hookEnvironment.emplace(OS_STR("DRV_PATH"), string_to_os_string(store.printStorePath(drvPath))); @@ -1030,50 +1121,34 @@ static void runPostBuildHook( OS_STR("OUT_PATHS"), string_to_os_string(chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))))); hookEnvironment.emplace(OS_STR("NIX_CONFIG"), string_to_os_string(globalConfig.toKeyValue())); - struct LogSink : Sink - { - Activity & act; - std::string currentLine; + ProcessOptions processOptions; + processOptions.allowVfork = false; - LogSink(Activity & act) - : act(act) - { - } + state->pid = startProcess( + [&] { + replaceEnv(hookEnvironment); + if (dup2(state->out->writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); - void operator()(std::string_view data) override - { - for (auto c : data) { - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } + Strings args_; + args_.push_front(hook); - void flushLine() - { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } + unix::closeExtraFDs(); - ~LogSink() - { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; + restoreProcessContext(); - LogSink sink(act); + execvp(hook.c_str(), stringsToCharPtrs(args_).data()); - runProgram2({ - .program = workerSettings.postBuildHook.get(), - .environment = hookEnvironment, - .standardOut = &sink, - .mergeStderrToStdout = true, - }); + throw SysError("executing %s", PathFmt(hook)); + }, + processOptions); + + state->out->writeSide.close(); + + return state; +#endif } BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailureError e, BuildLog & buildLog) From 892d870d9640fe5a51b0e5e38107b27ab276c2f7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 13 Apr 2026 22:11:52 +0300 Subject: [PATCH 240/555] libstore: Plug thread safety issues in recursive-nix --- .../include/nix/store/restricted-store.hh | 52 +++++++++++++++---- src/libstore/restricted-store.cc | 15 ++++-- .../unix/build/chroot-derivation-builder.cc | 2 - src/libstore/unix/build/derivation-builder.cc | 19 ++++--- 4 files changed, 65 insertions(+), 23 deletions(-) diff --git a/src/libstore/include/nix/store/restricted-store.hh b/src/libstore/include/nix/store/restricted-store.hh index ca4e0b8536a7..79167a3d08c8 100644 --- a/src/libstore/include/nix/store/restricted-store.hh +++ b/src/libstore/include/nix/store/restricted-store.hh @@ -2,6 +2,9 @@ ///@file #include "nix/store/store-api.hh" +#include "nix/util/sync.hh" + +#include namespace nix { @@ -28,15 +31,20 @@ struct RestrictionContext */ virtual const StorePathSet & originalPaths() = 0; - /** - * Paths that were added via recursive Nix calls. - */ - StorePathSet addedPaths; + struct State + { + /** + * Paths that were added via recursive Nix calls. + */ + std::map> addedPaths; - /** - * Realisations that were added via recursive Nix calls. - */ - std::set addedDrvOutputs; + /** + * Realisations that were added via recursive Nix calls. + */ + std::set addedDrvOutputs; + }; + + Sync state_; /** * Recursive Nix calls are only allowed to build or realize paths @@ -56,7 +64,33 @@ struct RestrictionContext { if (isAllowed(path)) return; - addDependencyImpl(path); + + std::promise promise; + + auto [future, shouldAdd] = [&]() -> std::pair, bool> { + auto state(state_.lock()); + if (auto iter = state->addedPaths.find(path); iter != state->addedPaths.end()) { + return {iter->second, false}; + } + auto [iter2, _] = state->addedPaths.emplace(path, promise.get_future().share()); + return {iter2->second, true}; + }(); + + /* Another daemon worker thread already started adding the dependency. Just wait for it + to complete. */ + if (!shouldAdd) { + future.get(); + return; + } + + try { + addDependencyImpl(path); + promise.set_value(); + } catch (...) { + /* Notify all other waiters that we are done. */ + promise.set_exception(std::current_exception()); + throw; + } } virtual ~RestrictionContext() = default; diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 8001d43ec8d8..03c130da9653 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -167,8 +167,11 @@ StorePathSet RestrictedStore::queryAllValidPaths() StorePathSet paths; for (auto & p : goal.originalPaths()) paths.insert(p); - for (auto & p : goal.addedPaths) - paths.insert(p); + for (auto & [p, future] : goal.state_.lock()->addedPaths) { + /* Only report the paths that have finished materialising in the sandbox. */ + if (future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) + paths.insert(p); + } return paths; } @@ -300,8 +303,12 @@ std::vector RestrictedStore::buildPathsWithResults( next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); - for (auto & real : newRealisations) - goal.addedDrvOutputs.insert(real.id); + + { + auto state(goal.state_.lock()); + for (auto & real : newRealisations) + state->addedDrvOutputs.insert(real.id); + } return results; } diff --git a/src/libstore/unix/build/chroot-derivation-builder.cc b/src/libstore/unix/build/chroot-derivation-builder.cc index d10d98245a11..157d9e319173 100644 --- a/src/libstore/unix/build/chroot-derivation-builder.cc +++ b/src/libstore/unix/build/chroot-derivation-builder.cc @@ -131,8 +131,6 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl std::pair addDependencyPrep(const StorePath & path) { - DerivationBuilderImpl::addDependencyImpl(path); - debug("materialising '%s' in the sandbox", store.printStorePath(path)); std::filesystem::path source = store.toRealPath(path); diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 85aa98c7ae87..3cf7977c50d2 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -238,12 +238,18 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder bool isAllowed(const StorePath & path) override { - return inputPaths.count(path) || addedPaths.count(path); + if (inputPaths.count(path)) + return true; + auto state(state_.lock()); + auto iter = state->addedPaths.find(path); + if (iter == state->addedPaths.end()) + return false; + return iter->second.wait_for(std::chrono::seconds(0)) == std::future_status::ready; } bool isAllowed(const DrvOutput & id) override { - return addedDrvOutputs.count(id); + return state_.lock()->addedDrvOutputs.count(id); } bool isAllowed(const DerivedPath & req); @@ -1165,7 +1171,7 @@ void DerivationBuilderImpl::startDaemon() ref(std::dynamic_pointer_cast(this->store.shared_from_this())), *this); - addedPaths.clear(); + state_.lock()->addedPaths.clear(); auto socketName = ".nix-socket"; std::filesystem::path socketPath = tmpDir / socketName; @@ -1246,10 +1252,7 @@ void DerivationBuilderImpl::stopDaemon() daemonSocket.close(); } -void DerivationBuilderImpl::addDependencyImpl(const StorePath & path) -{ - addedPaths.insert(path); -} +void DerivationBuilderImpl::addDependencyImpl(const StorePath & path) {} void DerivationBuilderImpl::chownToBuilder(const std::filesystem::path & path) { @@ -1434,7 +1437,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() referenceablePaths.insert(p); for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) + for (auto & [p, _] : state_.lock()->addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and make all From 83a0d7e3923648e38918e81582ea9888d4a63597 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 22:39:41 +0000 Subject: [PATCH 241/555] build(deps): bump cachix/install-nix-action from 31.10.3 to 31.10.4 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.10.3 to 31.10.4. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/96951a368ba55167b55f1c916f7d416bac6505fe...616559265b40713947b9c190a8ff4b507b5df49b) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.10.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51cd9b48c72c..be8cb0f0c4fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,7 +192,7 @@ jobs: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@96951a368ba55167b55f1c916f7d416bac6505fe # v31.10.3 + - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 if: ${{ !matrix.experimental-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} @@ -258,7 +258,7 @@ jobs: id: installer-tarball-url run: | echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@96951a368ba55167b55f1c916f7d416bac6505fe # v31.10.3 + - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} From 493acdfdd143e15863c948326f8df154c9618222 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Tue, 7 Apr 2026 16:03:05 +0200 Subject: [PATCH 242/555] Use limit from macOS kernel instead of hardcoded --- src/libmain/shared.cc | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 6ddabf12c3bd..c57f71cc2689 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -21,6 +21,9 @@ #ifndef _WIN32 # include #endif +#ifdef __APPLE__ +# include +#endif #ifdef __linux__ # include #endif @@ -132,15 +135,26 @@ void bumpFileLimit() if (getrlimit(RLIMIT_NOFILE, &limit) != 0) return; - if (limit.rlim_cur < limit.rlim_max) { - // Some software misbehaves really bad when we try to raise the - // limit to RLIM_INFINITY, so cap the limit at the 1048576 limit used - // by the daemon. - // - // GNU patch < 2.8 crashes with **** out of memory, which breaks in nixpkgs darwin bootstrap tools. - // This was fixed in: - // https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 - limit.rlim_cur = std::min(limit.rlim_max, rlim_t(1048576)); + rlim_t target = limit.rlim_max; + +# ifdef __APPLE__ + // On macOS the hard limit is typically RLIM_INFINITY, but + // setting rlim_cur to that causes problems: child processes + // (e.g. GNU patch in the Nix sandbox) may allocate memory + // proportional to the fd limit and OOM. Use the kernel's + // per-process file limit instead, which is the effective cap. + // + // GNU patch < 2.8 crashes with **** out of memory, which breaks in nixpkgs darwin bootstrap tools. + // This was fixed in: + // https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 + int maxfiles; + size_t len = sizeof(maxfiles); + if (sysctlbyname("kern.maxfilesperproc", &maxfiles, &len, nullptr, 0) == 0) + target = maxfiles; +# endif + + if (limit.rlim_cur < target) { + limit.rlim_cur = target; // Ignore errors, this is best effort. setrlimit(RLIMIT_NOFILE, &limit); } From de276c66d0b5a2dd7bc25822e3fbb7fd78d42dc1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Apr 2026 14:09:43 +0200 Subject: [PATCH 243/555] NarInfo::to_string(): Loosen fileInfo/fileSize requirements These fields are in fact optional, and various binary cache implementations do not return them (e.g. when NARs are compressed on the fly). --- src/libstore/nar-info.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d470569ea294..028302963e5c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -110,9 +110,12 @@ std::string NarInfo::to_string(const StoreDirConfig & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash && fileHash->algo == HashAlgorithm::SHA256); - res += "FileHash: " + fileHash->to_string(HashFormat::Nix32, true) + "\n"; - res += "FileSize: " + std::to_string(fileSize) + "\n"; + if (fileHash) { + assert(fileHash->algo == HashAlgorithm::SHA256); + res += "FileHash: " + fileHash->to_string(HashFormat::Nix32, true) + "\n"; + } + if (fileSize) + res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.algo == HashAlgorithm::SHA256); res += "NarHash: " + narHash.to_string(HashFormat::Nix32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; From 779b983a4dadb3674b6e817acf3bf55c73cebb3c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Apr 2026 17:41:03 +0200 Subject: [PATCH 244/555] LocalStore::addToStore(): Show hash mismatches in SRI format This is more consistent with e.g. hash mismatches in fixed-output derivations and the output of `nix path-info`. --- src/libstore/local-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d5e45773120c..d52daaae1fd1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1064,8 +1064,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF throw Error( "hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), - info.narHash.to_string(HashFormat::Nix32, true), - hashResult.hash.to_string(HashFormat::Nix32, true)); + info.narHash.to_string(HashFormat::SRI, true), + hashResult.hash.to_string(HashFormat::SRI, true)); if (hashResult.numBytesDigested != info.narSize) throw Error( From 2a71fbf41017b4b9ad87f7c6fa5506b80743e1a7 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 13 Nov 2025 12:45:01 +0700 Subject: [PATCH 245/555] docs: add time complexity to relevant primops --- src/libexpr/primops.cc | 192 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 191 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 98ed1b4509be..5d28cd0dca50 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2999,6 +2999,12 @@ static RegisterPrimOp primop_attrNames({ Return the names of the attributes in the set *set* in an alphabetically sorted list. For instance, `builtins.attrNames { y = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`. + + # Time Complexity + + - O(n log n), where: + + n = number of attributes in the set )", .fun = prim_attrNames, }); @@ -3031,6 +3037,12 @@ static RegisterPrimOp primop_attrValues({ .doc = R"( Return the values of the attributes in the set *set* in the order corresponding to the sorted attribute names. + + # Time Complexity + + - O(n log n), where: + + n = number of attributes in the set )", .fun = prim_attrValues, }); @@ -3056,6 +3068,10 @@ static RegisterPrimOp primop_getAttr({ aborts if the attribute doesn’t exist. This is a dynamic version of the `.` operator, since *s* is an expression rather than an identifier. + + # Time Complexity + + O(log n) where n = number of attributes in the set )", .fun = prim_getAttr, }); @@ -3144,6 +3160,10 @@ static RegisterPrimOp primop_hasAttr({ `hasAttr` returns `true` if *set* has an attribute named *s*, and `false` otherwise. This is a dynamic version of the `?` operator, since *s* is an expression rather than an identifier. + + # Time Complexity + + O(log n) where n = number of attributes in the set )", .fun = prim_hasAttr, }); @@ -3203,6 +3223,13 @@ static RegisterPrimOp primop_removeAttrs({ ``` evaluates to `{ y = 2; }`. + + # Time Complexity + + O(n + k log k) where: + + n = number of attributes in input set + k = number of attribute names to remove )", .fun = prim_removeAttrs, }); @@ -3290,6 +3317,10 @@ static RegisterPrimOp primop_listToAttrs({ ```nix { foo = 123; bar = 456; } ``` + + # Time Complexity + + O(n log n) where n = number of list elements )", .fun = prim_listToAttrs, }); @@ -3366,7 +3397,12 @@ static RegisterPrimOp primop_intersectAttrs({ Return a set consisting of the attributes in the set *e2* which have the same name as some attribute in *e1*. - Performs in O(*n* log *m*) where *n* is the size of the smaller set and *m* the larger set's size. + # Time Complexity + + O(n * log m) where: + + n = number of attributes in the smaller set + m = number of attributes in the larger set )", .fun = prim_intersectAttrs, }); @@ -3406,6 +3442,13 @@ static RegisterPrimOp primop_catAttrs({ ``` evaluates to `[1 2]`. + + # Time Complexity + + O(n * log m) where: + + n = number of sets in input list + m = number of attributes per set )", .fun = prim_catAttrs, }); @@ -3449,6 +3492,10 @@ static RegisterPrimOp primop_functionArgs({ "Formal argument" here refers to the attributes pattern-matched by the function. Plain lambdas are not included, e.g. `functionArgs (x: ...) = { }`. + + # Time Complexity + + O(n) where n = number of formal arguments )", .fun = prim_functionArgs, }); @@ -3481,6 +3528,14 @@ static RegisterPrimOp primop_mapAttrs({ ``` evaluates to `{ a = 10; b = 20; }`. + + # Time Complexity + + O(n) where: + + n = number of attributes + + Calls to `f` are performed afterwards, when needed )", .fun = prim_mapAttrs, }); @@ -3568,6 +3623,15 @@ static RegisterPrimOp primop_zipAttrsWith({ b = { name = "b"; values = [ "z" ]; }; } ``` + + # Time Complexity + + O(n * k * log k) worst case, where: + + n = number of attribute sets in input list + k = number of unique keys across all sets + + More precisely: O(n * m * log k) where m ≤ k is average number of attributes per set )", .fun = prim_zipAttrsWith, }); @@ -3633,6 +3697,10 @@ static RegisterPrimOp primop_head({ Return the first element of a list; abort evaluation if the argument isn’t a list or is an empty list. You can test whether a list is empty by comparing it with `[]`. + + # Time Complexity + + O(1) )", .fun = prim_head, }); @@ -3664,6 +3732,10 @@ static RegisterPrimOp primop_tail({ > This function should generally be avoided since it's inefficient: > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. + + # Time Complexity + + O(n) where n = list length (copies n-1 elements) )", .fun = prim_tail, }); @@ -3698,6 +3770,14 @@ static RegisterPrimOp primop_map({ ``` evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + # Time Complexity + + O(n) where: + + n = list length + + Calls to `f` are performed afterwards when needed. )", .fun = prim_map, }); @@ -3747,6 +3827,13 @@ static RegisterPrimOp primop_filter({ .doc = R"( Return a list consisting of the elements of *list* for which the function *f* returns `true`. + + # Time Complexity + + O(n * T_f) where: + + n = list length + T_f = predicate evaluation time )", .fun = prim_filter, }); @@ -3770,6 +3857,15 @@ static RegisterPrimOp primop_elem({ .doc = R"( Return `true` if a value equal to *x* occurs in the list *xs*, and `false` otherwise. + + # Time Complexity + + O(n * T) (worst case) where: + + n = list length + T = time to compare two elements + + returns early if the elements is found )", .fun = prim_elem, }); @@ -3792,6 +3888,12 @@ static RegisterPrimOp primop_concatLists({ .args = {"lists"}, .doc = R"( Concatenate a list of lists into a single list. + + # Time Complexity + + O(N) where: + + N = total number of elements across all lists )", .fun = prim_concatLists, }); @@ -3808,6 +3910,10 @@ static RegisterPrimOp primop_length({ .args = {"e"}, .doc = R"( Return the length of the list *e*. + + # Time Complexity + + O(1) )", .fun = prim_length, }); @@ -3851,6 +3957,13 @@ static RegisterPrimOp primop_foldlStrict({ argument is the current element being processed. The return value of each application of `op` is evaluated immediately, even for intermediate values. + + # Time Complexity + + O(n * T_op) where: + + n = list length + T_op = `op` call evaluation time )", .fun = prim_foldlStrict, }); @@ -3889,6 +4002,15 @@ static RegisterPrimOp primop_any({ .doc = R"( Return `true` if the function *pred* returns `true` for at least one element of *list*, and `false` otherwise. + + # Time Complexity + + O(n * T_pred) where: + + - n = `list` length + - T_pred = `pred` call evaluation time + + returns early when `pred` returns `true` )", .fun = prim_any, }); @@ -3904,6 +4026,15 @@ static RegisterPrimOp primop_all({ .doc = R"( Return `true` if the function *pred* returns `true` for all elements of *list*, and `false` otherwise. + + # Time Complexity + + O(n * T_f) where: + + - n = list length + - T_f = predicate evaluation time + + returns early when `pred` returns `false` )", .fun = prim_all, }); @@ -3942,6 +4073,17 @@ static RegisterPrimOp primop_genList({ ``` returns the list `[ 0 1 4 9 16 ]`. + + # Time Complexity + + Complexity of `genList generator n`: O(n) + + Complexity of `deepSeq (genList generator n)`: O(n * T_f) + + where: + + n = requested length + T_f = `generator` call evaluation time )", .fun = prim_genList, }); @@ -4036,6 +4178,16 @@ static RegisterPrimOp primop_sort({ If the *comparator* violates any of these properties, then `builtins.sort` reorders elements in an unspecified manner. + + # Time Complexity + + O(n log n * T_cmp), where: + + n = `list` length + T_cmp = `comparator` call evaluation time + + Uses an adaptive sort that exploits existing sorted runs in the input, + down to O(n * T_cmp) when the list is already sorted. )", .fun = prim_sort, }); @@ -4097,6 +4249,13 @@ static RegisterPrimOp primop_partition({ ```nix { right = [ 23 42 ]; wrong = [ 1 9 3 ]; } ``` + + # Time Complexity + + O(n * T_pred) where: + + n = list length + T_pred = `pred` call evaluation time )", .fun = prim_partition, }); @@ -4150,6 +4309,14 @@ static RegisterPrimOp primop_groupBy({ ```nix { b = [ "bar" "baz" ]; f = [ "foo" ]; } ``` + + # Time Complexity + + O(N * T_f + N * log k) where: + + N = number of `list` elements + T_f = `f` call evaluation time + k = number of unique groups )", .fun = prim_groupBy, }); @@ -4192,6 +4359,14 @@ static RegisterPrimOp primop_concatMap({ .doc = R"( This function is equivalent to `builtins.concatLists (map f list)` but is more efficient. + + # Time Complexity + + O(k * T_f + N) where: + + k = length of input list + T_f = time to call `f` on an element + N = total number of elements returned by `f` calls )", .fun = prim_concatMap, }); @@ -4888,6 +5063,13 @@ static RegisterPrimOp primop_concatStringsSep({ Concatenate a list of strings with a separator between each element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] == "usr/local/bin"`. + + # Time Complexity + + O(n + m) where: + + n = number of list elements + m = total length of output string )", .fun = prim_concatStringsSep, }); @@ -4972,6 +5154,14 @@ static RegisterPrimOp primop_replaceStrings({ ``` evaluates to `"fabir"`. + + # Time Complexity + + O(n * k * c) where: + + n = length of input string + k = number of replacement patterns + c = average length of patterns in 'from' list )", .fun = prim_replaceStrings, }); From 3872910b2a545816688ea027e42cc632597617a1 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 14 Apr 2026 19:38:27 +0200 Subject: [PATCH 246/555] docs: big-o fix minor inconsistencies --- src/libexpr/primops.cc | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5d28cd0dca50..d5a0c2428071 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3447,7 +3447,7 @@ static RegisterPrimOp primop_catAttrs({ O(n * log m) where: - n = number of sets in input list + n = list length m = number of attributes per set )", .fun = prim_catAttrs, @@ -3535,7 +3535,7 @@ static RegisterPrimOp primop_mapAttrs({ n = number of attributes - Calls to `f` are performed afterwards, when needed + Calls to `f` are performed afterwards, when needed. )", .fun = prim_mapAttrs, }); @@ -3626,12 +3626,10 @@ static RegisterPrimOp primop_zipAttrsWith({ # Time Complexity - O(n * k * log k) worst case, where: + O(N * log k) where: - n = number of attribute sets in input list + N = total attributes across all sets k = number of unique keys across all sets - - More precisely: O(n * m * log k) where m ≤ k is average number of attributes per set )", .fun = prim_zipAttrsWith, }); @@ -3830,7 +3828,7 @@ static RegisterPrimOp primop_filter({ # Time Complexity - O(n * T_f) where: + O(n * T_f) (eager; predicate is forced) where: n = list length T_f = predicate evaluation time @@ -5066,7 +5064,7 @@ static RegisterPrimOp primop_concatStringsSep({ # Time Complexity - O(n + m) where: + O(n + m) (amortized) where: n = number of list elements m = total length of output string @@ -5157,7 +5155,7 @@ static RegisterPrimOp primop_replaceStrings({ # Time Complexity - O(n * k * c) where: + O(n * k * c) (worst case) where: n = length of input string k = number of replacement patterns From 58aeb0683d05a00b3152b617b012cba1ee5e5e0a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Apr 2026 23:55:40 +0300 Subject: [PATCH 247/555] edit: Allow opening paths that don't exist on disk in EDITOR Since https://github.com/NixOS/nix/pull/14050 we now mount over the store paths in the rootFS accessor in the evaluator. This broke :e and nix edit on something like `:e "${outPath}/src/nix-manual/utils.nix"` when having the `github:nixos/nix` flake loaded in a `--pure-eval` repl. `nix edit` was also broken similarly with: `because it has no physical path`. For regular files, we can copy the outputs into a temporary read-only file and open that instead. For directories, we could open a directory listing in the editor, but I haven't implemented that here. Copying the whole directory tree seems a bit wasteful to me, so we can't truthfully mimic the old behavior of `:e` on directories (that editors usually allow the user to traverse when opened). This also implements some infrastructure for getting a writable file descriptor for the file that would get edited - I intend to use it for https://github.com/NixOS/nix/issues/15633. --- src/libcmd/editor-for.cc | 54 +++++++++++++++++---- src/libcmd/include/nix/cmd/editor-for.hh | 13 ++++- src/libcmd/repl.cc | 15 +++--- src/libutil/include/nix/util/file-system.hh | 4 ++ src/libutil/include/nix/util/os-string.hh | 10 ++++ src/libutil/unix/file-system.cc | 2 +- src/libutil/windows/file-system.cc | 2 +- src/libutil/windows/os-string.cc | 14 ++++++ src/nix/edit.cc | 21 ++++---- tests/functional/flakes/edit.sh | 4 ++ tests/functional/repl.sh | 3 ++ 11 files changed, 112 insertions(+), 30 deletions(-) diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index 95fdf95ad00c..83aba78d8f4b 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,22 +1,56 @@ #include "nix/cmd/editor-for.hh" #include "nix/util/environment-variables.hh" #include "nix/util/source-path.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" namespace nix { -Strings editorFor(const SourcePath & file, uint32_t line) +std::tuple editorFor(const SourcePath & file, uint32_t line, bool readOnly) { auto path = file.getPhysicalPath(); - if (!path) - throw Error("cannot open '%s' in an editor because it has no physical path", file); - auto editor = getEnv("EDITOR").value_or("cat"); - auto args = tokenizeString(editor); + OsString editor = getEnvOsNonEmpty(OS_STR("EDITOR")).value_or(OS_STR("cat")); + auto args = tokenizeString(editor); if (line > 0 - && (editor.find("emacs") != std::string::npos || editor.find("nano") != std::string::npos - || editor.find("vim") != std::string::npos || editor.find("kak") != std::string::npos)) - args.push_back(fmt("+%d", line)); - args.push_back(path->string()); - return args; + && (editor.contains(OS_STR("emacs")) || editor.contains(OS_STR("nano")) || editor.contains(OS_STR("vim")) + || editor.contains(OS_STR("kak")))) + args.push_back(string_to_os_string(fmt("+%d", line))); + + if (path) { + args.push_back(path->native()); + return {std::move(args), AutoCloseFD{}, AutoDelete{}}; + } + + /* Resolve symlinks when creating a temporary. That's how a regular editor + would behave. Also GitSourceAccessor behaves poorly with symlinks in the + path and fails with "«...» does not exist". */ + auto file2 = file.resolveSymlinks(); + auto stat = file2.lstat(); + /* TODO: Maybe we should print a directory listing and open that instead? */ + if (stat.type != SourceAccessor::tRegular) + throw Error("can't open a file %s of type '%s'", file2.to_string(), stat.typeString()); + + auto tempDir = createTempDir(defaultTempDir(), "nix-edit", 0700); + AutoDelete autoDel(tempDir, /*recursive=*/true); + auto tempPath = tempDir / file2.path.baseName().value_or("nix-edit"); + + /* Create the file with the same name, so editors that recognise file + extensions spin up syntax highlighting and LSPs. */ + auto tempFd = openNewFileForWrite( + tempPath, + readOnly ? 0400 : 0600, + {.truncateExisting = false, .followSymlinksOnTruncate = false, .writeOnly = false}); + + if (!tempFd) + throw NativeSysError("failed to create temporary file %s", PathFmt(tempPath)); + + /* Copy the contents into the created copy. */ + FdSink fileSink(tempFd.get()); + file2.readFile(fileSink); + fileSink.flush(); + args.push_back(tempPath); + + return {std::move(args), std::move(tempFd), std::move(autoDel)}; } } // namespace nix diff --git a/src/libcmd/include/nix/cmd/editor-for.hh b/src/libcmd/include/nix/cmd/editor-for.hh index 3fb8a072e732..998b60f4ee25 100644 --- a/src/libcmd/include/nix/cmd/editor-for.hh +++ b/src/libcmd/include/nix/cmd/editor-for.hh @@ -1,15 +1,24 @@ #pragma once ///@file -#include "nix/util/types.hh" #include "nix/util/source-path.hh" +#include "nix/util/os-string.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" namespace nix { /** * Helper function to generate args that invoke $EDITOR on * filename:lineno. + * + * When file doesn't have a physical path, the contents get copied into a + * temporary file, and a file descriptor and RAII cleanup guard for it are + * returned. + * + * @param readOnly make the temporary file readonly if the file has no physical + * path. Ignored otherwise. */ -Strings editorFor(const SourcePath & file, uint32_t line); +std::tuple editorFor(const SourcePath & file, uint32_t line, bool readOnly = true); } // namespace nix diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 9968cd2983d7..2eabde9d6b6a 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -484,8 +484,8 @@ ProcessLineResult NixRepl::processLine(std::string line) } }(); - // Open in EDITOR - auto args = editorFor(path, line); + /* Open file in EDITOR, or edit a read-only copy if the file doesn't have a physical path. */ + auto [args, fd, autoDel] = editorFor(path, line, /*readOnly=*/true); auto editor = args.front(); args.pop_front(); @@ -494,13 +494,16 @@ ProcessLineResult NixRepl::processLine(std::string line) runProgram2({ .program = editor, .lookupPath = true, - .args = toOsStrings(std::move(args)), + .args = std::move(args), .isInteractive = true, }); - // Reload right after exiting the editor - state->resetFileCache(); - reloadFilesAndFlakes(); + /* If we had to open a temporary read-only file, there's no need to + reload (no files could have changed anyway). */ + if (!fd) { + state->resetFileCache(); + reloadFilesAndFlakes(); + } } else if (command == ":t") { diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index ff225eae0f11..30301d97daee 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -232,6 +232,10 @@ struct OpenNewFileForWriteParams * Whether to follow symlinks if @ref truncateExisting is true. */ bool followSymlinksOnTruncate:1 = false; + /** + * Whether to open the newly created file as write-only. + */ + bool writeOnly:1 = true; }; /** diff --git a/src/libutil/include/nix/util/os-string.hh b/src/libutil/include/nix/util/os-string.hh index 7e3bf47d38e8..af686a62cb82 100644 --- a/src/libutil/include/nix/util/os-string.hh +++ b/src/libutil/include/nix/util/os-string.hh @@ -104,4 +104,14 @@ inline OsStrings toOsStrings(std::list ss) # define OS_STR(s) L##s #endif +#ifdef _WIN32 + +template +C tokenizeString(OsStringView s, OsStringView separators = OS_STR(" \t\n\r")); + +extern template std::list tokenizeString(OsStringView s, OsStringView separators); +extern template std::vector tokenizeString(OsStringView s, OsStringView separators); + +#endif + } // namespace nix diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index b154ee215225..9181f4466e1b 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -37,7 +37,7 @@ AutoCloseFD openFileReadonly(const std::filesystem::path & path, FinalSymlink fi AutoCloseFD openNewFileForWrite(const std::filesystem::path & path, mode_t mode, OpenNewFileForWriteParams params) { - auto flags = O_WRONLY | O_CREAT | O_CLOEXEC; + auto flags = (params.writeOnly ? O_WRONLY : O_RDWR) | O_CREAT | O_CLOEXEC; if (params.truncateExisting) { flags |= O_TRUNC; if (!params.followSymlinksOnTruncate) diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index e644f5258029..98d41a9caa25 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -57,7 +57,7 @@ openNewFileForWrite(const std::filesystem::path & path, [[maybe_unused]] mode_t { return AutoCloseFD{CreateFileW( path.c_str(), - GENERIC_WRITE, + GENERIC_WRITE | (params.writeOnly ? 0 : GENERIC_READ), FILE_SHARE_READ | FILE_SHARE_DELETE, /*lpSecurityAttributes=*/nullptr, params.truncateExisting ? CREATE_ALWAYS : CREATE_NEW, /* TODO: Reparse points. */ diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index f0fdb0c02b18..9d3cf583c368 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -4,6 +4,7 @@ #include #include "nix/util/os-string.hh" +#include "nix/util/strings-inline.hh" namespace nix { @@ -29,4 +30,17 @@ OsString string_to_os_string(std::string s) return string_to_os_string(std::string_view{s}); } +#ifdef _WIN32 + +template +C tokenizeString(OsStringView s, OsStringView separators) +{ + return basicTokenizeString(s, separators); +} + +template std::list tokenizeString(OsStringView s, OsStringView separators); +template std::vector tokenizeString(OsStringView s, OsStringView separators); + +#endif + } // namespace nix diff --git a/src/nix/edit.cc b/src/nix/edit.cc index aeb26915ba0e..a57371bea54e 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -44,16 +44,17 @@ struct CmdEdit : InstallableValueCommand logger->stop(); - auto args = editorFor(file, line); - - restoreProcessContext(); - - execvp(args.front().c_str(), stringsToCharPtrs(args).data()); - - std::string command; - for (const auto & arg : args) - command += " '" + arg + "'"; - throw SysError("cannot run command%s", command); + auto [args, tempFd, delTemp] = editorFor(file, line, /*readOnly=*/true); + auto program = args.front(); + args.pop_front(); + + runProgram2( + RunOptions{ + .program = program, + .lookupPath = true, + .args = std::move(args), + .isInteractive = true, + }); } }; diff --git a/tests/functional/flakes/edit.sh b/tests/functional/flakes/edit.sh index fda5c29fd084..0d926c28745d 100755 --- a/tests/functional/flakes/edit.sh +++ b/tests/functional/flakes/edit.sh @@ -6,3 +6,7 @@ createFlake1 export EDITOR=cat nix edit "$flake1Dir#" | grepQuiet simple.builder.sh +tar --exclude=".git*" -czf "$TEST_ROOT"/flake1Dir.tar.gz -C "$(dirname "$flake1Dir")" "$(basename "$flake1Dir")" +# Test that editing a file from a tarball flake works and the file is readonly. +nix edit "file://$TEST_ROOT/flake1Dir.tar.gz" | grepQuiet simple.builder.sh +EDITOR='test ! -w' nix edit "file://$TEST_ROOT/flake1Dir.tar.gz" diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 31f2fbdd88e7..9fc803d092e0 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -381,6 +381,9 @@ import $testDir/lang/parse-fail-eof-pos.nix " \ '.*error: syntax error, unexpected end of file.*' +EDITOR='cat' nix repl <<< ':e derivation' 2>&1 | grepQuiet 'derivationStrict' +EDITOR='cat' nix repl <<< ':e ' 2>&1 | grepQuiet 'builtin:fetchurl' + # TODO: move init to characterisation/framework.sh badDiff=0 badExitCode=0 From 291e2662ad0b5a302bb50e2258e50c000f3fd2f9 Mon Sep 17 00:00:00 2001 From: steveoliphant <8796107+steveoliphant@users.noreply.github.com> Date: Wed, 15 Apr 2026 08:47:54 -0600 Subject: [PATCH 248/555] Fix typo in content-address documentation --- doc/manual/source/store/derivation/outputs/content-address.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/store/derivation/outputs/content-address.md b/doc/manual/source/store/derivation/outputs/content-address.md index aa65fbe4932a..9fadcff3c6ca 100644 --- a/doc/manual/source/store/derivation/outputs/content-address.md +++ b/doc/manual/source/store/derivation/outputs/content-address.md @@ -13,7 +13,7 @@ Given the method, the output's name (computed from the derivation name and outpu ## Fixed-output content-addressing {#fixed} In this case the content address of the *fixed* in advanced by the derivation itself. -In other words, when the derivation has finished [building](@docroot@/store/building.md), and the provisional output' content-address is computed as part of the process to turn it into a *bona fide* store object, the calculated content address must much that given in the derivation, or the build of that derivation will be deemed a failure. +In other words, when the derivation has finished [building](@docroot@/store/building.md), and the provisional output' content-address is computed as part of the process to turn it into a *bona fide* store object, the calculated content address must match that given in the derivation, or the build of that derivation will be deemed a failure. The output spec for an output with a fixed content addresses additionally contains: From e89a8dbde63121f684b4de7896069cf8c4db2aec Mon Sep 17 00:00:00 2001 From: drorspei Date: Wed, 15 Apr 2026 18:44:56 +0200 Subject: [PATCH 249/555] fix concurrent localstore migrations --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d52daaae1fd1..b0c41c35ce65 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -584,7 +584,7 @@ void LocalStore::upgradeDBSchema(State & state) debug("executing Nix database schema migration '%s'...", migrationName); SQLiteTxn txn(state.db); - state.db.exec(stmt + fmt(";\ninsert into SchemaMigrations values('%s')", migrationName)); + state.db.exec(stmt + fmt(";\ninsert or ignore into SchemaMigrations values('%s')", migrationName)); txn.commit(); schemaMigrations.insert(migrationName); From 099fd303c0b94e717fe70c0eedfc3ea37b3d038a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:20 +0300 Subject: [PATCH 250/555] libstore: Reduce memory usage of the Worker even more Several things. Derivation is the primary memory hog in the build loop, with the total memory usage getting into gigabytes just to hold the derivations in memory. Previously I've optimised most goals to share a single ref, but turns out I missed DerivationResolutionGoal. Also fixes a bug in the move assignment of Goal::Co that didn't destroy the coroutine frame of the Goal being moved into. Just this single commit lowers the memory usage of the build loop by 25% in my tests (from 4.2G -> 3.2G) on a 70k derivation closure. --- src/libstore/build/derivation-goal.cc | 17 +++++++---------- .../build/derivation-resolution-goal.cc | 4 ++-- src/libstore/build/goal.cc | 9 +++++++-- src/libstore/build/worker.cc | 2 +- .../store/build/derivation-resolution-goal.hh | 5 +++-- src/libstore/include/nix/store/build/goal.hh | 4 +++- src/libstore/include/nix/store/build/worker.hh | 2 +- 7 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6e2d3223b10f..6d76cd9d275e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -144,11 +144,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) worker.store.printStorePath(drvPath)); } - auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, *drv, buildMode); - { - Goals waitees{resolutionGoal}; - co_await await(std::move(waitees)); - } + auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, drv, buildMode); + co_await await({resolutionGoal}); + if (nrFailed != 0) { co_return doneFailure({BuildResult::Failure::DependencyFailed, "Build failed due to failed dependency"}); } @@ -219,6 +217,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) assert(false); } + /* We don't need it any more and don't want to hold on to it while suspended. */ + resolutionGoal.reset(); + /* Give up on substitution for the output we want, actually build this derivation */ auto g = worker.makeDerivationBuildingGoal(drvPath, drv, buildMode, storeDerivation); @@ -226,11 +227,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* We will finish with it ourselves, as if we were the derivational goal. */ g->preserveFailure = true; - { - Goals waitees; - waitees.insert(g); - co_await await(std::move(waitees)); - } + co_await await({g}); trace("outer build done"); diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index 81c698e18563..1343267eff31 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -7,10 +7,10 @@ namespace nix { DerivationResolutionGoal::DerivationResolutionGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode) : Goal(worker, resolveDerivation()) , drvPath(drvPath) - , drv{std::make_unique(drv)} + , drv(std::move(drv)) , buildMode{buildMode} { name = fmt("resolving derivation '%s'", worker.store.printStorePath(drvPath)); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 946314afe140..1bfa563d159a 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -65,10 +65,15 @@ Co::Co(Co && rhs) rhs.handle = nullptr; } -void Co::operator=(Co && rhs) +Co & Co::operator=(Co && rhs) { - this->handle = rhs.handle; + if (handle) { + handle.promise().alive = false; + handle.destroy(); + } + handle = rhs.handle; rhs.handle = nullptr; + return *this; } Co::~Co() diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 0c63beb4dd1e..002bb1a30d79 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -104,7 +104,7 @@ std::shared_ptr Worker::makeDerivationGoal( } std::shared_ptr -Worker::makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode) +Worker::makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode) { return initGoalIfNeeded(derivationResolutionGoals[drvPath], drvPath, drv, *this, buildMode); } diff --git a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh index 843e4031aa7e..972558e9706f 100644 --- a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh @@ -37,7 +37,8 @@ struct DerivationResolutionGoal : public Goal { friend class Worker; - DerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode); + DerivationResolutionGoal( + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode); /** * If the derivation needed to be resolved, this is resulting @@ -55,7 +56,7 @@ private: /** * The derivation stored at drvPath. */ - std::unique_ptr drv; + ref drv; /** * The remainder is state held during the build. diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 0b7367ff7e02..f8293609e11c 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -233,8 +233,10 @@ public: explicit Co(handle_type handle) : handle(handle) {}; - void operator=(Co &&); + Co & operator=(Co &&); Co(Co && rhs); + Co & operator=(const Co &) = delete; + Co(const Co & rhs) = delete; ~Co(); bool await_ready() diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 4c836986811f..12e6c0122e1c 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -265,7 +265,7 @@ public: * @ref DerivationResolutionGoal "derivation resolution goal" */ std::shared_ptr - makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode); + makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode); /** * @ref DerivationBuildingGoal "derivation building goal" From 88c137708fc581dd64c67a9ae6fa3b452af6eb3f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:32 +0300 Subject: [PATCH 251/555] libstore: Move childEvents from the promise into Goal itself ChildEvents has no business being in the coroutines promise type - it bloats it unnecessarily. The whole promise type was like 500 bytes just because it stored TimedOut inline - now it's behind a unique_ptr as it should have been. Shaves off like 300MB of memory usage on my stress-testing benchmark. --- .../build/derivation-building-goal.cc | 8 +- src/libstore/build/goal.cc | 24 +++--- src/libstore/include/nix/store/build/goal.hh | 82 +++++++++---------- 3 files changed, 55 insertions(+), 59 deletions(-) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 57a4017e8e24..248ef7507c4e 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -764,9 +764,9 @@ Goal::Co DerivationBuildingGoal::buildWithHook( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { hook.reset(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } @@ -1035,9 +1035,9 @@ Goal::Co DerivationBuildingGoal::buildLocally( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { builder->killChild(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 1bfa563d159a..7651282e3e4f 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -12,16 +12,15 @@ TimedOut::TimedOut(time_t maxDuration) using Co = nix::Goal::Co; using promise_type = nix::Goal::promise_type; -using ChildEvents = decltype(promise_type::childEvents); -void ChildEvents::pushChildEvent(ChildOutput event) +void Goal::ChildEvents::pushChildEvent(ChildOutput event) { if (childTimeout) return; // Already timed out, ignore childOutputs.push(std::move(event)); } -void ChildEvents::pushChildEvent(ChildEOF event) +void Goal::ChildEvents::pushChildEvent(ChildEOF event) { if (childTimeout) return; // Already timed out, ignore @@ -29,20 +28,20 @@ void ChildEvents::pushChildEvent(ChildEOF event) childEOF = std::move(event); } -void ChildEvents::pushChildEvent(TimedOut event) +void Goal::ChildEvents::pushChildEvent(TimedOut event) { // Timeout is immediate - flush pending events childOutputs = {}; childEOF.reset(); - childTimeout = std::move(event); + childTimeout = std::make_unique(std::move(event)); } -bool ChildEvents::hasChildEvent() const +bool Goal::ChildEvents::hasChildEvent() const { return !childOutputs.empty() || childEOF || childTimeout; } -Goal::ChildEvent ChildEvents::popChildEvent() +Goal::ChildEvent Goal::ChildEvents::popChildEvent() { if (!childOutputs.empty()) { auto event = std::move(childOutputs.front()); @@ -52,7 +51,7 @@ Goal::ChildEvent ChildEvents::popChildEvent() if (childEOF) return *std::exchange(childEOF, std::nullopt); if (childTimeout) - return *std::exchange(childTimeout, std::nullopt); + return std::exchange(childTimeout, nullptr); unreachable(); } @@ -278,22 +277,19 @@ void Goal::work() void Goal::handleChildOutput(Descriptor fd, std::string_view data) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); + childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); worker.wakeUp(shared_from_this()); } void Goal::handleEOF(Descriptor fd) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildEOF{fd}); + childEvents.pushChildEvent(ChildEOF{fd}); worker.wakeUp(shared_from_this()); } void Goal::timedOut(TimedOut && ex) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(std::move(ex)); + childEvents.pushChildEvent(std::move(ex)); worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index f8293609e11c..6ddc73250d34 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -74,7 +74,43 @@ enum struct JobCategory { struct Goal : public std::enable_shared_from_this { + /** + * Event types for child process communication, delivered via coroutines. + */ + struct ChildOutput + { + Descriptor fd; + std::string data; + }; + + struct ChildEOF + { + Descriptor fd; + }; + + using ChildEvent = std::variant>; + private: + class ChildEvents + { + /** + * Structured queue of child events: + * - outputs: stream of data from child + * - eof: optional end-of-stream marker + * - timeout: optional timeout that flushes/overrides other events + */ + std::queue childOutputs; + std::optional childEOF; + std::unique_ptr childTimeout; + + public: + void pushChildEvent(ChildOutput event); + void pushChildEvent(ChildEOF event); + void pushChildEvent(TimedOut event); + bool hasChildEvent() const; + ChildEvent popChildEvent(); + }; + /** * Goals that this goal is waiting for. */ @@ -85,6 +121,8 @@ private: */ std::optional cachedKey; + ChildEvents childEvents; + public: typedef enum { ecBusy, ecSuccess, ecFailed, ecNoSubstituters } ExitCode; @@ -152,22 +190,6 @@ public: friend Goal; }; - /** - * Event types for child process communication, delivered via coroutines. - */ - struct ChildOutput - { - Descriptor fd; - std::string data; - }; - - struct ChildEOF - { - Descriptor fd; - }; - - using ChildEvent = std::variant; - /** * Tag type for `co_await`-ing child events. * Returns a `ChildEvent` when resumed. @@ -321,28 +343,6 @@ public: */ bool alive = true; - class - { - /** - * Structured queue of child events: - * - outputs: stream of data from child - * - eof: optional end-of-stream marker - * - timeout: optional timeout that flushes/overrides other events - */ - std::queue childOutputs; - std::optional childEOF; - std::optional childTimeout; - - public: - - void pushChildEvent(ChildOutput event); - void pushChildEvent(ChildEOF event); - void pushChildEvent(TimedOut event); - bool hasChildEvent() const; - ChildEvent popChildEvent(); - - } childEvents; - /** * The awaiter used by @ref final_suspend. */ @@ -450,7 +450,7 @@ public: bool await_ready() { - assert(!promise.childEvents.hasChildEvent()); + assert(!promise.goal->childEvents.hasChildEvent()); return false; } @@ -478,7 +478,7 @@ public: bool await_ready() { - return handle && handle.promise().childEvents.hasChildEvent(); + return handle && handle.promise().goal->childEvents.hasChildEvent(); } void await_suspend(handle_type h) @@ -489,7 +489,7 @@ public: ChildEvent await_resume() { assert(handle); - return handle.promise().childEvents.popChildEvent(); + return handle.promise().goal->childEvents.popChildEvent(); } }; From 1cded5e512927a79a6e60b5a68c97aca4c1ef0a4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:39 +0300 Subject: [PATCH 252/555] derivation-resolution-goal: Make use of deducing this for a recursive lambda, get rid of allocationsA std::function allocates for the captures, but with C++23 we have a much better way of writing recursive lambdas - so use it. --- .../build/derivation-resolution-goal.cc | 66 +++++++++---------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index 1343267eff31..f457c5614a6f 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -40,42 +40,38 @@ Goal::Co DerivationResolutionGoal::resolveDerivation() std::map, GoalPtr, value_comparison> inputGoals; - { - std::function, const DerivedPathMap::ChildNode &)> - addWaiteeDerivedPath; - - addWaiteeDerivedPath = [&](ref inputDrv, - const DerivedPathMap::ChildNode & inputNode) { - if (!inputNode.value.empty()) { - auto g = worker.makeGoal( - DerivedPath::Built{ - .drvPath = inputDrv, - .outputs = inputNode.value, - }, - buildMode == bmRepair ? bmRepair : bmNormal); - inputGoals.insert_or_assign(inputDrv, g); - waitees.insert(std::move(g)); - } - for (const auto & [outputName, childNode] : inputNode.childMap) - addWaiteeDerivedPath( - make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); - }; - - for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { - /* Ensure that pure, non-fixed-output derivations don't - depend on impure derivations. */ - if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() - && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); - if (inputDrv.type().isImpure()) - throw Error( - "pure derivation '%s' depends on impure derivation '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(inputDrvPath)); - } - - addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); + auto addWaiteeDerivedPath = [&worker = worker, buildMode = buildMode, &waitees, &inputGoals]( + this const auto & self, + ref inputDrv, + const DerivedPathMap::ChildNode & inputNode) -> void { + if (!inputNode.value.empty()) { + auto g = worker.makeGoal( + DerivedPath::Built{ + .drvPath = inputDrv, + .outputs = inputNode.value, + }, + buildMode == bmRepair ? bmRepair : bmNormal); + inputGoals.insert_or_assign(inputDrv, g); + waitees.insert(std::move(g)); } + for (const auto & [outputName, childNode] : inputNode.childMap) + self(make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); + }; + + for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { + /* Ensure that pure, non-fixed-output derivations don't + depend on impure derivations. */ + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() + && !drv->type().isFixed()) { + auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); + if (inputDrv.type().isImpure()) + throw Error( + "pure derivation '%s' depends on impure derivation '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(inputDrvPath)); + } + + addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); } co_await await(std::move(waitees)); From 50923476b9dca028051263922e607a1a1887abd8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:47 +0300 Subject: [PATCH 253/555] derivation-resolution-goal: Replace value_comparison with a lambda Having such helpers in the namespace scope in a translation unit is a bad time when using unity builds. At some point this was duplicated between several files and caused name collisions with unity builds. A lambda is much cleaner and doesn't have the same footgun. Lambdas can have template parameters too now. --- .../build/derivation-resolution-goal.cc | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index f457c5614a6f..6f55ea29d96e 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -22,23 +22,16 @@ std::string DerivationResolutionGoal::key() return "dc$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } -/** - * Used for `inputGoals` local variable below - */ -struct value_comparison -{ - template - bool operator()(const ref & lhs, const ref & rhs) const - { - return *lhs < *rhs; - } -}; - Goal::Co DerivationResolutionGoal::resolveDerivation() { Goals waitees; - std::map, GoalPtr, value_comparison> inputGoals; + using ValueComparison = decltype([](const ref & lhs, const ref & rhs) { + /* Compare the values, not the pointers themselves. */ + return *lhs < *rhs; + }); + + std::map, GoalPtr, ValueComparison> inputGoals; auto addWaiteeDerivedPath = [&worker = worker, buildMode = buildMode, &waitees, &inputGoals]( this const auto & self, From d3d6e7cdb15406b832da0d8f5a94ec0c765e6ef9 Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Thu, 16 Apr 2026 12:53:07 +1000 Subject: [PATCH 254/555] packaging/dependencies: bump mimalloc to 3.3.0 --- packaging/dependencies.nix | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 9903a93ba9ed..b44f6ae46c80 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -16,15 +16,19 @@ in scope: { inherit stdenv; - mimalloc = pkgs.mimalloc.overrideAttrs rec { - version = "3.1.6"; - src = pkgs.fetchFromGitHub { - owner = "microsoft"; - repo = "mimalloc"; - tag = "v${version}"; - hash = "sha256-7zG0Sqloanz/b+fkJ4wzO86uBmtf9fdYNAT9ixLouyY="; - }; - }; + mimalloc = + if lib.versionAtLeast pkgs.mimalloc.version "3.3.0" then + pkgs.mimalloc + else + pkgs.mimalloc.overrideAttrs rec { + version = "3.3.0"; + src = pkgs.fetchFromGitHub { + owner = "microsoft"; + repo = "mimalloc"; + tag = "v${version}"; + hash = "sha256-xy9gPihw3xvhnd6BrCYfMnnRp5dPSodynKRToYwxuzg="; + }; + }; boehmgc = (pkgs.boehmgc.override { From c485da7fdcf18d79b7ffc278c04e539057db92d8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 16 Apr 2026 23:26:36 +0300 Subject: [PATCH 255/555] libfethers: Removed unused input parameter to runHg --- src/libfetchers/mercurial.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index db0333e9087d..af5c94c1b494 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -28,10 +28,9 @@ static RunOptions hgOptions(OsStrings args) } // runProgram wrapper that uses hgOptions instead of stock RunOptions. -static std::string runHg(OsStrings args, const std::optional & input = {}) +static std::string runHg(OsStrings args) { RunOptions opts = hgOptions(std::move(args)); - opts.input = input; auto res = runProgram(std::move(opts)); From 355e3c030aa5a4077b4cdb4b95b1d598ff4c7bce Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 16 Apr 2026 23:36:16 +0300 Subject: [PATCH 256/555] libfetchers: Pass commit message in a temp file This is much easier and will allow us to get rid of a bunch of code. This is a cross port of https://gerrit.lix.systems/c/lix/+/1509. Lix patch was done by eldritch horrors - I just reimplemented this. This is not a cherry-pick because it would to too painful with how far the codebases have diverged. --- src/libfetchers/git.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 3c191d32f5a1..8b1d4dffec25 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -515,6 +515,10 @@ struct GitInputScheme : InputScheme }); if (commitMsg) { + auto [tempFd, tempPath] = createTempFile("nix-msg"); + AutoDelete delTemp(tempPath, /*recursive=*/false); + writeFull(tempFd.get(), *commitMsg); + // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` auto suspension = logger->suspend(); runProgram( @@ -528,9 +532,10 @@ struct GitInputScheme : InputScheme OS_STR("commit"), string_to_os_string(std::string(path.rel())), OS_STR("-F"), - OS_STR("-"), - }, - *commitMsg); + tempPath.native(), + }); + + delTemp.deletePath(); } } } From 28a799548c387a647175eefd118bbd5262ab01e1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 16 Apr 2026 23:52:32 +0300 Subject: [PATCH 257/555] Remove all input redirection code from runCommand This was only used for one thing (see previous commit), which is much more cleanly handled by just a temporary file. That allows us to get rid of all the threading entirely. This is the same stuff as in https://gerrit.lix.systems/c/lix/+/1510. --- src/libcmd/include/nix/cmd/repl.hh | 2 +- src/libcmd/repl.cc | 6 +- src/libfetchers/git.cc | 2 +- src/libutil/include/nix/util/processes.hh | 3 - src/libutil/unix/processes.cc | 63 ++----------------- src/libutil/windows/processes.cc | 75 +++-------------------- src/nix/repl.cc | 3 +- 7 files changed, 19 insertions(+), 135 deletions(-) diff --git a/src/libcmd/include/nix/cmd/repl.hh b/src/libcmd/include/nix/cmd/repl.hh index d46aa94b6b50..81c7b8df5a2d 100644 --- a/src/libcmd/include/nix/cmd/repl.hh +++ b/src/libcmd/include/nix/cmd/repl.hh @@ -28,7 +28,7 @@ struct AbstractNixRepl * @param programName Name of the command, e.g. `nix` or `nix-env`. * @param args arguments to the command. */ - using RunNix = void(const std::string & programName, OsStrings args, const std::optional & input); + using RunNix = void(const std::string & programName, OsStrings args); /** * @param runNix Function to run the nix CLI to support various diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 9968cd2983d7..7109eafce071 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -75,7 +75,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc RunNix * runNixPtr; - void runNix(const std::string & program, OsStrings args, const std::optional & input = {}); + void runNix(const std::string & program, OsStrings args); std::unique_ptr interacter; @@ -889,10 +889,10 @@ void NixRepl::evalString(std::string s, Value & v) state->forceValue(v, v.determinePos(noPos)); } -void NixRepl::runNix(const std::string & program, OsStrings args, const std::optional & input) +void NixRepl::runNix(const std::string & program, OsStrings args) { if (runNixPtr) - (*runNixPtr)(program, std::move(args), input); + (*runNixPtr)(program, std::move(args)); else throw Error( "Cannot run '%s' because no method of calling the Nix CLI was provided. This is a configuration problem pertaining to how this program was built. See Nix 2.25 release notes", diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8b1d4dffec25..3941c3425660 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -455,7 +455,7 @@ struct GitInputScheme : InputScheme args.push_back(destDir.native()); - runProgram("git", true, args, {}, true); + runProgram("git", true, args, true); } std::optional getSourcePath(const Input & input) const override diff --git a/src/libutil/include/nix/util/processes.hh b/src/libutil/include/nix/util/processes.hh index 6a9f75c85956..d2cb7adb7dcc 100644 --- a/src/libutil/include/nix/util/processes.hh +++ b/src/libutil/include/nix/util/processes.hh @@ -116,7 +116,6 @@ std::string runProgram( std::filesystem::path program, bool lookupPath = false, const OsStrings & args = OsStrings(), - const std::optional & input = {}, bool isInteractive = false); struct RunOptions @@ -130,8 +129,6 @@ struct RunOptions #endif std::optional chdir; std::optional environment; - std::optional input; - Source * standardIn = nullptr; Sink * standardOut = nullptr; bool mergeStderrToStdout = false; bool isInteractive = false; diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 9f23da876aaa..cb17499aa719 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -291,20 +291,15 @@ pid_t startProcess(fun processMain, const ProcessOptions & options) return pid; } -std::string runProgram( - std::filesystem::path program, - bool lookupPath, - const OsStrings & args, - const std::optional & input, - bool isInteractive) +std::string runProgram(std::filesystem::path program, bool lookupPath, const OsStrings & args, bool isInteractive) { auto res = runProgram( RunOptions{ .program = program, .lookupPath = lookupPath, .args = args, - .input = input, - .isInteractive = isInteractive}); + .isInteractive = isInteractive, + }); if (!statusOk(res.first)) throw ExecError(res.first, "program %s %s", PathFmt(program), statusToString(res.first)); @@ -333,22 +328,10 @@ void runProgram2(const RunOptions & options) { checkInterrupt(); - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - /* Create a pipe. */ - Pipe out, in; + Pipe out; if (options.standardOut) out.create(); - if (source) - in.create(); ProcessOptions processOptions; // vfork implies that the environment of the main process and the fork will @@ -368,8 +351,6 @@ void runProgram2(const RunOptions & options) if (options.mergeStderrToStdout) if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) throw SysError("cannot dup stdout into stderr"); - if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); if (options.chdir && chdir((*options.chdir).c_str()) == -1) throw SysError("chdir failed"); @@ -399,47 +380,11 @@ void runProgram2(const RunOptions & options) out.writeSide.close(); - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&] { - if (writerThread.joinable()) - writerThread.join(); - }); - - if (source) { - in.readSide.close(); - writerThread = std::thread([&] { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - if (options.standardOut) drainFD(out.readSide.get(), *options.standardOut); /* Wait for the child to finish. */ int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) - promise.get_future().get(); - if (status) throw ExecError(status, "program %1% %2%", PathFmt(options.program), statusToString(status)); } diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 46a149603fa6..b2e7b8af08bd 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -85,20 +85,15 @@ int Pid::wait(bool allowInterrupts) } // TODO: Merge this with Unix's runProgram since it's identical logic. -std::string runProgram( - std::filesystem::path program, - bool lookupPath, - const OsStrings & args, - const std::optional & input, - bool isInteractive) +std::string runProgram(std::filesystem::path program, bool lookupPath, const OsStrings & args, bool isInteractive) { auto res = runProgram( RunOptions{ .program = program, .lookupPath = lookupPath, .args = args, - .input = input, - .isInteractive = isInteractive}); + .isInteractive = isInteractive, + }); if (!statusOk(res.first)) throw ExecError(res.first, "program %s %s", PathFmt(program), statusToString(res.first)); @@ -197,7 +192,7 @@ OsString windowsEscape(const OsString & str, bool cmd) return buffer + L'"'; } -Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & options, Pipe & out, Pipe & in) +Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & options, Pipe & out) { // Setup pipes. if (options.standardOut) { @@ -206,17 +201,13 @@ Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & o } else { out.writeSide = nullFD(); } - if (options.standardIn) { - // Don't inherit the write end of the input pipe - setFDInheritable(in.writeSide, false); - } else { - in.readSide = nullFD(); - } + + AutoCloseFD in = nullFD(); STARTUPINFOW startInfo = {0}; startInfo.cb = sizeof(startInfo); startInfo.dwFlags = STARTF_USESTDHANDLES; - startInfo.hStdInput = in.readSide.get(); + startInfo.hStdInput = in.get(); startInfo.hStdOutput = out.writeSide.get(); startInfo.hStdError = out.writeSide.get(); @@ -305,24 +296,12 @@ void runProgram2(const RunOptions & options) { checkInterrupt(); - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - /* Create a pipe. */ - Pipe out, in; + Pipe out; // TODO: I copied this from unix but this is handled again in spawnProcess, so might be weird to split it up like // this if (options.standardOut) out.create(); - if (source) - in.create(); std::filesystem::path realProgram = options.program; // TODO: Implement shebang / program interpreter lookup on Windows @@ -330,52 +309,16 @@ void runProgram2(const RunOptions & options) auto suspension = logger->suspendIf(options.isInteractive); - Pid pid = spawnProcess(interpreter.has_value() ? *interpreter : realProgram, options, out, in); + Pid pid = spawnProcess(interpreter.has_value() ? *interpreter : realProgram, options, out); // TODO: This is identical to unix, deduplicate? out.writeSide.close(); - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&] { - if (writerThread.joinable()) - writerThread.join(); - }); - - if (source) { - in.readSide.close(); - writerThread = std::thread([&] { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - if (options.standardOut) drainFD(out.readSide.get(), *options.standardOut); /* Wait for the child to finish. */ int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) - promise.get_future().get(); - if (status) throw ExecError(status, "program %1% %2%", PathFmt(options.program), statusToString(status)); } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 85192faa6490..bc7cdedfdce1 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -11,7 +11,7 @@ namespace nix { -void runNix(const std::string & program, OsStrings args, const std::optional & input = {}) +void runNix(const std::string & program, OsStrings args) { auto subprocessEnv = getEnvOs(); subprocessEnv[OS_STR("NIX_CONFIG")] = string_to_os_string(globalConfig.toKeyValue()); @@ -21,7 +21,6 @@ void runNix(const std::string & program, OsStrings args, const std::optional Date: Fri, 17 Apr 2026 00:00:25 +0300 Subject: [PATCH 258/555] Deduplicate runProgram between unix/windows The code was identical already, so we can safely dedup it. --- src/libutil/include/nix/util/processes.hh | 1 + src/libutil/processes.cc | 17 +++++++++++++++++ src/libutil/unix/processes.cc | 17 ----------------- src/libutil/windows/processes.cc | 18 ------------------ 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/src/libutil/include/nix/util/processes.hh b/src/libutil/include/nix/util/processes.hh index d2cb7adb7dcc..4db35f7e8afe 100644 --- a/src/libutil/include/nix/util/processes.hh +++ b/src/libutil/include/nix/util/processes.hh @@ -134,6 +134,7 @@ struct RunOptions bool isInteractive = false; }; +// Output = error code + "standard out" output stream std::pair runProgram(RunOptions && options); void runProgram2(const RunOptions & options); diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc index a9ce2b521211..d36feb19a5c4 100644 --- a/src/libutil/processes.cc +++ b/src/libutil/processes.cc @@ -1,4 +1,5 @@ #include "nix/util/processes.hh" +#include "nix/util/serialise.hh" namespace nix { @@ -8,4 +9,20 @@ Pid & Pid::operator=(Pid && other) noexcept return *this; } +std::pair runProgram(RunOptions && options) +{ + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} + } // namespace nix diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index cb17499aa719..ef37aee22602 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -307,23 +307,6 @@ std::string runProgram(std::filesystem::path program, bool lookupPath, const OsS return res.second; } -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - void runProgram2(const RunOptions & options) { checkInterrupt(); diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index b2e7b8af08bd..3a8703462b80 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -274,24 +274,6 @@ Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & o return process; } -// TODO: Merge this with Unix's runProgram since it's identical logic. -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - void runProgram2(const RunOptions & options) { checkInterrupt(); From fbc6f35dff0bf5710e8697f52a41d1dc0c8c4d87 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Apr 2026 16:04:02 +0200 Subject: [PATCH 259/555] Move Deleter into a separate header --- src/libfetchers/git-lfs-fetch.cc | 1 + src/libfetchers/git-utils.cc | 1 + .../include/nix/fetchers/git-utils.hh | 11 ----------- src/libutil/include/nix/util/deleter.hh | 19 +++++++++++++++++++ src/libutil/include/nix/util/file-system.hh | 11 ++--------- src/libutil/include/nix/util/meson.build | 1 + 6 files changed, 24 insertions(+), 20 deletions(-) create mode 100644 src/libutil/include/nix/util/deleter.hh diff --git a/src/libfetchers/git-lfs-fetch.cc b/src/libfetchers/git-lfs-fetch.cc index 4585e68e58ee..9d2fb928d603 100644 --- a/src/libfetchers/git-lfs-fetch.cc +++ b/src/libfetchers/git-lfs-fetch.cc @@ -8,6 +8,7 @@ #include "nix/util/util.hh" #include "nix/util/hash.hh" #include "nix/store/ssh.hh" +#include "nix/util/deleter.hh" #include #include diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index abc257fb9ed5..6bc474395af8 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -13,6 +13,7 @@ #include "nix/util/util.hh" #include "nix/util/thread-pool.hh" #include "nix/util/pool.hh" +#include "nix/util/deleter.hh" #include #include diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 24a7b80087f8..725fbf398410 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -133,17 +133,6 @@ struct GitRepo virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; -// A helper to ensure that the `git_*_free` functions get called. -template -struct Deleter -{ - template - void operator()(T * p) const - { - del(p); - }; -}; - // A helper to ensure that we don't leak objects returned by libgit2. template struct Setter diff --git a/src/libutil/include/nix/util/deleter.hh b/src/libutil/include/nix/util/deleter.hh new file mode 100644 index 000000000000..7f349b10ee4b --- /dev/null +++ b/src/libutil/include/nix/util/deleter.hh @@ -0,0 +1,19 @@ +#pragma once + +namespace nix { + +/** + * A helper for `std::unique_ptr` that ensures that C APIs that require manual memory management get properly freed. The + * template parameter `del` is a function that takes a pointer and frees it. + */ +template +struct Deleter +{ + template + void operator()(T * p) const + { + del(p); + }; +}; + +} // namespace nix \ No newline at end of file diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 30301d97daee..f977e089bfef 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -12,6 +12,7 @@ #include "nix/util/types.hh" #include "nix/util/file-descriptor.hh" #include "nix/util/file-path.hh" +#include "nix/util/deleter.hh" #include #include @@ -436,15 +437,7 @@ public: } }; -struct DIRDeleter -{ - void operator()(DIR * dir) const - { - closedir(dir); - } -}; - -typedef std::unique_ptr AutoCloseDir; +typedef std::unique_ptr> AutoCloseDir; /** * Create a temporary directory. diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index e8f6bc345a53..0f7a40df7a8c 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -33,6 +33,7 @@ headers = [ config_pub_h ] + files( 'config-impl.hh', 'configuration.hh', 'current-process.hh', + 'deleter.hh', 'demangle.hh', 'english.hh', 'environment-variables.hh', From 136e1a2c93e153eb05518f4175aa453071a443b6 Mon Sep 17 00:00:00 2001 From: dramforever Date: Tue, 7 Apr 2026 15:53:57 +0800 Subject: [PATCH 260/555] fetchClosure: Add temproot before checking path This avoids the fetched path from getting garbage collected if it is already valid. --- src/libexpr/primops/fetchClosure.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index db182ab499b2..fe3873c101b4 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -23,6 +23,8 @@ static void runFetchClosureWithRewrite( const std::optional & toPathMaybe, Value & v) { + if (toPathMaybe) + state.store->addTempRoot(*toPathMaybe); // establish toPath or throw @@ -74,6 +76,7 @@ static void runFetchClosureWithRewrite( static void runFetchClosureWithContentAddressedPath( EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + state.store->addTempRoot(fromPath); if (!state.store->isValidPath(fromPath)) copyClosure(fromStore, *state.store, RealisedPath::Set{fromPath}); @@ -103,6 +106,7 @@ static void runFetchClosureWithContentAddressedPath( static void runFetchClosureWithInputAddressedPath( EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + state.store->addTempRoot(fromPath); if (!state.store->isValidPath(fromPath)) copyClosure(fromStore, *state.store, RealisedPath::Set{fromPath}); From 7fe57da4b294f8e28f6155194293dde1f59a33ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 20 Apr 2026 13:43:08 +0000 Subject: [PATCH 261/555] libstore: don't print URL userinfo in FileTransfer diagnostics When a URL with embedded credentials (scheme://user:pass@host/...) is passed to builtins.fetchurl or any other downloadFile caller, the password is printed verbatim in progress, retry-warning and error messages because they all format request.uri directly: warning: unable to download 'http://user:SECRET@host/file': ...; retrying in 256 ms Add FileTransferRequest::displayUri() which returns uri with the userinfo component stripped, and use it at every diagnostic call site. request.uri itself is unchanged so CURLOPT_URL, result.urls and fetcher-cache keys are unaffected. --- src/libstore-tests/filetransfer-request.cc | 19 +++++++++ src/libstore-tests/meson.build | 1 + src/libstore/filetransfer.cc | 42 +++++++++++++------ .../include/nix/store/filetransfer.hh | 8 ++++ tests/nixos/fetchurl.nix | 20 +++++++++ 5 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 src/libstore-tests/filetransfer-request.cc diff --git a/src/libstore-tests/filetransfer-request.cc b/src/libstore-tests/filetransfer-request.cc new file mode 100644 index 000000000000..b89bb7ed0bc1 --- /dev/null +++ b/src/libstore-tests/filetransfer-request.cc @@ -0,0 +1,19 @@ +#include + +#include "nix/store/filetransfer.hh" + +namespace nix { + +TEST(FileTransferRequest, displayUriStripsUserinfo) +{ + FileTransferRequest req(VerbatimURL{std::string{"https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"}}); + // uri itself is untouched (used for CURLOPT_URL, result.urls, cache keys). + EXPECT_EQ(req.uri.to_string(), "https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"); + // displayUri() drops the userinfo for diagnostics. + EXPECT_EQ(req.displayUri(), "https://example.org:8443/path/file.toml?x=1"); + + FileTransferRequest plain(VerbatimURL{std::string{"https://example.org/file"}}); + EXPECT_EQ(plain.displayUri(), "https://example.org/file"); +} + +} // namespace nix diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index a126a87aca8b..78a1cc4a12dd 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -64,6 +64,7 @@ sources = files( 'derived-path.cc', 'downstream-placeholder.cc', 'dummy-store.cc', + 'filetransfer-request.cc', 'filetransfer-retry.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 85bc650dc8d4..6d97fb4e3f5d 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -281,7 +281,11 @@ struct curlFileTransfer : public FileTransfer try { if (!done && enqueued) fail(FileTransferError( - Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri)); + Interrupted, + {}, + "%s of '%s' was interrupted", + Uncolored(request.noun()), + request.displayUri())); } catch (...) { ignoreExceptionInDestructor(); } @@ -297,7 +301,7 @@ struct curlFileTransfer : public FileTransfer /* Already descriptive enough. */ } catch (nix::Error & e) { /* Add more context to the error message. */ - e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.uri.to_string()); + e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.displayUri()); } catch (...) { /* Can't add more context to the error. */ } @@ -361,7 +365,7 @@ struct curlFileTransfer : public FileTransfer try { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); + printMsg(lvlVomit, "got header for '%s': %s", request.displayUri(), trim(line)); static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); if (std::smatch match; std::regex_match(line, match, statusLine)) { @@ -450,8 +454,8 @@ struct curlFileTransfer : public FileTransfer *logger, lvlTalkative, actFileTransfer, - fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri), - Logger::Fields{request.uri.to_string()}, + fmt("%s '%s'", request.verb(/*continuous=*/true), request.displayUri()), + Logger::Fields{request.displayUri()}, request.parentAct); // Reset the start time to when we actually started the download. startTime = std::chrono::steady_clock::now(); @@ -716,7 +720,7 @@ struct curlFileTransfer : public FileTransfer debug( "finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", Uncolored(request.noun()), - request.uri, + request.displayUri(), code, httpStatus, result.bodySize, @@ -815,14 +819,14 @@ struct curlFileTransfer : public FileTransfer std::move(response), "%s of '%s' was interrupted", Uncolored(request.noun()), - request.uri) + request.displayUri()) : httpStatus != 0 ? FileTransferError( err, std::move(response), "unable to %s '%s': HTTP error %d%s", Uncolored(request.verb()), - request.uri, + request.displayUri(), httpStatus, code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError( @@ -830,7 +834,7 @@ struct curlFileTransfer : public FileTransfer std::move(response), "unable to %s '%s': %s (%d) %s", Uncolored(request.verb()), - request.uri, + request.displayUri(), curl_easy_strerror(code), code, errbuf); @@ -1081,7 +1085,7 @@ struct curlFileTransfer : public FileTransfer } for (auto & item : incoming) { - debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.uri); + debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.displayUri()); item->init(); curl_multi_add_handle(curlm.get(), item->req); item->active = true; @@ -1133,7 +1137,7 @@ struct curlFileTransfer : public FileTransfer { if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https" && item->request.uri.scheme() != "s3") - throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string()); + throw nix::Error("uploading to '%s' is not supported", item->request.displayUri()); { auto state(state_.lock()); @@ -1194,6 +1198,20 @@ ref makeFileTransfer(const FileTransferSettings & settings) return makeCurlFileTransfer(settings); } +std::string FileTransferRequest::displayUri() const +{ + try { + auto parsed = uri.parsed(); + if (parsed.authority && parsed.authority->user) { + parsed.authority->user.reset(); + parsed.authority->password.reset(); + return parsed.to_string(); + } + } catch (BadURL &) { + } + return uri.to_string(); +} + void FileTransferRequest::setupForS3() { auto parsedS3 = ParsedS3URL::parse(uri.parsed()); @@ -1283,7 +1301,7 @@ void FileTransfer::download( state->request.notify_one(); }); - request.dataCallback = [_state, uri = request.uri.to_string()](std::string_view data) -> PauseTransfer { + request.dataCallback = [_state, uri = request.displayUri()](std::string_view data) -> PauseTransfer { auto state(_state->lock()); if (state->quit) diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index a423249e6634..2bd9ce201daa 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -307,6 +307,14 @@ struct FileTransferRequest { } + /** + * `uri` with any userinfo (`user:password@`) stripped, for use in + * progress, warning and error messages so credentials embedded in + * the URL don't leak into logs. Returns `uri` verbatim if it can't + * be parsed. + */ + std::string displayUri() const; + /** * Returns the method description for logging purposes. */ diff --git a/tests/nixos/fetchurl.nix b/tests/nixos/fetchurl.nix index e8663debbcd4..b3e744424e3c 100644 --- a/tests/nixos/fetchurl.nix +++ b/tests/nixos/fetchurl.nix @@ -54,6 +54,14 @@ in echo 'foobar' > "$out/index.html" ''; }; + + virtualHosts."auth" = { + basicAuth.alice = "s3cr3t"; + root = pkgs.runCommand "nginx-root" { } '' + mkdir "$out" + echo 'authed' > "$out/index.html" + ''; + }; }; security.pki.certificateFiles = [ "${goodCert}/cert.pem" ]; @@ -61,6 +69,7 @@ in networking.hosts."127.0.0.1" = [ "good" "bad" + "auth" ]; virtualisation.writableStore = true; @@ -89,5 +98,16 @@ in # Fetching from a server with a trusted cert should work via environment variable override. machine.succeed("NIX_SSL_CERT_FILE=/tmp/cafile.pem nix build --no-substitute --expr 'import { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }'") + + # builtins.fetchurl should authenticate using userinfo from the URL. + out = machine.succeed("nix eval --raw --impure --no-substitute --expr 'builtins.readFile (builtins.fetchurl { url = \"http://alice:s3cr3t@auth/index.html\"; sha256 = \"sha256-67y3HalfTt2zfgMt7BwU5vkaGWcelFlR4jryIkA1dTo=\"; })'") + assert out == "authed\n", out + + # On failure the transport-layer diagnostic must show the stripped URL, + # not the userinfo. (The eval trace still echoes the source expression; + # that is the user's own input, not a leak.) + err = machine.fail("nix eval --raw --impure --no-substitute --option tarball-ttl 0 --expr 'builtins.fetchurl { url = \"http://alice:wrong-secret@auth/index.html\"; sha256 = \"sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\"; }' 2>&1") + print(err) + assert "unable to download 'http://auth/index.html'" in err, err ''; } From f0d9109fa355d59ffdd5f07220e5b3d396256360 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:10:54 +0300 Subject: [PATCH 262/555] libexpr: Fix error message in InvalidPathError It was only printing the base name, which isn't how we usually print store paths. --- src/libexpr/eval-error.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 45cb1e409dd1..67e8d5ea1ded 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -1,11 +1,12 @@ #include "nix/expr/eval-error.hh" #include "nix/expr/eval.hh" #include "nix/expr/value.hh" +#include "nix/store/store-api.hh" namespace nix { InvalidPathError::InvalidPathError(EvalState & state, const StorePath & path) - : CloneableError(state, "path '%s' is not valid", path.to_string()) + : CloneableError(state, "path '%s' is not valid", state.store->printStorePath(path)) , path{path} { } From 2949729038395be70f99173388e992244e4678f0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Apr 2026 21:38:23 +0300 Subject: [PATCH 263/555] libflake: Fix argument order evaluation footgun mountInput modifies lockedRef.input to stuff narHash into it. --- src/libflake/flake.cc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index deb1c16b71e9..45871cff87ee 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -391,14 +391,9 @@ static Flake getFlake( lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } + auto rootDir = state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)); // Re-parse flake.nix from the store. - return readFlake( - state, - originalRef, - resolvedRef, - lockedRef, - state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)), - lockRootAttrPath); + return readFlake(state, originalRef, resolvedRef, lockedRef, rootDir, lockRootAttrPath); } Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries) From 49b2680eaf80bda3759139df6094dd979bc2b1b0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:52:59 +0300 Subject: [PATCH 264/555] nix flake archive: Use 'deducing this' recursive lambda instead of std::function One slight blemish I noticed while touching this code. With C++23 we can simplify things. --- src/nix/flake.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 42de78453451..15e9700d9d18 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1095,8 +1095,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs sources.insert(storePath); // FIXME: use graph output, handle cycles. - std::function traverse; - traverse = [&](const flake::Node & node) { + auto traverse = [&store, json = json, dryRun = dryRun, &sources]( + this const auto & self, const flake::Node & node) -> nlohmann::json { nlohmann::json jsonObj2 = json ? nlohmann::json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { @@ -1110,9 +1110,9 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs auto & jsonObj3 = jsonObj2[inputName]; if (storePath) jsonObj3["path"] = store->printStorePath(*storePath); - jsonObj3["inputs"] = traverse(**inputNode); + jsonObj3["inputs"] = self(**inputNode); } else - traverse(**inputNode); + self(**inputNode); } } return jsonObj2; From d5f162d1166f71a8eb1e343727a51fcfb8a88235 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 15:12:45 +0300 Subject: [PATCH 265/555] libexpr-c: Fix UAF on readOnlyMode Turns out the readOnlyMode was always dangling in the C API and nobody noticed... Previous commits just started accessing it, which was caught by ASan. This is the minimal fix I can think of. --- src/libexpr-c/nix_api_expr.cc | 21 ++++++++----------- src/libexpr-c/nix_api_expr_internal.h | 3 +-- src/libexpr/eval-settings.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/include/nix/expr/eval-settings.hh | 9 +++++++- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 97680ac6bfe7..99c8e42cc2ba 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -129,23 +129,20 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto if (context) context->last_err_code = NIX_OK; try { - return unsafe_new_with_self([&](auto * self) { - return nix_eval_state_builder{ - .store = nix::ref(store->ptr), - .settings = nix::EvalSettings{/* &bool */ self->readOnlyMode}, - .fetchSettings = nix::fetchers::Settings{}, - .readOnlyMode = true, - }; - }); + auto readOnly = nix::make_ref(true); + return new nix_eval_state_builder{ + .store = nix::ref(store->ptr), + .settings = nix::EvalSettings{/* &bool */ *readOnly}, + .fetchSettings = nix::fetchers::Settings{}, + .readOnlyMode = readOnly, + }; } NIXC_CATCH_ERRS_NULL } void nix_eval_state_builder_free(nix_eval_state_builder * builder) { - if (builder) - builder->~nix_eval_state_builder(); - operator delete(builder, static_cast(alignof(nix_eval_state_builder))); + delete builder; } nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder) @@ -154,7 +151,7 @@ nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_buil context->last_err_code = NIX_OK; try { // TODO: load in one go? - builder->settings.readOnlyMode = nix::settings.readOnlyMode; + builder->settings.readOnlyMode = &nix::settings.readOnlyMode; loadConfFile(builder->settings); loadConfFile(builder->fetchSettings); } diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index b38aeaf7b498..3f7e4bf1df12 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -18,8 +18,7 @@ struct nix_eval_state_builder nix::EvalSettings settings; nix::fetchers::Settings fetchSettings; nix::LookupPath lookupPath; - // TODO: make an EvalSettings setting own this instead? - bool readOnlyMode; + nix::ref readOnlyMode; }; struct EvalState diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 5cf0ae04304e..a6a3e829f20c 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -70,7 +70,7 @@ Strings EvalSettings::parseNixPath(const std::string & s) } EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks) - : readOnlyMode{readOnlyMode} + : readOnlyMode{&readOnlyMode} , lookupPathHooks{lookupPathHooks} { auto var = getEnv("NIX_ABORT_ON_WARN"); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1d66b8ead1c9..e4fe3ba6a5f8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2583,7 +2583,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat fetchSettings, *store, path.resolveSymlinks(SymlinkResolution::Ancestors), - settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + settings.isReadOnly() ? FetchMode::DryRun : FetchMode::Copy, path.baseName(), ContentAddressMethod::Raw::NixArchive, nullptr, diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index d9dba95370b1..a03a8dadbef1 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -71,7 +71,14 @@ struct EvalSettings : Config EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {}); - bool & readOnlyMode; + /* FIXME: This really shouldn't be public. The C API should have non-global settings instead. */ + bool * readOnlyMode = nullptr; + + bool isReadOnly() const + { + assert(readOnlyMode); + return *readOnlyMode; + } static Strings getDefaultNixPath(); From d34b442634de5d27a221e4c5885724f271228fcd Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 20 Apr 2026 21:40:25 +0300 Subject: [PATCH 266/555] flake archive: Factor out storePath computation/fetching into a lambda This is necessary for making flake store paths lazier and also slightly more concise anyway. --- src/nix/flake.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 15e9700d9d18..2a5239549e0f 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1090,20 +1090,25 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs StorePathSet sources; - auto storePath = store->toStorePath(flake.flake.path.path.abs()).first; + auto getStorePath = [&](const FlakeRef & lockedRef) { + return dryRun ? lockedRef.input.computeStorePath(*store) + : std::get(lockedRef.input.fetchToStore(fetchSettings, *store)); + }; + + auto storePath = getStorePath(flake.flake.lockedRef); sources.insert(storePath); // FIXME: use graph output, handle cycles. - auto traverse = [&store, json = json, dryRun = dryRun, &sources]( + auto traverse = [&store, json = json, &sources, &getStorePath]( this const auto & self, const flake::Node & node) -> nlohmann::json { nlohmann::json jsonObj2 = json ? nlohmann::json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { std::optional storePath; - if (!(*inputNode)->lockedRef.input.isRelative()) { - storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetchToStore(fetchSettings, *store).first; + const auto & lockedRef = (*inputNode)->lockedRef; + if (!lockedRef.input.isRelative()) { + storePath = getStorePath(lockedRef); sources.insert(*storePath); } if (json) { From 9fcb58f388611e0660db51fed4195594eb21a985 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 20 Apr 2026 21:59:22 +0300 Subject: [PATCH 267/555] libexpr-c: Remove unsafe_new_with_self This is thankfully not used by anything else anymore. Good riddance, since all usages of it had bugs in them. --- src/libexpr-c/nix_api_expr.cc | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 99c8e42cc2ba..2387ee8c38c2 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -19,27 +19,6 @@ # include #endif -/** - * @brief Allocate and initialize using self-reference - * - * This allows a brace initializer to reference the object being constructed. - * - * @warning Use with care, as the pointer points to an object that is not fully constructed yet. - * - * @tparam T Type to allocate - * @tparam F A function type for `init`, taking a T* and returning the initializer for T - * @param init Function that takes a T* and returns the initializer for T - * @return Pointer to allocated and initialized object - */ -template -static T * unsafe_new_with_self(F && init) -{ - // Allocate - void * p = ::operator new(sizeof(T), static_cast(alignof(T))); - // Initialize with placement new - return new (p) T(init(static_cast(p))); -} - extern "C" { nix_err nix_libexpr_init(nix_c_context * context) From 7d12269604077f6f302f75449c20c50d38b03f7f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:11:38 +0300 Subject: [PATCH 268/555] libfetchers: Use source accessor if the mercurial fetcher, use makeFSSourceAccessor This significantly simplifies path filtering and gets rid of raw file system accesses. --- src/libfetchers/mercurial.cc | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index af5c94c1b494..ab1d31ed330f 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -240,26 +240,25 @@ struct MercurialInputScheme : InputScheme }), "\0"s); - auto actualPath = absPath(localPath); + /* FIXME: Check that the access to this path is allowed. */ + auto accessor = makeFSSourceAccessor(absPath(localPath)); PathFilter filter = [&](const std::string & p) -> bool { - assert(hasPrefix(p, actualPath.string())); - std::string file(p, actualPath.string().size() + 1); + auto cp = CanonPath(p); + auto st = accessor->lstat(cp); - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; + if (st.type == SourceAccessor::tDirectory) { + auto prefix = cp.rel() + "/"; auto i = files.lower_bound(prefix); return i != files.end() && hasPrefix(*i, prefix); } - return files.count(file); + return files.count(cp.rel()); }; return store.addToStore( input.getName(), - {getFSSourceAccessor(), CanonPath(actualPath.string())}, + {accessor, CanonPath::root}, ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, @@ -381,7 +380,7 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir / ".hg_archival.txt"); - auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir.string())}); + auto storePath = store.addToStore(name, {makeFSSourceAccessor(tmpDir), CanonPath::root}); Attrs infoAttrs({ {"revCount", (uint64_t) revCount}, From 48100ab18c99a63c195d1d17f249fe9fc685b60f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 00:14:41 +0300 Subject: [PATCH 269/555] libutil: Add a dirFd callback for iterative openFileEnsureBeneathNoSymlinks This would be useful for caching parent directory file descriptors in the source accessor. I don't think the windows code works too well now, so I left the callback there as a FIXME to be called. It's just a perf improvement in general and shouldn't break if it never gets called. --- .../include/nix/util/file-system-at.hh | 8 ++++--- src/libutil/unix/file-system-at.cc | 23 +++++++++++++++---- src/libutil/windows/file-system-at.cc | 8 ++++++- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh index 0f4678461ca0..17cd8b46688a 100644 --- a/src/libutil/include/nix/util/file-system-at.hh +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -90,6 +90,8 @@ OsString readLinkAt(Descriptor dirFd, const CanonPath & path); * @param flags (Unix) `O_*` flags (must not include `O_NOFOLLOW`) * @param mode (Unix) Mode for `O_{CREAT,TMPFILE}` * + * @param dirFdCallback Callback invoked that gets the ownership of an intermediate directory fd. + * * @pre `path.isRoot()` is false * * @throws SymlinkNotAllowed if an interior path component is a @@ -118,12 +120,12 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( #ifdef _WIN32 ACCESS_MASK desiredAccess, ULONG createOptions, - ULONG createDisposition = FILE_OPEN + ULONG createDisposition = FILE_OPEN, #else int flags, - mode_t mode = 0 + mode_t mode = 0, #endif -); + std::function dirFdCallback = nullptr); #ifdef __linux__ namespace linux { diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index 762a245ded04..11c7bcd064fa 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -140,20 +140,27 @@ void unix::fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t } } -static AutoCloseFD -openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) +static AutoCloseFD openFileEnsureBeneathNoSymlinksIterative( + Descriptor dirFd, + const CanonPath & path, + int flags, + mode_t mode, + std::function dirFdCallback) { AutoCloseFD parentFd; auto nrComponents = std::ranges::distance(path); assert(nrComponents >= 1); auto components = std::views::take(path, nrComponents - 1); /* Everything but last component */ auto getParentFd = [&]() { return parentFd ? parentFd.get() : dirFd; }; + auto currentRelPath = CanonPath::root; /* This rather convoluted loop is necessary to avoid TOCTOU when validating that no inner path component is a symlink. */ for (auto it = components.begin(); it != components.end(); ++it) { auto component = std::string(*it); /* Copy into a string to make NUL terminated. */ assert(component != ".." && !component.starts_with('/')); /* In case invariant is broken somehow.. */ + auto prevRelPath = currentRelPath; + currentRelPath = currentRelPath / *it; AutoCloseFD parentFd2 = ::openat( getParentFd(), /* First iteration uses dirFd. */ @@ -188,6 +195,9 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat return AutoCloseFD{}; } + if (dirFdCallback && parentFd) + dirFdCallback(std::move(parentFd), std::move(prevRelPath)); + parentFd = std::move(parentFd2); } @@ -221,7 +231,12 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat return res; } -AutoCloseFD openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) +AutoCloseFD openFileEnsureBeneathNoSymlinks( + Descriptor dirFd, + const CanonPath & path, + int flags, + mode_t mode, + std::function dirFdCallback) { /* Just in case the invariant is somehow broken. */ assert(!path.rel().starts_with('/')); @@ -263,7 +278,7 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & } #endif - return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode); + return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode, std::move(dirFdCallback)); } OsString readLinkAt(Descriptor dirFd, const CanonPath & path) diff --git a/src/libutil/windows/file-system-at.cc b/src/libutil/windows/file-system-at.cc index 668bd7eb402f..54248f2a36ca 100644 --- a/src/libutil/windows/file-system-at.cc +++ b/src/libutil/windows/file-system-at.cc @@ -229,7 +229,13 @@ PosixStat fstat(Descriptor fd) } AutoCloseFD openFileEnsureBeneathNoSymlinks( - Descriptor dirFd, const CanonPath & path, ACCESS_MASK desiredAccess, ULONG createOptions, ULONG createDisposition) + Descriptor dirFd, + const CanonPath & path, + ACCESS_MASK desiredAccess, + ULONG createOptions, + ULONG createDisposition, + /* FIXME: Actually call this callback. */ + [[maybe_unused]] std::function dirFdCallback) { assert(!path.isRoot()); assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */ From 293aa8ded34dd79d04217505897b678a9d7bc522 Mon Sep 17 00:00:00 2001 From: Krish Jaiswal Date: Tue, 21 Apr 2026 02:52:45 +0530 Subject: [PATCH 270/555] Add redirect for language/values.html --- doc/manual/source/_redirects | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 07b3130f9ce9..7e4557f7d595 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -36,6 +36,7 @@ /expressions/language-values /language/values 301! /expressions/* /language/:splat 301! /language/values /language/types 301! +/language/values.html /language/types 301! /language/constructs /language/syntax 301! /language/builtin-constants /language/builtins 301! From e069dae4ef9f7e97cf5869f9c0c124979b54364a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 21 Apr 2026 00:01:32 +0200 Subject: [PATCH 271/555] release-jobs: include all buildCross.nix-everything targets Avoids drifting from upload-release.pl when new cross targets such as x86_64-unknown-freebsd are added to fallback-paths. Also drop the reference to a not-yet-existing Python rewrite of the upload script. Addresses review comments on #15640. --- packaging/release-jobs.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix index 3da8c80de315..23add3905e33 100644 --- a/packaging/release-jobs.nix +++ b/packaging/release-jobs.nix @@ -1,5 +1,5 @@ # Hydra jobset containing only the artifacts consumed by -# `maintainers/upload-release.{pl,py}`, so a release can be cut without +# `maintainers/upload-release.pl`, so a release can be cut without # waiting on the full `hydraJobs` CI matrix. # # Evaluated as a legacy (non-flake) jobset because Hydra hard-codes flake @@ -36,8 +36,7 @@ let # `fallback-paths.nix` and (on x86_64-linux) the rendered manual via # its `doc` output. build.nix-everything = hydraJobs.build.nix-everything; - buildCross.nix-everything.riscv64-unknown-linux-gnu = - hydraJobs.buildCross.nix-everything.riscv64-unknown-linux-gnu; + buildCross.nix-everything = hydraJobs.buildCross.nix-everything; inherit (hydraJobs) manual From 7981f28017b7af317546749e25bddb412b3f4c52 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 01:54:04 +0300 Subject: [PATCH 272/555] libutil: Implement unix source accessors that work with file descriptors --- src/libutil/posix-source-accessor.cc | 351 +++++++++++++++++- tests/functional/fetchGit.sh | 2 +- ...val-fail-readDir-not-a-directory-1.err.exp | 2 +- ...val-fail-readDir-not-a-directory-2.err.exp | 2 +- 4 files changed, 352 insertions(+), 5 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 609855281389..bd28f81fa594 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -1,11 +1,19 @@ #include "nix/util/posix-source-accessor.hh" #include "nix/util/file-system-at.hh" +#include "nix/util/lru-cache.hh" +#include "nix/util/sync.hh" #include "nix/util/memory-source-accessor.hh" #include "nix/util/source-path.hh" #include "nix/util/signals.hh" #include +#include + +#ifndef _WIN32 +# include +#endif + namespace nix { static SourceAccessor::Stat sourceAccessorStatFromPosixStat(const PosixStat & st) @@ -31,8 +39,14 @@ static SourceAccessor::Stat sourceAccessorStatFromPosixStat(const PosixStat & st namespace { +#ifndef _WIN32 + +class PosixDirectorySourceAccessor; + class PosixFileSourceAccessor : public detail::PosixSourceAccessorBase { + friend class PosixDirectorySourceAccessor; + AutoCloseFD fd; std::filesystem::path fsPath; /** @@ -122,6 +136,331 @@ std::string PosixFileSourceAccessor::readLink(const CanonPath & path) throw NotASymlink("path '%1%' is not a symlink", showPath(path)); } +static unsigned getGlobalDirFdCacheLimit() +{ + ::rlimit lim{}; + if (::getrlimit(RLIMIT_NOFILE, &lim) == -1) + throw SysError("querying RLIMIT_NOFILE"); + /* Some sane upper bound in case we have a huge rlimit. */ + return std::min(4096, lim.rlim_cur / 8); +} + +class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase +{ +public: + static unsigned getGlobalFdLimit() + { + static auto res = getGlobalDirFdCacheLimit(); + return res; + } + + static void registerAccessor(ref accessor) + { + auto reg = globalDirFdCacheRegistry.lock(); + std::erase_if(*reg, [](auto & maybeAccessor) { return maybeAccessor.expired(); }); + reg->push_back(accessor.get_ptr()); + } + +private: + AutoCloseFD dirFd; + std::filesystem::path fsPath; + + std::shared_ptr>>> dirFdCache; + + static inline std::atomic globalDirFdCount = 0; + + static inline Sync>> globalDirFdCacheRegistry; + + static void maybeEvictFromGlobalCaches() + { + if (globalDirFdCount.load(std::memory_order_relaxed) < getGlobalFdLimit()) + return; + + auto registry(globalDirFdCacheRegistry.lock()); + for (auto it = registry->begin(); it != registry->end();) { + if (globalDirFdCount.load(std::memory_order_relaxed) < getGlobalFdLimit()) + break; + + auto accessor = it->lock(); + if (!accessor) { + it = registry->erase(it); + continue; + } + + /* TODO: Would be nicer if we could evict a portion of the utilised + cache to avoid cold-start issues. Should be fine for now. */ + if (accessor->dirFdCache) { + auto cache = accessor->dirFdCache->lock(); + globalDirFdCount.fetch_sub(cache->size(), std::memory_order_relaxed); + cache->clear(); + } + + ++it; + } + } + + void insertIntoDirFdCache(const CanonPath & key, ref fd) + { + assert(dirFdCache); + auto cache = dirFdCache->lock(); + auto before = cache->size(); + cache->upsert(key, std::move(fd)); + globalDirFdCount.fetch_add(cache->size() - before, std::memory_order_relaxed); + } + + /** + * Get the parent directory of path. The second pair element might be an owning file descriptor + * if path.parent().isRoot() is false. + */ + std::pair> openParent(const CanonPath & path); + + std::function makeDirFdCallback(); + +public: + PosixDirectorySourceAccessor( + AutoCloseFD fd, std::filesystem::path path, bool trackLastModified, unsigned dirFdCacheSize) + : PosixSourceAccessorBase(trackLastModified) + , dirFd(std::move(fd)) + , fsPath(std::move(path)) + { + assert(fsPath.is_absolute()); /* Only used for error messages, but still nice to enforce this invariant. */ + setPathDisplay(fsPath.generic_string()); + + if (dirFdCacheSize) + dirFdCache = std::make_shared>>>(dirFdCacheSize); + } + + PosixDirectorySourceAccessor(PosixDirectorySourceAccessor &&) = delete; + PosixDirectorySourceAccessor(const PosixDirectorySourceAccessor &) = delete; + PosixDirectorySourceAccessor & operator=(PosixDirectorySourceAccessor &&) = delete; + PosixDirectorySourceAccessor & operator=(const PosixDirectorySourceAccessor &) = delete; + + ~PosixDirectorySourceAccessor() + { + if (dirFdCache) { + auto cache = dirFdCache->lock(); + globalDirFdCount.fetch_sub(cache->size(), std::memory_order_relaxed); + } + } + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override + { + if (path.isRoot()) + return fsPath; + return std::filesystem::path(fsPath) / path.rel(); /* RHS *must* be a relative path. */ + } + + std::string showPath(const CanonPath & path) override + { + if (path.isRoot()) + return displayPrefix; /* No trailing slash. */ + if (displayPrefix.ends_with('/')) + return displayPrefix + path.rel(); + return displayPrefix + path.abs(); + } +}; + +std::function PosixDirectorySourceAccessor::makeDirFdCallback() +{ + if (!dirFdCache) + return nullptr; + + return [this](AutoCloseFD fd, CanonPath key) { + assert(fd); + insertIntoDirFdCache(std::move(key), make_ref(std::move(fd))); + }; +} + +std::pair> PosixDirectorySourceAccessor::openParent(const CanonPath & path) +{ + assert(!path.isRoot()); + auto parent = path.parent().value(); + if (parent.isRoot()) + return {dirFd.get(), nullptr}; + + maybeEvictFromGlobalCaches(); + + if (dirFdCache) { + if (auto cachedFd = dirFdCache->lock()->get(parent)) { + assert((*cachedFd)->get()); + return {(*cachedFd)->get(), *cachedFd}; + } + } + + AutoCloseFD parentFdOwning = openFileEnsureBeneathNoSymlinks( + dirFd.get(), parent, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, makeDirFdCallback()); + + return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; +} + +std::optional PosixDirectorySourceAccessor::maybeLstat(const CanonPath & path) +try { + PosixStat st; + + if (path.isRoot()) { + /* Must never fail - we already have the file descriptor for the directory. */ + st = nix::fstat(dirFd.get()); + } else { + auto [parentFd, parentFdOwning] = openParent(path); + if (parentFd == INVALID_DESCRIPTOR) { + if (errno == ENOENT || errno == ENOTDIR) + return std::nullopt; + throw SysError("opening directory '%1%'", showPath(path.parent().value())); + } + + if (dirFdCache && parentFdOwning) { + assert(*parentFdOwning); + insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); + } + + /* We know that CanonPath returns a NUL-terminated string_view, so the use of ->data() here is safe. */ + if (::fstatat(parentFd, path.baseName()->data(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) + return std::nullopt; + throw SysError("getting status of '%1%'", showPath(path)); + } + } + + maybeUpdateMtime(st.st_mtime); + return sourceAccessorStatFromPosixStat(st); +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +void PosixDirectorySourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) +try { + if (path.isRoot()) + throw NotARegularFile("'%s' is not a regular file", showPath(path)); + + AutoCloseFD fileFd = + openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + + if (!fileFd) { + if (errno == ENOENT || errno == ENOTDIR) /* Intermediate component might not exist. */ + throw FileNotFound("file '%s' does not exist", showPath(path)); + throw SysError("opening '%s'", showPath(path)); + } + + auto st = nix::fstat(fileFd.get()); + if (!S_ISREG(st.st_mode)) + throw Error("file '%s' has an unsupported type", showPath(path)); + PosixFileSourceAccessor fileAccessor(std::move(fileFd), fsPath / path.rel(), trackLastModified, st); + maybeUpdateMtime(st.st_mtime); + fileAccessor.readFile(CanonPath::root, sink, sizeCallback); +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) +try { + AutoCloseFD dirFdOwning; + + if (path.isRoot()) { + /* Get a fresh file descriptor for thread-safety. */ + dirFdOwning = ::openat(dirFd.get(), ".", O_DIRECTORY | O_RDONLY | O_CLOEXEC); + if (!dirFdOwning) + throw SysError("opening directory '%s'", showPath(path)); + } else { + dirFdOwning = openFileEnsureBeneathNoSymlinks( + dirFd.get(), path, O_DIRECTORY | O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + + if (!dirFdOwning) { + if (errno == ENOTDIR) + throw NotADirectory("'%s' is not a directory", showPath(path)); + throw SysError("opening directory '%s'", showPath(path)); + } + } + + AutoCloseDir dir(::fdopendir(dirFdOwning.get())); + if (!dir) + throw SysError("reading directory '%s'", showPath(path)); + dirFdOwning.release(); + + DirEntries entries; + const ::dirent * dirent = nullptr; + + while (errno = 0, dirent = ::readdir(dir.get())) { + checkInterrupt(); + std::string_view name(dirent->d_name); + if (name == "." || name == "..") + continue; + + std::optional type; + switch (dirent->d_type) { + case DT_REG: + type = tRegular; + break; + case DT_DIR: + type = tDirectory; + break; + case DT_LNK: + type = tSymlink; + break; + case DT_CHR: + type = tChar; + break; + case DT_BLK: + type = tBlock; + break; + case DT_FIFO: + type = tFifo; + break; + case DT_SOCK: + type = tSocket; + break; + default: + type = std::nullopt; + break; + } + entries.emplace(name, type); + } + + if (errno) + throw SysError("reading directory '%1%'", showPath(path)); + + return entries; +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +std::string PosixDirectorySourceAccessor::readLink(const CanonPath & path) +try { + if (path.isRoot()) + throw NotASymlink("file '%s' is not a symlink", showPath(path)); + + auto [parentFd, parentFdOwning] = openParent(path); + if (parentFd == INVALID_DESCRIPTOR) { + if (errno == ENOENT || errno == ENOTDIR) + throw FileNotFound("path '%s' does not exist", showPath(path)); + throw SysError("opening directory '%1%'", showPath(path.parent().value())); + } + + if (dirFdCache && parentFdOwning) { + assert(*parentFdOwning); + insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); + } + + try { + return readLinkAt(parentFd, CanonPath(path.baseName().value())); + } catch (SysError & e) { + if (e.errNo == EINVAL) + throw NotASymlink("file '%s' is not a symlink", showPath(path)); + throw; + } +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +#endif + } // namespace PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) @@ -303,7 +642,7 @@ void PosixSourceAccessor::assertNoSymlinks(CanonPath path) ref getFSSourceAccessor() { - static auto rootFS = make_ref(); + static auto rootFS = makeFSSourceAccessor("/", /*trackLastModified=*/false); return rootFS; } @@ -383,7 +722,15 @@ ref makeFSSourceAccessor(std::filesystem::path root, bool trackL if (S_ISREG(st.st_mode)) return make_ref(std::move(fd), std::move(root), trackLastModified, st); - /* TODO: Use the file descriptor for fd-relative operations on the directory. */ + else if (S_ISDIR(st.st_mode)) { + auto res = make_ref( + std::move(fd), std::move(root), trackLastModified, PosixDirectorySourceAccessor::getGlobalFdLimit() / 8); + PosixDirectorySourceAccessor::registerAccessor(res); + return res; + } + + else + throw Error("file %1% has an unsupported type", PathFmt(root)); #endif return make_ref(std::move(root), trackLastModified); diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 2992020f16a8..ef57e471962d 100755 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -35,7 +35,7 @@ nix-instantiate --eval -E "builtins.readFile ((builtins.fetchGit \"file://$TEST_ # Fetch a worktree. unset _NIX_FORCE_HTTP -expectStderr 0 nix eval -vvvv --impure --raw --expr "(builtins.fetchGit \"file://$TEST_ROOT/worktree\").outPath" | grepQuiet "copying '$TEST_ROOT/worktree/' to the store" +expectStderr 0 nix eval -vvvv --impure --raw --expr "(builtins.fetchGit \"file://$TEST_ROOT/worktree\").outPath" | grepQuiet "copying '$TEST_ROOT/worktree' to the store" path0=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$TEST_ROOT/worktree\").outPath") path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/worktree\"; }).outPath") [[ $path0 = "$path0_" ]] diff --git a/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp b/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp index f94a7ed74521..7db863b03d53 100644 --- a/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp +++ b/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp @@ -13,4 +13,4 @@ error: | ^ 3| } - error: cannot read directory "/pwd/lang/readDir/bar": Not a directory + error: '/pwd/lang/readDir/bar' is not a directory diff --git a/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp b/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp index f5e6775545a4..20bbc3072e42 100644 --- a/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp +++ b/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp @@ -13,4 +13,4 @@ error: | ^ 3| } - error: cannot read directory "/pwd/lang/readDir/foo/git-hates-directories": Not a directory + error: '/pwd/lang/readDir/foo/git-hates-directories' is not a directory From 732f9c1cc06baf13f5b992e6ed6967da932b44dc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:47:10 +0300 Subject: [PATCH 273/555] libstore: LocalStoreAccessor uses makeFSSourceAccessor This more honestly wraps the underlying FS accessor (we want to use the unix-specific dirfd-based one). --- src/libstore/local-fs-store.cc | 52 ++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 9f5864ee4425..12818f32a447 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,4 +1,3 @@ -#include "nix/util/posix-source-accessor.hh" #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" #include "nix/util/compression.hh" @@ -27,13 +26,14 @@ LocalFSStore::LocalFSStore(const Config & config) { } -struct LocalStoreAccessor : PosixSourceAccessor +struct LocalStoreAccessor : SourceAccessor { + ref accessor; ref store; bool requireValidPath; LocalStoreAccessor(ref store, bool requireValidPath) - : PosixSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()}) + : accessor(makeFSSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()})) , store(store) , requireValidPath(requireValidPath) { @@ -54,25 +54,61 @@ struct LocalStoreAccessor : PosixSourceAccessor return Stat{.type = tDirectory}; requireStoreObject(path); - return PosixSourceAccessor::maybeLstat(path); + return accessor->maybeLstat(path); + } + + Stat lstat(const CanonPath & path) override + { + /* Also allow `path` to point to the entire store, which is + needed for resolving symlinks. */ + if (path.isRoot()) + return Stat{.type = tDirectory}; + + requireStoreObject(path); + return accessor->lstat(path); } DirEntries readDirectory(const CanonPath & path) override { requireStoreObject(path); - return PosixSourceAccessor::readDirectory(path); + return accessor->readDirectory(path); } void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { requireStoreObject(path); - return PosixSourceAccessor::readFile(path, sink, sizeCallback); + return accessor->readFile(path, sink, sizeCallback); } std::string readLink(const CanonPath & path) override { requireStoreObject(path); - return PosixSourceAccessor::readLink(path); + return accessor->readLink(path); + } + + std::string showPath(const CanonPath & path) override + { + return accessor->showPath(path); + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return accessor->getPhysicalPath(path); + } + + std::pair> getFingerprint(const CanonPath & path) override + { + return accessor->getFingerprint(path); + } + + std::optional getLastModified() override + { + return accessor->getLastModified(); + } + + bool pathExists(const CanonPath & path) override + { + return accessor->pathExists(path); } }; @@ -96,7 +132,7 @@ std::shared_ptr LocalFSStore::getFSAccessor(const StorePath & pa if (!pathExists(absPath)) return nullptr; } - return std::make_shared(std::move(absPath)); + return makeFSSourceAccessor(std::move(absPath)); } const std::filesystem::path LocalFSStore::drvsLogDir = "drvs"; From 7847c5136fce2c5f7fed343b82eaebe493e5fdc6 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:54:36 +0300 Subject: [PATCH 274/555] libstore: Use requireStoreObjectAccessor in addToStore --- src/libstore/local-store.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b0c41c35ce65..b41856871bd2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1077,8 +1077,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF if (info.ca) { auto & specified = *info.ca; auto actualHash = ({ - auto accessor = getFSAccessor(false); - CanonPath path{info.path.to_string()}; + SourcePath sourcePath = requireStoreObjectAccessor(info.path, /*requireValidPath=*/false); Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++ auto fim = specified.method.getFileIngestionMethod(); switch (fim) { @@ -1088,12 +1087,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF specified.hash.algo, std::string{info.path.hashPart()}, }; - dumpPath({accessor, path}, caSink, (FileSerialisationMethod) fim); + dumpPath(sourcePath, caSink, (FileSerialisationMethod) fim); h = caSink.finish().hash; break; } case FileIngestionMethod::Git: - h = git::dumpHash(specified.hash.algo, {accessor, path}).hash; + h = git::dumpHash(specified.hash.algo, sourcePath).hash; break; } ContentAddress{ From 344ab0a5ee5b6fe2147047d7c449db20095d0297 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:09:10 +0300 Subject: [PATCH 275/555] libstore: Use makeFSStoreAccessor in derivation builder With the addition of the file descriptor based source accessor (that does caching) we now create a fresh instance of the accessor and the last path component is not followed, so the transformation is safe (actually tests fail without this because some directories get unlinked and result in ENOENT if the file descriptor gets cached). --- src/libstore/unix/build/derivation-builder.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 85aa98c7ae87..2df2b74bb105 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1713,12 +1713,11 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() HashModuloSink caSink{outputHash.hashAlgo, oldHashPart}; auto fim = outputHash.method.getFileIngestionMethod(); dumpPath( - {getFSSourceAccessor(), CanonPath(actualPath.native())}, caSink, (FileSerialisationMethod) fim); + {makeFSSourceAccessor(actualPath), CanonPath::root}, caSink, (FileSerialisationMethod) fim); return caSink.finish().hash; } case FileIngestionMethod::Git: { - return git::dumpHash(outputHash.hashAlgo, {getFSSourceAccessor(), CanonPath(actualPath.native())}) - .hash; + return git::dumpHash(outputHash.hashAlgo, {makeFSSourceAccessor(actualPath), CanonPath::root}).hash; } } assert(false); @@ -1740,7 +1739,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath.native())}, + {makeFSSourceAccessor(actualPath), CanonPath::root}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); newInfo0.narHash = narHashAndSize.hash; @@ -1783,7 +1782,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() std::string{scratchPath->hashPart()}, std::string{requiredFinalPath.hashPart()}); rewriteOutput(outputRewrites); HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath.native())}, + {makeFSSourceAccessor(actualPath), CanonPath::root}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); ValidPathInfo newInfo0{requiredFinalPath, {store, narHashAndSize.hash}}; From b16a5c365c45fce2f3a77632bff883a100c2c5dc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:27:31 +0300 Subject: [PATCH 276/555] nix: Use makeFSSourceAccessor in place of createAtRoot The API is the same and uses the unix-specific implementation. We can also drop all makeParentCanonical calls, because the function follows symlinks in the parents already when opening the dirFd/file. One slight wrinkle it that we first have to do absPath - this is fine. --- src/nix/add-to-store.cc | 2 +- src/nix/hash.cc | 4 +--- src/nix/nix-store/nix-store.cc | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 4f0fe722f197..09972a2890af 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -36,7 +36,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand if (!namePart) namePart = path.filename().string(); - auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); + auto sourcePath = makeFSSourceAccessor(absPath(path)); auto storePath = dryRun ? store->computeStorePath(*namePart, sourcePath, caMethod, hashAlgo, {}).first : store->addToStoreSlow(*namePart, sourcePath, caMethod, hashAlgo, {}).path; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 90f44fdecd26..9cd0592073e9 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -84,9 +84,7 @@ struct CmdHashBase : Command return std::make_unique(hashAlgo); }; - auto makeSourcePath = [&]() -> SourcePath { - return PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); - }; + auto makeSourcePath = [&]() -> SourcePath { return makeFSSourceAccessor(absPath(path)); }; Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++ switch (mode) { diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index c21625e0409c..15a4e878f5ac 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -191,7 +191,7 @@ static void opAdd(Strings opFlags, Strings opArgs) throw UsageError("unknown flag"); for (auto & i : opArgs) { - auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); + auto sourcePath = makeFSSourceAccessor(absPath(i)); std::cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), sourcePath))); } } @@ -215,7 +215,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) { - auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); + auto sourcePath = makeFSSourceAccessor(absPath(i)); std::cout << fmt( "%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), sourcePath, method, hashAlgo).path)); } From a8bcc083a270f86334b5ef0268ddc56a23235713 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:34:47 +0300 Subject: [PATCH 277/555] libutil: Use makeFSSourceAccessor in dumpPath Unlike previously, we now follow symlinks in parents (not the last path component), but this is secure and what versions like 2.18 did. --- src/libutil/archive.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index f704acdf78b4..6188860ce13e 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -101,14 +101,15 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & time_t dumpPathAndGetMtime(const std::filesystem::path & path, Sink & sink, PathFilter & filter) { - auto path2 = PosixSourceAccessor::createAtRoot(path, /*trackLastModified=*/true); + SourcePath path2 = makeFSSourceAccessor(absPath(path), /*trackLastModified=*/true); path2.dumpPath(sink, filter); return path2.accessor->getLastModified().value(); } void dumpPath(const std::filesystem::path & path, Sink & sink, PathFilter & filter) { - dumpPathAndGetMtime(path, sink, filter); + SourcePath path2 = makeFSSourceAccessor(absPath(path), /*trackLastModified=*/false); + path2.dumpPath(sink, filter); } void dumpString(std::string_view s, Sink & sink) From c349fbf4c835b2eea7ab8c266f1c02d829a83029 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 02:26:06 +0300 Subject: [PATCH 278/555] libutil: Add a new overload of readDirectory (for fd-relative operations) and use in recursive traversal This ensures race-free traversal throughout. --- src/libutil/archive.cc | 34 +++++++++++-------- src/libutil/fs-sink.cc | 10 +++--- .../include/nix/util/source-accessor.hh | 16 +++++++++ src/libutil/posix-source-accessor.cc | 27 ++++++++++++++- 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6188860ce13e..91af4b9e7f54 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -34,10 +34,10 @@ PathFilter defaultPathFilter = [](const std::string &) { return true; }; void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter) { - auto dumpContents = [&](const CanonPath & path) { + auto dumpContents = [&sink](SourceAccessor & accessor, const CanonPath & path) { sink << "contents"; std::optional size; - readFile(path, sink, [&](uint64_t _size) { + accessor.readFile(path, sink, [&](uint64_t _size) { size = _size; sink << _size; }); @@ -47,10 +47,14 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & sink << narVersionMagic1; - [&, &this_(*this)](this const auto & dump, const CanonPath & path) -> void { + [&sink, &filter, &dumpContents]( + this const auto & dump, + SourceAccessor & accessor, + const CanonPath & path, + const CanonPath & filterPath) -> void { checkInterrupt(); - auto st = this_.lstat(path); + auto st = accessor.lstat(path); sink << "("; @@ -58,7 +62,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & sink << "type" << "regular"; if (st.isExecutable) sink << "executable" << ""; - dumpContents(path); + dumpContents(accessor, path); } else if (st.type == tDirectory) { @@ -67,7 +71,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & /* If we're on a case-insensitive system like macOS, undo the case hack applied by restorePath(). */ StringMap unhacked; - for (auto & i : this_.readDirectory(path)) + for (auto & i : accessor.readDirectory(path)) if (archiveSettings.useCaseHack) { std::string name(i.first); size_t pos = i.first.find(caseHackSuffix); @@ -81,22 +85,24 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & } else unhacked.emplace(i.first, i.first); - for (auto & i : unhacked) - if (filter((path / i.first).abs())) { - sink << "entry" << "(" << "name" << i.first << "node"; - dump(path / i.second); - sink << ")"; - } + accessor.readDirectory(path, [&](SourceAccessor & subdirAccessor, const CanonPath & subdirRelPath) { + for (auto & i : unhacked) + if (filter((filterPath / i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(subdirAccessor, subdirRelPath / i.second, filterPath / i.second); + sink << ")"; + } + }); } else if (st.type == tSymlink) - sink << "type" << "symlink" << "target" << this_.readLink(path); + sink << "type" << "symlink" << "target" << accessor.readLink(path); else throw Error("file '%s' has an unsupported type", path); sink << ")"; - }(path); + }(*this, path, path); } time_t dumpPathAndGetMtime(const std::filesystem::path & path, Sink & sink, PathFilter & filter) diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index e41d153e92db..deb8ec3c8f7b 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -34,10 +34,12 @@ void copyRecursive(SourceAccessor & accessor, const CanonPath & from, FileSystem } case SourceAccessor::tDirectory: { - sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPath) { - for (auto & [name, _] : accessor.readDirectory(from)) { - copyRecursive(accessor, from / name, dirSink, relDirPath / name); - } + sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPathTo) { + accessor.readDirectory(from, [&](SourceAccessor & subdirAccessor, const CanonPath & relDirPathFrom) { + for (auto & [name, _] : subdirAccessor.readDirectory(relDirPathFrom)) { + copyRecursive(subdirAccessor, relDirPathFrom / name, dirSink, relDirPathTo / name); + } + }); }); break; } diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index c458cf8b5c35..737383aba282 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -138,6 +138,22 @@ struct SourceAccessor : std::enable_shared_from_this */ virtual DirEntries readDirectory(const CanonPath & path) = 0; + /** + * Variation of readDirectory that receives a SourceAccessor possibly scoped to \ref dirPath. + * Primary meant for recursive traversal functions that would benefit from *at-style syscalls + * relative to a particular directory. + * + * @note Like `readFile`, this method should *not* follow symlinks. + * @param callback Caller-provided function invoked with a maximally deeply scoped SourceAccessor and the path that + * would have to be prepended to each path relative to dirPath to access a particular file with it. + */ + virtual void readDirectory( + const CanonPath & dirPath, + std::function callback) + { + callback(*this, dirPath); + } + virtual std::string readLink(const CanonPath & path) = 0; virtual void dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter = defaultPathFilter); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index bd28f81fa594..6a69a16128df 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -216,6 +216,8 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase std::function makeDirFdCallback(); + AutoCloseFD openSubdirectory(const CanonPath & path); + public: PosixDirectorySourceAccessor( AutoCloseFD fd, std::filesystem::path path, bool trackLastModified, unsigned dirFdCacheSize) @@ -249,6 +251,10 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase DirEntries readDirectory(const CanonPath & path) override; + void readDirectory( + const CanonPath & dirPath, + std::function callback) override; + std::string readLink(const CanonPath & path) override; std::optional getPhysicalPath(const CanonPath & path) override @@ -359,7 +365,7 @@ try { throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } -SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) +AutoCloseFD PosixDirectorySourceAccessor::openSubdirectory(const CanonPath & path) try { AutoCloseFD dirFdOwning; @@ -379,6 +385,14 @@ try { } } + return dirFdOwning; +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) +try { + AutoCloseFD dirFdOwning = openSubdirectory(path); AutoCloseDir dir(::fdopendir(dirFdOwning.get())); if (!dir) throw SysError("reading directory '%s'", showPath(path)); @@ -431,6 +445,17 @@ try { throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } +void PosixDirectorySourceAccessor::readDirectory( + const CanonPath & dirPath, + std::function callback) +{ + auto fd = openSubdirectory(dirPath); + PosixDirectorySourceAccessor accessor{ + std::move(fd), fsPath / dirPath.rel(), trackLastModified, /*dirFdCacheSize=*/0}; + callback(accessor, CanonPath::root); + PosixSourceAccessorBase::maybeUpdateMtime(accessor.mtime); +} + std::string PosixDirectorySourceAccessor::readLink(const CanonPath & path) try { if (path.isRoot()) From ba7db4e05635e351201327db0a5bbbd85ca4a07a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:38:08 +0300 Subject: [PATCH 279/555] nix-perl: Get rid of the last occurence of createAtRoot --- src/perl/lib/Nix/Store.xs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index b0f70be3a9e8..8b28b0e5397c 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -259,7 +259,7 @@ hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath( - PosixSourceAccessor::createAtRoot(path), + makeFSSourceAccessor(absPath(path)), FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first; auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); @@ -339,7 +339,7 @@ StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat; auto path = THIS->store->addToStore( std::string(baseNameOf(srcPath)), - PosixSourceAccessor::createAtRoot(srcPath), + makeFSSourceAccessor(absPath(srcPath)), method, parseHashAlgo(algo)); XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); } catch (Error & e) { From 316e33a14834de36cd0792c021e00fddc260e3ba Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:39:14 +0300 Subject: [PATCH 280/555] libutil: Get rid of PosixSourceAccessor::createAtRoot --- .../include/nix/util/posix-source-accessor.hh | 30 ------------------- src/libutil/posix-source-accessor.cc | 10 ------- 2 files changed, 40 deletions(-) diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index d93b68e0afea..0da7dcdea858 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -76,36 +76,6 @@ public: std::optional getPhysicalPath(const CanonPath & path) override; - /** - * Create a `PosixSourceAccessor` and `SourcePath` corresponding to - * some native path. - * - * @param Whether the accessor should return a non-null getLastModified. - * When true the accessor must be used only by a single thread. - * - * The `PosixSourceAccessor` is rooted as far up the tree as - * possible, (e.g. on Windows it could scoped to a drive like - * `C:\`). This allows more `..` parent accessing to work. - * - * @note When `path` is trusted user input, canonicalize it using - * `std::filesystem::canonical`, `makeParentCanonical`, `std::filesystem::weakly_canonical`, etc, - * as appropriate for the use case. At least weak canonicalization is - * required for the `SourcePath` to do anything useful at the location it - * points to. - * - * @note A canonicalizing behavior is not built in `createAtRoot` so that - * callers do not accidentally introduce symlink-related security vulnerabilities. - * Furthermore, `createAtRoot` does not know whether the file pointed to by - * `path` should be resolved if it is itself a symlink. In other words, - * `createAtRoot` can not decide between aforementioned `canonical`, `makeParentCanonical`, etc. for its callers. - * - * See - * [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path) - * and - * [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path). - */ - static SourcePath createAtRoot(const std::filesystem::path & path, bool trackLastModified = false); - void invalidateCache(const CanonPath & path) override; private: diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 6a69a16128df..affad4fbbe1f 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -3,7 +3,6 @@ #include "nix/util/lru-cache.hh" #include "nix/util/sync.hh" #include "nix/util/memory-source-accessor.hh" -#include "nix/util/source-path.hh" #include "nix/util/signals.hh" #include @@ -501,15 +500,6 @@ PosixSourceAccessor::PosixSourceAccessor() { } -SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path, bool trackLastModified) -{ - std::filesystem::path path2 = absPath(path); - return { - make_ref(path2.root_path(), trackLastModified), - CanonPath{path2.relative_path().string()}, - }; -} - std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) { return root.empty() ? (std::filesystem::path{path.abs()}) From 52011de1b22206b080dc935b18a781a53417eeb0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 02:49:31 +0300 Subject: [PATCH 281/555] libstore: Use copyRecursive when copying FOD outputs This completely bypasses coroutines and serialisation/deserialisation overhead and could start using reflinking in the future too. --- src/libstore/unix/build/derivation-builder.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 2df2b74bb105..73eff4dfd6c7 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1761,8 +1761,10 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() any stale writable file descriptors. Copy through the serialisation/deserialisation. TODO: Use copyRecursive here and make use of reflinking. */ - auto source = sinkToSource([&](Sink & nextSink) { dumpPath(actualPath, nextSink); }); - restorePath(tmpOutput, *source, store.config->getLocalSettings().fsyncStorePaths); + auto pathAccessor = makeFSSourceAccessor(actualPath); + RestoreSink restoreSink{store.config->getLocalSettings().fsyncStorePaths}; + restoreSink.dstPath = tmpOutput; + copyRecursive(*pathAccessor, CanonPath::root, restoreSink, CanonPath::root); /* This makes it slightly harder to make sense of the control flow. The rule of thumb is that actualPath points to the current location of the stuff that we'll end up registering. */ From 8e7702d7365ef5b88b0498949d8ee2a5109f44ea Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:02:49 +0300 Subject: [PATCH 282/555] makeFSSourceAccessor: add finalSymlink parameter Useful to avoid costly resolveSymlinks() when possible. --- src/libutil/include/nix/util/source-accessor.hh | 3 ++- src/libutil/posix-source-accessor.cc | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 737383aba282..92bd5d30fae2 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -278,7 +278,8 @@ ref getFSSourceAccessor(); * * Symlinks in parents of `root` are resolved. Final symlink is not. */ -ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified = false); +ref makeFSSourceAccessor( + std::filesystem::path root, bool trackLastModified = false, FinalSymlink finalSymlink = FinalSymlink::DontFollow); /** * Construct an accessor that presents a "union" view of a vector of diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index affad4fbbe1f..a8046c5c1704 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -661,14 +661,14 @@ ref getFSSourceAccessor() return rootFS; } -ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified) +ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified, FinalSymlink finalSymlink) { #ifndef _WIN32 assert(root.is_absolute()); - AutoCloseFD fd = openFileReadonly(root, FinalSymlink::DontFollow); + AutoCloseFD fd = openFileReadonly(root, finalSymlink); if (!fd) { - if (errno != ELOOP) + if (finalSymlink == FinalSymlink::Follow || errno != ELOOP) throw NativeSysError("opening file %1%", PathFmt(root)); /* A helper class that holds the symlink destination in memory. */ From c296e254b133c9c6e97c47d85fb43c5ef957b7f4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:39:39 +0300 Subject: [PATCH 283/555] PosixDirectorySourceAccessor: Improve dirFd caching --- src/libutil/posix-source-accessor.cc | 37 ++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index a8046c5c1704..5dd4f56dd33e 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -293,15 +293,42 @@ std::pair> PosixDirectorySourceAccessor maybeEvictFromGlobalCaches(); + std::shared_ptr intermediateParentFd; + CanonPath anchor = CanonPath::root; + if (dirFdCache) { - if (auto cachedFd = dirFdCache->lock()->get(parent)) { - assert((*cachedFd)->get()); - return {(*cachedFd)->get(), *cachedFd}; + auto cache = dirFdCache->lock(); + auto p = parent; + while (true) { + if (auto intermediateDirFdHit = cache->get(p)) { + if (p == parent) + return {(*intermediateDirFdHit)->get(), *intermediateDirFdHit}; + intermediateParentFd = intermediateDirFdHit->get_ptr(); + anchor = p; + break; + } + if (p.isRoot()) + break; + p.pop(); + } + } + + Descriptor startFd = intermediateParentFd ? intermediateParentFd->get() : dirFd.get(); + CanonPath relPath = intermediateParentFd ? parent.removePrefix(anchor) : parent; + + std::function cb; + if (auto base = makeDirFdCallback()) { + if (intermediateParentFd) { + cb = [base = std::move(base), prefix = anchor](AutoCloseFD fd, CanonPath relKey) { + base(std::move(fd), prefix / relKey); + }; + } else { + cb = std::move(base); } } - AutoCloseFD parentFdOwning = openFileEnsureBeneathNoSymlinks( - dirFd.get(), parent, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, makeDirFdCallback()); + AutoCloseFD parentFdOwning = + openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; } From 043cafab68d9e87de0e35a35f3abd8426ec5094b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:50:36 +0300 Subject: [PATCH 284/555] SourceAccessor: remove invalidateCache, remove raw usage of PosixSourceAccessor PosixSourceAccessor is going to be renamed to WindowsSourceAccessor in the following commit and hopefully deleted soon. The lstat cache is finally removed from the unix case too. --- src/libfetchers/filtering-source-accessor.cc | 5 ----- .../include/nix/fetchers/filtering-source-accessor.hh | 2 -- src/libflake/flake.cc | 2 -- src/libstore/optimise-store.cc | 8 +------- src/libutil/include/nix/util/posix-source-accessor.hh | 2 -- src/libutil/include/nix/util/source-path.hh | 5 ----- src/libutil/mounted-source-accessor.cc | 6 ------ src/libutil/posix-source-accessor.cc | 5 ----- src/libutil/union-source-accessor.cc | 6 ------ 9 files changed, 1 insertion(+), 40 deletions(-) diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 6fe7d2504ec3..b8455710fa96 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -62,11 +62,6 @@ std::pair> FilteringSourceAccessor::getFin return next->getFingerprint(prefix / path); } -void FilteringSourceAccessor::invalidateCache(const CanonPath & path) -{ - next->invalidateCache(prefix / path); -} - void FilteringSourceAccessor::checkAccess(const CanonPath & path) { if (!isAllowed(path)) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 13272719fe3d..b259e8ef8f0d 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -53,8 +53,6 @@ struct FilteringSourceAccessor : SourceAccessor std::pair> getFingerprint(const CanonPath & path) override; - void invalidateCache(const CanonPath & path) override; - /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` * exception if `isAllowed()` returns `false` for `path`. diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 45871cff87ee..e8dbf42f5451 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -863,8 +863,6 @@ LockedFlake lockFlake( CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), newLockFileS, commitMessage); - - flake.lockFilePath().invalidateCache(); } /* Rewriting the lockfile changed the top-level diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 220e37fe9efe..15f2b1f3d137 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -153,13 +153,7 @@ void LocalStore::optimisePath_( Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = ({ - hashPath( - {make_ref(), CanonPath(path.string())}, - FileSerialisationMethod::NixArchive, - HashAlgorithm::SHA256) - .hash; - }); + Hash hash = hashPath(makeFSSourceAccessor(path), FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).hash; debug("%s has hash '%s'", PathFmt(path), hash.to_string(HashFormat::Nix32, true)); /* Check if this is a known hash. */ diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index 0da7dcdea858..c8c54953eb81 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -76,8 +76,6 @@ public: std::optional getPhysicalPath(const CanonPath & path) override; - void invalidateCache(const CanonPath & path) override; - private: /** diff --git a/src/libutil/include/nix/util/source-path.hh b/src/libutil/include/nix/util/source-path.hh index 24932e4cddbe..f15b44fdd437 100644 --- a/src/libutil/include/nix/util/source-path.hh +++ b/src/libutil/include/nix/util/source-path.hh @@ -114,11 +114,6 @@ struct SourcePath return {accessor, accessor->resolveSymlinks(path, mode)}; } - void invalidateCache() const - { - accessor->invalidateCache(path); - } - friend class std::hash; }; diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index aab95f775b77..84840352b37a 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -99,12 +99,6 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor auto [accessor, subpath] = resolve(path); return accessor->getFingerprint(subpath); } - - void invalidateCache(const CanonPath & path) override - { - auto [accessor, subpath] = resolve(path); - accessor->invalidateCache(subpath); - } }; ref makeMountedSourceAccessor(std::map> mounts) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 5dd4f56dd33e..e3850d1b6db7 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -591,11 +591,6 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path return st; } -void PosixSourceAccessor::invalidateCache(const CanonPath & path) -{ - cache.erase(makeAbsPath(path).string()); -} - std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) { if (auto parent = path.parent()) diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index de50c75e3733..da71903e6adf 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -90,12 +90,6 @@ struct UnionSourceAccessor : SourceAccessor } return {path, std::nullopt}; } - - void invalidateCache(const CanonPath & path) override - { - for (auto & accessor : accessors) - accessor->invalidateCache(path); - } }; ref makeUnionSourceAccessor(std::vector> && accessors) From 5aa60ea0229ec093830c33b2c05064f6fb1a2be1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:55:55 +0300 Subject: [PATCH 285/555] Rename PosixSourceAccessor to WindowsSourceAccessor This implementations sucks and really should just be removed. The assertNoSymlinks check is also useless on windows. We might as well just remove it. --- .../include/nix/util/posix-source-accessor.hh | 43 ---------- src/libutil/posix-source-accessor.cc | 86 ++++++++++++++----- 2 files changed, 63 insertions(+), 66 deletions(-) diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index c8c54953eb81..686c66471cdb 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -45,47 +45,4 @@ protected: } // namespace detail -/** - * A source accessor that uses the Unix filesystem. - */ -class PosixSourceAccessor : public detail::PosixSourceAccessorBase -{ - /** - * Optional root path to prefix all operations into the native file - * system. This allows prepending funny things like `C:\` that - * `CanonPath` intentionally doesn't support. - */ - const std::filesystem::path root; - -public: - - PosixSourceAccessor(); - PosixSourceAccessor(std::filesystem::path && root, bool trackLastModified = false); - - void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; - - using SourceAccessor::readFile; - - bool pathExists(const CanonPath & path) override; - - std::optional maybeLstat(const CanonPath & path) override; - - DirEntries readDirectory(const CanonPath & path) override; - - std::string readLink(const CanonPath & path) override; - - std::optional getPhysicalPath(const CanonPath & path) override; - -private: - - /** - * Throw an error if `path` or any of its ancestors are symlinks. - */ - void assertNoSymlinks(CanonPath path); - - std::optional cachedLstat(const CanonPath & path); - - std::filesystem::path makeAbsPath(const CanonPath & path); -}; - } // namespace nix diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index e3850d1b6db7..d5dc7be282d0 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -510,11 +510,53 @@ try { throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } -#endif +#else -} // namespace +/** + * A source accessor that uses the Windows filesystem. + * @todo Should be moved into a separate file. + */ +class WindowsSourceAccessor : public detail::PosixSourceAccessorBase +{ + /** + * Optional root path to prefix all operations into the native file + * system. This allows prepending funny things like `C:\` that + * `CanonPath` intentionally doesn't support. + */ + const std::filesystem::path root; + +public: + + WindowsSourceAccessor(); + WindowsSourceAccessor(std::filesystem::path && root, bool trackLastModified = false); + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; + + using SourceAccessor::readFile; -PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; + +private: + + /** + * Throw an error if `path` or any of its ancestors are symlinks. + */ + void assertNoSymlinks(CanonPath path); + + std::optional cachedLstat(const CanonPath & path); + + std::filesystem::path makeAbsPath(const CanonPath & path); +}; + +WindowsSourceAccessor::WindowsSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) : PosixSourceAccessorBase(trackLastModified) , root(std::move(argRoot)) { @@ -522,12 +564,12 @@ PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool displayPrefix = root.string(); } -PosixSourceAccessor::PosixSourceAccessor() - : PosixSourceAccessor(std::filesystem::path{}) +WindowsSourceAccessor::WindowsSourceAccessor() + : WindowsSourceAccessor(std::filesystem::path{}) { } -std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) +std::filesystem::path WindowsSourceAccessor::makeAbsPath(const CanonPath & path) { return root.empty() ? (std::filesystem::path{path.abs()}) : path.isRoot() ? /* Don't append a slash for the root of the accessor, since @@ -537,19 +579,13 @@ std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) : root / path.rel(); } -void PosixSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) +void WindowsSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) { assertNoSymlinks(path); auto ap = makeAbsPath(path); - AutoCloseFD fd = toDescriptor(open( - ap.string().c_str(), - O_RDONLY -#ifndef _WIN32 - | O_NOFOLLOW | O_CLOEXEC -#endif - )); + AutoCloseFD fd = toDescriptor(open(ap.string().c_str(), O_RDONLY)); if (!fd) throw SysError("opening file '%1%'", ap.string()); @@ -563,7 +599,7 @@ void PosixSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun>; static Cache cache; -std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) +std::optional WindowsSourceAccessor::cachedLstat(const CanonPath & path) { // Note: we convert std::filesystem::path to std::string because the // former is not hashable on libc++. @@ -591,7 +627,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path return st; } -std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) +std::optional WindowsSourceAccessor::maybeLstat(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); @@ -603,7 +639,7 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP return sourceAccessorStatFromPosixStat(*st); } -SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +SourceAccessor::DirEntries WindowsSourceAccessor::readDirectory(const CanonPath & path) { assertNoSymlinks(path); DirEntries res; @@ -655,19 +691,19 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & return res; } -std::string PosixSourceAccessor::readLink(const CanonPath & path) +std::string WindowsSourceAccessor::readLink(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); return nix::readLink(makeAbsPath(path)).string(); } -std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +std::optional WindowsSourceAccessor::getPhysicalPath(const CanonPath & path) { return makeAbsPath(path); } -void PosixSourceAccessor::assertNoSymlinks(CanonPath path) +void WindowsSourceAccessor::assertNoSymlinks(CanonPath path) { while (!path.isRoot()) { auto st = cachedLstat(path); @@ -677,6 +713,10 @@ void PosixSourceAccessor::assertNoSymlinks(CanonPath path) } } +#endif + +} // namespace + ref getFSSourceAccessor() { static auto rootFS = makeFSSourceAccessor("/", /*trackLastModified=*/false); @@ -768,9 +808,9 @@ ref makeFSSourceAccessor(std::filesystem::path root, bool trackL else throw Error("file %1% has an unsupported type", PathFmt(root)); +#else + return make_ref(std::move(root), trackLastModified); #endif - - return make_ref(std::move(root), trackLastModified); } } // namespace nix From 5450d99984d6f000964d4589f288ee6cb37a4551 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 04:44:31 +0300 Subject: [PATCH 286/555] tests/functional/multiple-output: Move invalid outtput name tests above nuking the store We now more robustly require the existence of the store directory. These tests have just been tacked on at the end of the test, even though it nukes the store right before them. --- tests/functional/multiple-outputs.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/functional/multiple-outputs.sh b/tests/functional/multiple-outputs.sh index f703fb02be63..dec22ff817f3 100755 --- a/tests/functional/multiple-outputs.sh +++ b/tests/functional/multiple-outputs.sh @@ -96,6 +96,13 @@ if nix-build multiple-outputs.nix -A cyclic --no-out-link; then exit 1 fi +# TODO inspect why this doesn't work with floating content-addressing +# derivations. +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + expect 1 nix build -f multiple-outputs.nix invalid-output-name-1 2>&1 | grep 'contains illegal character' + expect 1 nix build -f multiple-outputs.nix invalid-output-name-2 2>&1 | grep 'contains illegal character' +fi + # Do a GC. This should leave an empty store. echo "collecting garbage..." rm "$TEST_ROOT"/result* @@ -103,10 +110,3 @@ nix-store --gc --keep-derivations --keep-outputs nix-store --gc --print-roots rm -rf "$NIX_STORE_DIR"/.links rmdir "$NIX_STORE_DIR" - -# TODO inspect why this doesn't work with floating content-addressing -# derivations. -if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then - expect 1 nix build -f multiple-outputs.nix invalid-output-name-1 2>&1 | grep 'contains illegal character' - expect 1 nix build -f multiple-outputs.nix invalid-output-name-2 2>&1 | grep 'contains illegal character' -fi From 0214ee7abbf7d0bbadbd7de9d2f6aa887328370f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 21 Apr 2026 12:41:05 +0200 Subject: [PATCH 287/555] repl: Make :reload robust against load failures Previously a failing :l/:lf was recorded before evaluation, so a typo'd path or broken flake ref would be retried on every :reload. Worse, :reload cleared the file/flake lists up front and rebuilt them while iterating, so the first error aborted the loop and silently dropped all later entries (and skipped CLI installables and flakes entirely). Record loaded files/flakes only after they actually load, and during reload catch errors per entry so one broken file no longer wipes out the rest of the session. --- src/libcmd/repl.cc | 42 ++++++++++++++++++++++++++++------------ tests/functional/repl.sh | 24 +++++++++++++++++++++++ 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 777fc33256ca..f7833ea3a478 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -688,12 +688,13 @@ ProcessLineResult NixRepl::processLine(std::string line) void NixRepl::loadFile(const std::filesystem::path & path) { - loadedFiles.remove(path); - loadedFiles.push_back(path); Value v, v2; state->evalFile(lookupFileArg(*state, path.string()), v); state->autoCallFunction(*autoArgs, v, v2); addAttrsToScope(v2); + // Remember for :reload only on success. + loadedFiles.remove(path); + loadedFiles.push_back(path); } void NixRepl::loadFlake(const std::string & flakeRefS) @@ -701,9 +702,6 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - loadedFlakes.remove(flakeRefS); - loadedFlakes.push_back(flakeRefS); - std::filesystem::path cwd; try { cwd = std::filesystem::current_path(); @@ -730,6 +728,10 @@ void NixRepl::loadFlake(const std::string & flakeRefS) }), v); addAttrsToScope(v); + + // Remember for :reload only on success. + loadedFlakes.remove(flakeRefS); + loadedFlakes.push_back(flakeRefS); } void NixRepl::initEnv() @@ -772,28 +774,44 @@ void NixRepl::reloadFilesAndFlakes() void NixRepl::loadFiles() { - decltype(loadedFiles) old = loadedFiles; - loadedFiles.clear(); + // loadFile() rebuilds loadedFiles; keep failed entries and continue. + decltype(loadedFiles) old; + std::swap(old, loadedFiles); for (auto & i : old) { notice("Loading %1%...", PathFmt(i)); - loadFile(i); + try { + loadFile(i); + } catch (Error & e) { + loadedFiles.push_back(i); + printMsg(lvlError, e.msg()); + } } for (auto & [i, what] : getValues()) { notice("Loading installable '%1%'...", what); - addAttrsToScope(*i); + try { + addAttrsToScope(*i); + } catch (Error & e) { + printMsg(lvlError, e.msg()); + } } } void NixRepl::loadFlakes() { - Strings old = loadedFlakes; - loadedFlakes.clear(); + // See loadFiles(). + Strings old; + std::swap(old, loadedFlakes); for (auto & i : old) { notice("Loading flake '%1%'...", i); - loadFlake(i); + try { + loadFlake(i); + } catch (Error & e) { + loadedFlakes.push_back(i); + printMsg(lvlError, e.msg()); + } } } diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 9fc803d092e0..0dbc047ed827 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -281,6 +281,30 @@ exec 3>&- # Close fifo wait $repl_pid # Wait for process to finish grep -q "afterChange" repl_output +# Regression: a failed `:l` / `:lf` must not be remembered for `:reload`, +# and an error in one loaded file must not drop later ones from the reload list. +cat > reloadA.nix < reloadB.nix < Date: Tue, 21 Apr 2026 12:49:46 +0200 Subject: [PATCH 288/555] release: name artifact jobsets maintenance-X.Y-release Suffixing the existing maintenance-X.Y identifier keeps the full-CI and release-artifact jobsets adjacent in Hydra's alphabetical list and lets tooling derive one name from the other, whereas a parallel release-X.Y namespace scatters the pair and leaves no obvious slot for master. Teach upload-release.pl to read the git revision from the legacy `src` input so it works against evaluations of the new jobset, and trim buildCross in release-jobs.nix to the two targets fallback-paths.nix actually consumes so the artifact jobset stays minimal. --- .github/workflows/upload-release.yml | 2 +- maintainers/release-process.md | 28 ++++++++++++++++------------ maintainers/upload-release.pl | 8 +++++++- packaging/release-jobs.nix | 9 ++++++++- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml index cd21336c8913..f00dce4a5c6b 100644 --- a/.github/workflows/upload-release.yml +++ b/.github/workflows/upload-release.yml @@ -3,7 +3,7 @@ on: workflow_dispatch: inputs: eval_id: - description: "Hydra evaluation ID" + description: "Hydra evaluation ID (from the maintenance-X.Y-release jobset)" required: true type: number is_latest: diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 19e5e050534e..aff0088a0d93 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -84,24 +84,27 @@ release: * Create two jobsets for the release branch on Hydra: `maintenance-$VERSION` runs the full `hydraJobs` CI matrix. - `release-$VERSION` builds only the artifacts consumed by + `maintenance-$VERSION-release` builds only the artifacts consumed by `upload-release`, so a release can be cut without waiting on the full - matrix. + matrix. The `-release` suffix keeps the pair adjacent in Hydra's + alphabetical jobset list and lets scripts derive one name from the + other. * Clone the previous `maintenance-*` jobset, set identifier `maintenance-$VERSION`, description `$VERSION release branch`, flake URL `github:NixOS/nix/$VERSION-maintenance`. - * Clone the previous `release-*` jobset (or create a new **legacy** - jobset), set identifier `release-$VERSION`, description `$VERSION - release artifacts`, Nix expression `packaging/release-jobs.nix` in - input `src`, and add input `src` of type *Git checkout* pointing at + * Clone the previous `maintenance-*-release` jobset (or create a new + **legacy** jobset), set identifier `maintenance-$VERSION-release`, + description `$VERSION release artifacts`, Nix expression + `packaging/release-jobs.nix` in input `src`, and add input `src` of + type *Git checkout* pointing at `https://github.com/NixOS/nix $VERSION-maintenance`. -* Wait for the `release-$VERSION` jobset to evaluate and build. If - impatient, go to the evaluation and select `Actions -> Bump builds to - front of queue`. The aggregate job `release` turns green once every - required artifact is available. +* Wait for the `maintenance-$VERSION-release` jobset to evaluate and + build. If impatient, go to the evaluation and select `Actions -> Bump + builds to front of queue`. The aggregate job `release` turns green + once every required artifact is available. * When the release jobset evaluation has succeeded building, take note of the evaluation ID (e.g. `1780832` in @@ -177,8 +180,9 @@ release: $ git push ``` -* Wait for the desired evaluation of the `release-$VERSION` jobset to - finish building (the `release` aggregate job is the gating signal). +* Wait for the desired evaluation of the `maintenance-XX.YY-release` + jobset to finish building (the `release` aggregate job is the gating + signal). * Tag the release diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 06678553e712..b618bd900d4e 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -64,7 +64,13 @@ sub fetch { #print Dumper($evalInfo); my $flakeUrl = $evalInfo->{flake}; my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; -my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; +# Flake jobsets (`maintenance-X.Y`) expose the rev via the flake URL. +# The release-artifacts jobset (`maintenance-X.Y-release`) is a legacy +# jobset whose checkout is passed in as input `src`. +my $nixRev = ($flakeInfo + ? $flakeInfo->{revision} + : $evalInfo->{jobsetevalinputs}->{src}->{revision} + // $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix-everything.x86_64-linux", 'application/json')); #print Dumper($buildInfo); diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix index 23add3905e33..f82c23c9c1ac 100644 --- a/packaging/release-jobs.nix +++ b/packaging/release-jobs.nix @@ -8,6 +8,7 @@ # and share builds through the binary cache. # # Hydra jobset configuration: +# Identifier: maintenance--release # Type: Legacy # Nix expression: packaging/release-jobs.nix in input `src` # Inputs: @@ -36,7 +37,13 @@ let # `fallback-paths.nix` and (on x86_64-linux) the rendered manual via # its `doc` output. build.nix-everything = hydraJobs.build.nix-everything; - buildCross.nix-everything = hydraJobs.buildCross.nix-everything; + buildCross.nix-everything = { + # Only the cross targets that end up in `fallback-paths.nix`. + inherit (hydraJobs.buildCross.nix-everything) + riscv64-unknown-linux-gnu + x86_64-unknown-freebsd + ; + }; inherit (hydraJobs) manual From 50bce0b6de7cf1e6603616c8e1ea3b14b6ed05f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 21 Apr 2026 13:21:45 +0200 Subject: [PATCH 289/555] repl: Invalidate git workdir-info cache on :reload Since 7ba933e989b9, GitRepo::getCachedWorkdirInfo() memoises the workdir status (dirty flag, tracked files, head rev) in a process-global static to avoid repeated libgit2 status walks within a single command. In a long-lived `nix repl` session this cache outlives the underlying work tree: after the first `:load-flake` of a clean git checkout, every `:reload` keeps seeing the original head rev, so the source is served from the same fingerprinted store path even after the user edits files. Hook the cache invalidation into InputCache::clear(), which already expresses the "per evaluation" lifetime that resetFileCache() flushes on `:reload`, so reloading a git-backed flake (whether via `:lf` or as a CLI installable) now picks up uncommitted changes again. --- src/libfetchers/git-utils.cc | 9 +++++- .../include/nix/fetchers/git-utils.hh | 3 ++ src/libfetchers/input-cache.cc | 5 +++ tests/functional/repl.sh | 32 +++++++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 6bc474395af8..03a7783c1874 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1502,9 +1502,11 @@ ref Settings::getTarballCache() const } // namespace fetchers +static Sync> workdirInfoCache_; + GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) { - static Sync> _cache; + auto & _cache = workdirInfoCache_; { auto cache(_cache.lock()); auto i = cache->find(path); @@ -1516,6 +1518,11 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & return workdirInfo; } +void GitRepo::invalidateWorkdirInfoCache() +{ + workdirInfoCache_.lock()->clear(); +} + bool isLegalRefName(const std::string & refName) { initLibGit2(); diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 725fbf398410..871bf5e43f56 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -88,6 +88,9 @@ struct GitRepo static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path); + /* Drop all entries from the getCachedWorkdirInfo() cache. */ + static void invalidateWorkdirInfoCache(); + /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 85a611355e2f..3fe96d8503bc 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -1,4 +1,5 @@ #include "nix/fetchers/input-cache.hh" +#include "nix/fetchers/git-utils.hh" #include "nix/fetchers/registry.hh" #include "nix/util/sync.hh" @@ -65,6 +66,10 @@ struct InputCacheImpl : InputCache void clear() override { cache_.lock()->clear(); + /* The workdir info cache has the same "per evaluation" lifetime + as the input cache, so flush it here as well so that e.g. + `:reload` in `nix repl` picks up changes in git work trees. */ + GitRepo::invalidateWorkdirInfoCache(); } }; diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 0dbc047ed827..88d6e91cd90f 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -281,6 +281,38 @@ exec 3>&- # Close fifo wait $repl_pid # Wait for process to finish grep -q "afterChange" repl_output +# Regression: `:reload` on a flake loaded from a *git* work tree must pick up +# uncommitted changes. Guards against the per-process workdir-info cache +# pinning the tree to the rev seen on first load. +if [[ $(type -p git) ]]; then + createGitRepo gitflake + cat > gitflake/flake.nix <> repl_output 2>&1 & + repl_pid=$! + exec 3>repl_fifo + echo "changingThing" >&3 + for _ in $(seq 1 1000); do + grep -q "beforeChange" repl_output && break + sleep 0.1 + done + grep -q "beforeChange" repl_output || fail "git flake didn't load" + sed -i 's/beforeChange/afterChange/' gitflake/flake.nix + echo ":reload" >&3 + echo "changingThing" >&3 + echo "exit" >&3 + exec 3>&- + wait $repl_pid + grep -q "afterChange" repl_output || fail ":reload didn't pick up git work tree change" +fi + # Regression: a failed `:l` / `:lf` must not be remembered for `:reload`, # and an error in one loaded file must not drop later ones from the reload list. cat > reloadA.nix < Date: Tue, 21 Apr 2026 12:36:37 +0200 Subject: [PATCH 290/555] PosixDirectorySourceAccessor: Drop dirfd cache on resetFileCache() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit openParent() caches O_DIRECTORY fds keyed by CanonPath. When a directory at a cached path is removed and recreated, the cached fd keeps referring to the now-orphaned inode and *at() calls through it report ENOENT for entries that exist in the replacement. This shows up in `nix repl` (and any LSP / tool that keeps an EvalState alive) via the long-lived rootFS accessor: $ cat proj/default.nix { x = builtins.readFile ./sub/f; } $ echo old > proj/sub/f $ nix repl -f proj/default.nix nix-repl> x "old\n" # in another shell: rm -rf proj/sub && mkdir proj/sub && echo new > proj/sub/f # (e.g. git checkout, git stash pop, regenerating a directory) nix-repl> :r nix-repl> builtins.pathExists ./sub/f false # file is right there nix-repl> x error: path '.../proj/sub/f' does not exist Pre-#15718 nix returned "new\n" here. The evaluator already assumes the filesystem is immutable for the duration of a single evaluation, so rather than recovering lazily inside the accessor, repurpose the (now-unused) `invalidateCache()` virtual to clear the dirfd LRU and call it through `rootFS` from `EvalState::resetFileCache()` — the existing "filesystem may have changed" hook used by `:l`/`:r`/`:e` and the C API. The wrapping accessors (union, mounted, filtering) forward the call. --- src/libexpr/eval.cc | 1 + .../nix/fetchers/filtering-source-accessor.hh | 5 ++++ src/libutil-tests/source-accessor.cc | 26 +++++++++++++++++++ .../include/nix/util/source-accessor.hh | 5 ++-- src/libutil/mounted-source-accessor.cc | 5 ++++ src/libutil/posix-source-accessor.cc | 6 +++++ src/libutil/union-source-accessor.cc | 6 +++++ 7 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e4fe3ba6a5f8..fb5d8d7a5ecb 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1172,6 +1172,7 @@ void EvalState::resetFileCache() fileEvalCache->clear(); inputCache->clear(); positions.clear(); + rootFS->invalidateCache(); } void EvalState::eval(Expr * e, Value & v) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index b259e8ef8f0d..c5c1ce282b30 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -53,6 +53,11 @@ struct FilteringSourceAccessor : SourceAccessor std::pair> getFingerprint(const CanonPath & path) override; + void invalidateCache() override + { + next->invalidateCache(); + } + /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` * exception if `isAllowed()` returns `false` for `path`. diff --git a/src/libutil-tests/source-accessor.cc b/src/libutil-tests/source-accessor.cc index 1ee2c44cc6fc..836c5895a285 100644 --- a/src/libutil-tests/source-accessor.cc +++ b/src/libutil-tests/source-accessor.cc @@ -138,6 +138,32 @@ TEST_F(FSSourceAccessorTest, works) } } +TEST_F(FSSourceAccessorTest, invalidateCacheDropsStaleDirFds) +{ +#ifdef _WIN32 + GTEST_SKIP() << "fd-based accessor is Unix-only"; +#endif + auto accessor = makeFSSourceAccessor(tmpDir); + + createDirs(tmpDir / "a" / "b"); + writeFile(tmpDir / "a" / "b" / "f", "old"); + + EXPECT_TRUE(accessor->pathExists(CanonPath("a/b/f"))); + + deletePath(tmpDir / "a" / "b"); + createDirs(tmpDir / "a" / "b"); + writeFile(tmpDir / "a" / "b" / "g", "new"); + createSymlink("g", tmpDir / "a" / "b" / "l"); + + accessor->invalidateCache(); + + EXPECT_FALSE(accessor->pathExists(CanonPath("a/b/f"))); + EXPECT_TRUE(accessor->pathExists(CanonPath("a/b/g"))); + EXPECT_THAT(accessor, HasContents(CanonPath("a/b/g"), "new")); + EXPECT_THAT(accessor, HasDirectory(CanonPath("a/b"), (std::set{"g", "l"}))); + EXPECT_THAT(accessor, HasSymlink(CanonPath("a/b/l"), "g")); +} + /* ---------------------------------------------------------------------------- * RestoreSink non-directory at root (no dirFd) * --------------------------------------------------------------------------*/ diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 92bd5d30fae2..a2fbaac34993 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -231,9 +231,10 @@ struct SourceAccessor : std::enable_shared_from_this } /** - * Invalidate any cached value the accessor may have for the specified path. + * Drop any cached state that could go stale across external filesystem + * mutation (e.g. cached directory fds). */ - virtual void invalidateCache(const CanonPath & path) {} + virtual void invalidateCache() {} }; /** diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index 84840352b37a..8c546bc535c6 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -73,6 +73,11 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor } } + void invalidateCache() override + { + mounts.visit_all([](auto & kv) { kv.second->invalidateCache(); }); + } + std::optional getPhysicalPath(const CanonPath & path) override { auto [accessor, subpath] = resolve(path); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index d5dc7be282d0..da18d398e83f 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -237,10 +237,16 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase PosixDirectorySourceAccessor & operator=(const PosixDirectorySourceAccessor &) = delete; ~PosixDirectorySourceAccessor() + { + invalidateCache(); + } + + void invalidateCache() override { if (dirFdCache) { auto cache = dirFdCache->lock(); globalDirFdCount.fetch_sub(cache->size(), std::memory_order_relaxed); + cache->clear(); } } diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index da71903e6adf..9cb004c8fd8b 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -69,6 +69,12 @@ struct UnionSourceAccessor : SourceAccessor return SourceAccessor::showPath(path); } + void invalidateCache() override + { + for (auto & accessor : accessors) + accessor->invalidateCache(); + } + std::optional getPhysicalPath(const CanonPath & path) override { for (auto & accessor : accessors) { From 387ae98fe5ad2ce8837bd56c57eecb2bc3c07aef Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:12:16 +0300 Subject: [PATCH 291/555] clang-tidy: Ban raw std::filesystem::create_directories, fix hydraJobs.clangTidy It doesn't wrap exceptions. Also clang-tidy seems to have been broken with unity builds. --- nix-meson-build-support/common/clang-tidy/.clang-tidy | 6 ++++-- packaging/hydra.nix | 2 ++ src/libfetchers/git.cc | 2 +- src/libstore-tests/register-valid-paths-bench.cc | 2 +- .../include/nix/util/tests/characterization.hh | 2 +- .../include/nix/util/tests/json-characterization.hh | 4 ++-- src/libutil/file-system.cc | 2 +- 7 files changed, 12 insertions(+), 8 deletions(-) diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy index f8394f4fabd4..87ce4d663b37 100644 --- a/nix-meson-build-support/common/clang-tidy/.clang-tidy +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -41,8 +41,6 @@ Checks: - -bugprone-unused-local-non-trivial-variable # 2 warnings - returning const& from parameter - -bugprone-return-const-ref-from-parameter - # 1 warning - unsafe C functions (e.g., getenv) - - -bugprone-unsafe-functions # 1 warning - signed char misuse - -bugprone-signed-char-misuse # 1 warning - calling parent virtual instead of override @@ -90,3 +88,7 @@ CheckOptions: bugprone-reserved-identifier.AllowedIdentifiers: '__asan_default_options;__wrap___assert_fail;_SingleDerivedPathRaw;_DerivedPathRaw;_SingleBuiltPathRaw;_BuiltPathRaw' # Allow explicitly discarding return values with (void) cast bugprone-unused-return-value.AllowCastToVoid: true + bugprone-unsafe-functions.ReportDefaultFunctions: false + # Repurpose bugprone-unsafe-functions to lint functions that we'd want to wrap. + bugprone-unsafe-functions.CustomFunctions: > + ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 8f7e13c1f2ef..5558b0309efd 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -218,6 +218,8 @@ rec { tidyScope = pkgs.nixComponents2.overrideScope ( self: super: { withClangTidy = true; + # clang-tidy doesn't seem to like unity builds. + withUnityBuild = false; # nix-everything is built via callPackage (not the layer system), so # enableClangTidyLayer's doCheck=false doesn't reach it. Set it here # so checkInputs (the *-tests.tests.run derivations) aren't pulled in. diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 3941c3425660..8e959764f98b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -815,7 +815,7 @@ struct GitInputScheme : InputScheme repoDir = cacheDir; repoInfo.gitDir = "."; - std::filesystem::create_directories(cacheDir.parent_path()); + createDirs(cacheDir.parent_path()); PathLocks cacheDirLock({cacheDir.string()}); auto repo = GitRepo::openRepo(cacheDir, {.create = true, .bare = true}); diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 51bcb29aa903..6417178245af 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -22,7 +22,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) auto tmpRoot = createTempDir(); auto realStoreDir = tmpRoot / "nix/store"; - std::filesystem::create_directories(realStoreDir); + createDirs(realStoreDir); std::shared_ptr store = openStore(fmt("local?root=%s", tmpRoot.string())); auto localStore = std::dynamic_pointer_cast(store); diff --git a/src/libutil-test-support/include/nix/util/tests/characterization.hh b/src/libutil-test-support/include/nix/util/tests/characterization.hh index 6dd5f38866db..154573b5e507 100644 --- a/src/libutil-test-support/include/nix/util/tests/characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/characterization.hh @@ -60,7 +60,7 @@ struct CharacterizationTest : virtual ::testing::Test auto got = test(); if (testAccept()) { - std::filesystem::create_directories(file.parent_path()); + createDirs(file.parent_path()); writeFile2(file, got); GTEST_SKIP() << "Updating golden master " << file; } else { diff --git a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh index 9bd4d7fbf109..75eba2b06a1d 100644 --- a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh @@ -73,7 +73,7 @@ void checkpointJson(CharacterizationTest & test, std::string_view testStem, cons json gotJson = static_cast(got); if (testAccept()) { - std::filesystem::create_directories(file.parent_path()); + createDirs(file.parent_path()); writeFile(file, gotJson.dump(2) + "\n"); ADD_FAILURE() << "Updating golden master " << file; } else { @@ -98,7 +98,7 @@ void checkpointJson(CharacterizationTest & test, std::string_view testStem, cons json gotJson = static_cast(*got); if (testAccept()) { - std::filesystem::create_directories(file.parent_path()); + createDirs(file.parent_path()); writeFile(file, gotJson.dump(2) + "\n"); ADD_FAILURE() << "Updating golden master " << file; } else { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index b7e504e13a05..0169b2728a89 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -393,7 +393,7 @@ void createDir(const std::filesystem::path & path, mode_t mode) void createDirs(const std::filesystem::path & path) { try { - std::filesystem::create_directories(path); + std::filesystem::create_directories(path); // NOLINT(bugprone-unsafe-functions) } catch (std::filesystem::filesystem_error & e) { throw SystemError(e.code(), "creating directory %1%", PathFmt(path)); } From ad15006e36eabbb91c0fb7d28655e26168d60165 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:41:39 +0300 Subject: [PATCH 292/555] Clean up dead code, redundant .string() on std::filesystem::path makeParentCanonical is no longer needed with makeFSSourceAccessor achieving exactly the same behavior without the unnecessary syscalls. Also drops leftover .string() calls from the previous migrations to std::filesystem::path. --- src/libfetchers/git.cc | 2 +- src/libstore/local-store.cc | 6 +++--- src/libutil-tests/file-system.cc | 14 -------------- src/libutil/file-system.cc | 15 --------------- src/libutil/include/nix/util/file-system.hh | 17 ----------------- src/nix/dump-path.cc | 2 +- src/nix/profile.cc | 6 +++--- 7 files changed, 8 insertions(+), 54 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8e959764f98b..447b4a3694f7 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1098,7 +1098,7 @@ struct GitInputScheme : InputScheme for (auto & file : repoInfo.workdirInfo.dirtyFiles) { writeString("modified:", hashSink); writeString(file.abs(), hashSink); - dumpPath((*repoPath / file.rel()).string(), hashSink); + dumpPath(*repoPath / file.rel(), hashSink); } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b41856871bd2..52fc0925f079 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1207,7 +1207,7 @@ StorePath LocalStore::addToStoreFromDump( delTempDir = std::make_unique(tempDir); tempPath = tempDir / "x"; - restorePath(tempPath.string(), bothSource, dumpMethod, localSettings.fsyncStorePaths); + restorePath(tempPath, bothSource, dumpMethod, localSettings.fsyncStorePaths); dumpBuffer.reset(); dump = {}; @@ -1260,7 +1260,7 @@ StorePath LocalStore::addToStoreFromDump( } } else { /* Move the temporary path we restored above. */ - moveFile(tempPath.string(), realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1311,7 +1311,7 @@ std::pair LocalStore::createTempDirInStore() continue; } lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true); - } while (!pathExists(tmpDirFn.string()) || !lockedByUs); + } while (!pathExists(tmpDirFn) || !lockedByUs); return {tmpDirFn, std::move(tmpDirFd)}; } diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 995b75814f01..f290e1178090 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -256,20 +256,6 @@ TEST(pathExists, bogusPathDoesNotExist) ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); } -/* ---------------------------------------------------------------------------- - * makeParentCanonical - * --------------------------------------------------------------------------*/ - -TEST(makeParentCanonical, noParent) -{ - ASSERT_EQ(makeParentCanonical("file"), absPath(std::filesystem::path("file"))); -} - -TEST(makeParentCanonical, root) -{ - ASSERT_EQ(makeParentCanonical(FS_ROOT), FS_ROOT_NO_TRAILING_SLASH); -} - /* ---------------------------------------------------------------------------- * chmodIfNeeded * --------------------------------------------------------------------------*/ diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 0169b2728a89..20dbf3ff3d61 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -660,21 +660,6 @@ bool isExecutableFileAmbient(const std::filesystem::path & exe) == 0; } -std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath) -{ - std::filesystem::path path(absPath(rawPath)); - try { - auto parent = path.parent_path(); - if (parent == path) { - // `path` is a root directory => trivially canonical - return parent; - } - return std::filesystem::canonical(parent) / path.filename(); - } catch (std::filesystem::filesystem_error & e) { - throw SystemError(e.code(), "canonicalising parent path of %1%", PathFmt(path)); - } -} - void chmod(const std::filesystem::path & path, mode_t mode) { if ( diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index f977e089bfef..2a13b311c9b1 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -159,23 +159,6 @@ std::optional maybeStat(const std::filesystem::path & path); */ bool pathExists(const std::filesystem::path & path); -/** - * Canonicalize a path except for the last component. - * - * This is useful for getting the canonical location of a symlink. - * - * Consider the case where `foo/l` is a symlink. `canonical("foo/l")` will - * resolve the symlink `l` to its target. - * `makeParentCanonical("foo/l")` will not resolve the symlink `l` to its target, - * but does ensure that the returned parent part of the path, `foo` is resolved - * to `canonical("foo")`, and can therefore be retrieved without traversing any - * symlinks. - * - * If a relative path is passed, it will be made absolute, so that the parent - * can always be canonicalized. - */ -std::filesystem::path makeParentCanonical(const std::filesystem::path & path); - /** * A version of pathExists that returns false on a permission error. * Useful for inferring default paths across directories that might not diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 25129487786b..f21374c62337 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -61,7 +61,7 @@ struct CmdDumpPath2 : Command void run() override { auto sink = getNarSink(); - dumpPath(path.string(), sink); + dumpPath(path, sink); sink.flush(); } }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index c85c406aa518..e63d0150c08e 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -125,7 +125,7 @@ struct ProfileManifest auto manifestPath = profile / "manifest.json"; if (std::filesystem::exists(manifestPath)) { - auto json = nlohmann::json::parse(readFile(manifestPath.string())); + auto json = nlohmann::json::parse(readFile(manifestPath)); auto version = json.value("version", 0); std::string sUrl; @@ -248,13 +248,13 @@ struct ProfileManifest } } - buildProfile(tempDir.string(), std::move(pkgs)); + buildProfile(tempDir, std::move(pkgs)); writeFile(tempDir / "manifest.json", toJSON(*store).dump()); /* Add the symlink tree to the store. */ StringSink sink; - dumpPath(tempDir.string(), sink); + dumpPath(tempDir, sink); auto narHash = hashString(HashAlgorithm::SHA256, sink.s); From 3e56d682f8e20c635f435af4ec91fdc5e8f661e2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:52:49 +0300 Subject: [PATCH 293/555] Fix FreeBSD build, return includes, now with IWYU comments Got needlessly broken by trimming includes: https://hydra.nixos.org/build/326607395/nixlog/1 --- src/libexpr-test-support/tests/value/context.cc | 4 +--- src/libexpr-tests/derived-path.cc | 4 +--- src/libstore-test-support/derived-path.cc | 5 +---- .../include/nix/store/tests/outputs-spec.hh | 2 +- src/libstore-test-support/path.cc | 4 +--- src/libutil-test-support/hash.cc | 4 +--- 6 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/libexpr-test-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc index ca7996acc16e..22f8aa7cf0ff 100644 --- a/src/libexpr-test-support/tests/value/context.cc +++ b/src/libexpr-test-support/tests/value/context.cc @@ -1,6 +1,4 @@ -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/expr/tests/value/context.hh" diff --git a/src/libexpr-tests/derived-path.cc b/src/libexpr-tests/derived-path.cc index c685f6a094a8..a67f3df77b96 100644 --- a/src/libexpr-tests/derived-path.cc +++ b/src/libexpr-tests/derived-path.cc @@ -1,8 +1,6 @@ #include #include -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libstore-test-support/derived-path.cc b/src/libstore-test-support/derived-path.cc index ee8018c3268f..c27edc95ef9b 100644 --- a/src/libstore-test-support/derived-path.cc +++ b/src/libstore-test-support/derived-path.cc @@ -1,7 +1,4 @@ - -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh index a30f83770257..6cdb0a60ef96 100644 --- a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh +++ b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/outputs-spec.hh" diff --git a/src/libstore-test-support/path.cc b/src/libstore-test-support/path.cc index 98a255ccc026..bca404cde455 100644 --- a/src/libstore-test-support/path.cc +++ b/src/libstore-test-support/path.cc @@ -1,6 +1,4 @@ -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include diff --git a/src/libutil-test-support/hash.cc b/src/libutil-test-support/hash.cc index d9c7a0f74798..2dc5da5f2c14 100644 --- a/src/libutil-test-support/hash.cc +++ b/src/libutil-test-support/hash.cc @@ -1,6 +1,4 @@ -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/util/hash.hh" From 61e1be2c6d1fd8bef7186959b85ea950b8aacf61 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:54:52 +0300 Subject: [PATCH 294/555] Fix Darwin build Also needlessly broken by trimming includes... https://hydra.nixos.org/build/326734471/nixlog/1 --- src/libstore/optimise-store.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 15f2b1f3d137..eeac67ad27f3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -7,6 +7,10 @@ #include #include +#ifdef __APPLE__ +# include +#endif + #include #include #include From b74401e8714151ef0e961ca7cd13bb7e2c9cc316 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 22:07:57 +0300 Subject: [PATCH 295/555] Fix functional_root tests This check isn't reliable with CAP_DAC_OVERRIDE. It's also not very critical anyway. --- tests/functional/flakes/edit.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/flakes/edit.sh b/tests/functional/flakes/edit.sh index 0d926c28745d..2758c397ec88 100755 --- a/tests/functional/flakes/edit.sh +++ b/tests/functional/flakes/edit.sh @@ -9,4 +9,3 @@ nix edit "$flake1Dir#" | grepQuiet simple.builder.sh tar --exclude=".git*" -czf "$TEST_ROOT"/flake1Dir.tar.gz -C "$(dirname "$flake1Dir")" "$(basename "$flake1Dir")" # Test that editing a file from a tarball flake works and the file is readonly. nix edit "file://$TEST_ROOT/flake1Dir.tar.gz" | grepQuiet simple.builder.sh -EDITOR='test ! -w' nix edit "file://$TEST_ROOT/flake1Dir.tar.gz" From d1b5ac384a22a8aeae29f54ea1898596c82fc172 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 23:35:07 +0300 Subject: [PATCH 296/555] Move std::filesystem cleanup Fix several issues: * Get rid of raw usage of std::filesystem::remove where it would only be removing links - not directories. std::filesystem::remove is equivalent to POSIX remove(3) and it there unlink is the better choice. * Get rid of more unnecessary .string() usage. * Replace exists(symlink_status()) with pathExists - it has the same semantics of not following symlinks. * Replace std::filesystem::remove_all usage with deletePath - mostly in tests to prepare to ban it via clang-tidy - C++ stdlib doesn't make any guarantees about symlink race safety for those. --- src/libfetchers/git-utils.cc | 4 ++-- .../include/nix/store/tests/nix_api_store.hh | 11 ++++------- src/libstore-tests/register-valid-paths-bench.cc | 2 +- src/libstore/local-overlay-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/optimise-store.cc | 6 +++--- src/libstore/profiles.cc | 11 +---------- src/libutil-tests/unix/file-system-at.cc | 14 ++++---------- src/libutil/file-system.cc | 8 ++++++++ src/libutil/include/nix/util/file-system.hh | 7 +++++++ 10 files changed, 32 insertions(+), 35 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 03a7783c1874..216dcb741e94 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -226,7 +226,7 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options options) { - if (pathExists(path.string())) + if (pathExists(path)) return; if (!options.create) @@ -570,7 +570,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this /* Get submodule info. */ auto modulesFile = path / ".gitmodules"; - if (pathExists(modulesFile.string())) + if (pathExists(modulesFile)) info.submodules = parseSubmodules(modulesFile); return info; diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index 15df329cb2b1..df48a7469ea8 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -22,13 +22,10 @@ public: }; ~nix_api_store_test_base() override - { - if (exists(std::filesystem::path{nixDir})) { - for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) { - std::filesystem::permissions(path, std::filesystem::perms::owner_all); - } - std::filesystem::remove_all(nixDir); - } + try { + nix::deletePath(nixDir); + } catch (...) { + nix::ignoreExceptionInDestructor(); } std::string nixDir; diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 6417178245af..ecea1c8010a4 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -66,7 +66,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) state.PauseTiming(); localStore.reset(); store.reset(); - std::filesystem::remove_all(tmpRoot); + deletePath(tmpRoot); state.ResumeTiming(); } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index afbd47b5bd60..8d1a16f91281 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -262,7 +262,7 @@ LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag StorePathSet done; auto existsInStoreDir = [&](const StorePath & storePath) { - return pathExists((config->realStoreDir.get() / storePath.to_string()).string()); + return pathExists(config->realStoreDir.get() / storePath.to_string()); }; bool errors = false; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 52fc0925f079..75f485c31f22 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1365,7 +1365,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printError( "link %s was modified! expected hash %s, got '%s'", PathFmt(link.path()), name.string(), hash); if (repair) { - std::filesystem::remove(link.path()); + unlinkIfExists(link.path()); printInfo("removed link %s", PathFmt(link.path())); } else { errors = true; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index eeac67ad27f3..0b85c6e3dc20 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -164,7 +164,7 @@ void LocalStore::optimisePath_( std::filesystem::path linkPath = std::filesystem::path{linksDir} / hash.to_string(HashFormat::Nix32, false); /* Maybe delete the link, if it has been corrupted. */ - if (std::filesystem::exists(std::filesystem::symlink_status(linkPath))) { + if (pathExists(linkPath)) { auto stLink = lstat(linkPath); if (st.st_size != stLink.st_size || (repair && hash != ({ hashPath( @@ -178,11 +178,11 @@ void LocalStore::optimisePath_( warn( "There may be more corrupted paths." "\nYou should run `nix-store --verify --check-contents --repair` to fix them all"); - std::filesystem::remove(linkPath); + unlinkIfExists(linkPath); } } - if (!std::filesystem::exists(std::filesystem::symlink_status(linkPath))) { + if (!pathExists(linkPath)) { /* Nope, create a hard link in the links directory. */ try { std::filesystem::create_hard_link(path, linkPath); diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 519c2abc9805..d015548e5e65 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -95,19 +95,10 @@ std::filesystem::path createGeneration(LocalFSStore & store, std::filesystem::pa return generation; } -static void removeFile(const std::filesystem::path & path) -{ - try { - std::filesystem::remove(path); - } catch (std::filesystem::filesystem_error & e) { - throw SystemError(e.code(), "removing file %1%", PathFmt(path)); - } -} - void deleteGeneration(const std::filesystem::path & profile, GenerationNumber gen) { std::filesystem::path generation = makeName(profile, gen); - removeFile(generation); + unlinkIfExists(generation); } /** diff --git a/src/libutil-tests/unix/file-system-at.cc b/src/libutil-tests/unix/file-system-at.cc index 09e427a26faf..e453a80d325b 100644 --- a/src/libutil-tests/unix/file-system-at.cc +++ b/src/libutil-tests/unix/file-system-at.cc @@ -42,8 +42,6 @@ TEST(fchmodatTryNoFollow, works) auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); ASSERT_TRUE(dirFd); - struct ::stat st; - using nix::testing::ThrowsSysError; /* Check that symlinks are not followed and targets are not changed. @@ -82,22 +80,18 @@ TEST(fchmodatTryNoFollow, works) }; expectSymlinkChmod("filelink", 0777); - ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0644); + ASSERT_EQ(stat(tmpDir / "file").st_mode & 0777, 0644); expectSymlinkChmod("dirlink", 0777); - ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0755); + ASSERT_EQ(stat(tmpDir / "dir").st_mode & 0777, 0755); /* Check fchmodatTryNoFollow works on regular files and directories. */ EXPECT_NO_THROW(fchmodatTryNoFollow(dirFd.get(), CanonPath("file"), 0600)); - ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0600); + ASSERT_EQ(stat(tmpDir / "file").st_mode & 0777, 0600); EXPECT_NO_THROW(fchmodatTryNoFollow(dirFd.get(), CanonPath("dir"), 0700)); - ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0700); + ASSERT_EQ(stat(tmpDir / "dir").st_mode & 0777, 0700); EXPECT_THAT([&] { fchmodatTryNoFollow(dirFd.get(), CanonPath("nonexistent"), 0600); }, ThrowsSysError(ENOENT)); } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 20dbf3ff3d61..b3087700d733 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -679,6 +679,14 @@ void chmod(const std::filesystem::path & path, mode_t mode) # define UNLINK_PROC ::unlink #endif +void unlinkIfExists(const std::filesystem::path & path) +{ + if (UNLINK_PROC(path.c_str()) == -1) { + if (errno != ENOENT) + throw SysError("removing %s", PathFmt(path)); + } +} + void unlink(const std::filesystem::path & path) { if (UNLINK_PROC(path.c_str()) == -1) diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 2a13b311c9b1..fca79ceb8ba6 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -514,6 +514,13 @@ void chown(const std::filesystem::path & path, uid_t owner, gid_t group); */ void unlink(const std::filesystem::path & path); +/** + * Remove a file, throwing an exception on error. ENOENT is ignored. + * + * @param path Path to the file to remove. + */ +void unlinkIfExists(const std::filesystem::path & path); + /** * Try to remove a file, ignoring errors. * From b3e84b77679e727d4cba0b54c49ba0588b387093 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 23:42:18 +0300 Subject: [PATCH 297/555] Ban std::filesystem::remove_all in the codebase All instances of those must use deletePath instead - standard library implementation is not guaranteed to be robust against symlink races. --- nix-meson-build-support/common/clang-tidy/.clang-tidy | 1 + src/libutil/windows/file-system.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy index 87ce4d663b37..daf4c7f7c4b8 100644 --- a/nix-meson-build-support/common/clang-tidy/.clang-tidy +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -92,3 +92,4 @@ CheckOptions: # Repurpose bugprone-unsafe-functions to lint functions that we'd want to wrap. bugprone-unsafe-functions.CustomFunctions: > ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; + ::std::filesystem::remove_all, nix::deletePath, "Use nix::deletePath (remove_all is not TOCTOU safe)"; diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 98d41a9caa25..2ac2f74f80c1 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -77,7 +77,7 @@ std::filesystem::path defaultTempDir() void deletePath(const std::filesystem::path & path) { std::error_code ec; - std::filesystem::remove_all(path, ec); + std::filesystem::remove_all(path, ec); // NOLINT(bugprone-unsafe-functions) if (ec && ec != std::errc::no_such_file_or_directory) throw SysError(ec.default_error_condition().value(), "recursively deleting %1%", PathFmt(path)); } From 77db3d45cd36386ee222730419ad15c68adf4d4b Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 15 Apr 2026 12:25:24 +0000 Subject: [PATCH 298/555] libutil: Bound NAR directory depth and guard coroutine stacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parse()/dumpPath() recurse per directory level with no bound. On the main stack this overflows at a few thousand levels (DoS). Inside a sinkToSource/sourceToSink coroutine the stack is a 128 KiB malloc() chunk with no guard page, so overflow writes into adjacent heap — the v<1.25 wopAddToStore handler runs parseDump() on exactly such a stack and any client can downgrade to that protocol version. Reject NARs deeper than narMaxDepth (64) in both parse() and dumpPath(), and allocate coroutine stacks with boost::coroutines2::protected_fixedsize_stack so any remaining overflow hits a guard page instead of the heap. --- src/libutil/archive.cc | 25 +++++++++++++++++++------ src/libutil/serialise.cc | 5 +++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 91af4b9e7f54..cfc0a510704f 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -30,6 +30,12 @@ static ArchiveSettings archiveSettings; static GlobalConfig::Register rArchiveSettings(&archiveSettings); +/* Maximum directory nesting depth for dumpPath()/parseDump(). Bounds + stack usage so deep trees cannot overflow the (possibly coroutine) + stack these run on. Chosen to fit comfortably in the default 128 KiB + boost coroutine stack. */ +static constexpr size_t narMaxDepth = 64; + PathFilter defaultPathFilter = [](const std::string &) { return true; }; void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter) @@ -51,9 +57,13 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & this const auto & dump, SourceAccessor & accessor, const CanonPath & path, - const CanonPath & filterPath) -> void { + const CanonPath & filterPath, + size_t depth) -> void { checkInterrupt(); + if (depth >= narMaxDepth) + throw Error("path '%s' exceeds maximum NAR directory depth of %d", accessor.showPath(path), narMaxDepth); + auto st = accessor.lstat(path); sink << "("; @@ -89,7 +99,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & for (auto & i : unhacked) if (filter((filterPath / i.first).abs())) { sink << "entry" << "(" << "name" << i.first << "node"; - dump(subdirAccessor, subdirRelPath / i.second, filterPath / i.second); + dump(subdirAccessor, subdirRelPath / i.second, filterPath / i.second, depth + 1); sink << ")"; } }); @@ -102,7 +112,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & throw Error("file '%s' has an unsupported type", path); sink << ")"; - }(*this, path, path); + }(*this, path, path, 0); } time_t dumpPathAndGetMtime(const std::filesystem::path & path, Sink & sink, PathFilter & filter) @@ -153,8 +163,11 @@ struct CaseInsensitiveCompare } }; -static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath & path) +static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath & path, size_t depth) { + if (depth >= narMaxDepth) + throw badArchive("NAR directory nesting exceeds maximum depth of %d", narMaxDepth); + auto getString = [&]() { checkInterrupt(); return readString(source); @@ -237,7 +250,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath expectTag("node"); - parse(dirSink, source, relDirPath / name); + parse(dirSink, source, relDirPath / name, depth + 1); expectTag(")"); } @@ -268,7 +281,7 @@ void parseDump(FileSystemObjectSink & sink, Source & source) } if (version != narVersionMagic1) throw badArchive("input doesn't look like a Nix archive"); - parse(sink, source, CanonPath::root); + parse(sink, source, CanonPath::root, 0); } void restorePath(const std::filesystem::path & path, Source & source, bool startFsync) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 6c77c15fe584..542cee2b0278 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -10,6 +10,7 @@ #include #include +#include #ifdef _WIN32 # include @@ -328,7 +329,7 @@ std::unique_ptr sourceToSink(fun reader) cur = in; if (!coro) { - coro = coro_t::push_type([&](coro_t::pull_type & yield) { + coro = coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { yield(); @@ -385,7 +386,7 @@ std::unique_ptr sinkToSource(fun writer, fun eof) { bool hasCoro = coro.has_value(); if (!hasCoro) { - coro = coro_t::pull_type([&](coro_t::push_type & yield) { + coro = coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) { yield(data); From 0d461dc28e9624685b3cc09ab877e2d3976c2b69 Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 15 Apr 2026 12:37:51 +0000 Subject: [PATCH 299/555] libutil: Bound string lengths in the NAR parser readString() defaults to no length limit, so every token, directory entry name, and symlink target in a NAR was read into a freshly allocated std::string of attacker-chosen size before any validation. A multi-GB length prefix where "(" is expected would be allocated and filled before expectTag() rejects it. Cap tags and keywords at 32 bytes, entry names at 255, and symlink targets at 4095. Overlong strings now throw SerialisationError("string is too long") without allocating. --- src/libutil/archive.cc | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index cfc0a510704f..3ddc280f9519 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -168,33 +168,42 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath if (depth >= narMaxDepth) throw badArchive("NAR directory nesting exceeds maximum depth of %d", narMaxDepth); - auto getString = [&]() { + /* NAR keywords are all <= 10 bytes; a little slack keeps error + messages useful for short garbage without allowing large + allocations. */ + constexpr size_t narMaxTag = 32; + /* Format-defined bounds, intentionally independent of host + NAME_MAX/PATH_MAX. */ + constexpr size_t narMaxName = 255; + constexpr size_t narMaxTarget = 4095; + + auto getString = [&](size_t max) { checkInterrupt(); - return readString(source); + return readString(source, max); }; auto expectTag = [&](std::string_view expected) { - auto tag = getString(); + auto tag = getString(narMaxTag); if (tag != expected) - throw badArchive("expected tag '%s', got '%s'", expected, tag.substr(0, 1024)); + throw badArchive("expected tag '%s', got '%s'", expected, tag); }; expectTag("("); expectTag("type"); - auto type = getString(); + auto type = getString(narMaxTag); if (type == "regular") { sink.createRegularFile(path, [&](auto & crf) { - auto tag = getString(); + auto tag = getString(narMaxTag); if (tag == "executable") { - auto s2 = getString(); + auto s2 = getString(0); if (s2 != "") throw badArchive("executable marker has non-empty value"); crf.isExecutable(); - tag = getString(); + tag = getString(narMaxTag); } if (tag != "contents") @@ -213,7 +222,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath std::string prevName; while (1) { - auto tag = getString(); + auto tag = getString(narMaxTag); if (tag == ")") break; @@ -225,7 +234,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath expectTag("name"); - auto name = getString(); + auto name = getString(narMaxName); if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos) throw badArchive("NAR contains invalid file name '%1%'", name); @@ -260,7 +269,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath else if (type == "symlink") { expectTag("target"); - auto target = getString(); + auto target = getString(narMaxTarget); sink.createSymlink(path, target); expectTag(")"); From ee68e870ece0332cf7886950ae9892296b36b821 Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 15 Apr 2026 12:37:51 +0000 Subject: [PATCH 300/555] libutil: Reject empty and NUL-containing symlink targets in NARs A NUL byte in a symlink target truncates at the symlink(2) boundary, so "foo\0junk" and "foo" restore to identical filesystem state but have different NAR hashes. LocalStore::addToStore() hashes the incoming stream, while `nix store verify` and re-export hash a fresh dump of the on-disk tree, so a non-canonical NAR would pass the ingest check but fail every later verification. Rejecting NUL (and empty, which symlink(2) refuses anyway) keeps parse() injective on accepted inputs, which the narHash model depends on. --- src/libutil/archive.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 3ddc280f9519..cd7f58263c7d 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -270,6 +270,8 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath expectTag("target"); auto target = getString(narMaxTarget); + if (target.empty() || target.find((char) 0) != std::string::npos) + throw badArchive("NAR contains invalid symlink target"); sink.createSymlink(path, target); expectTag(")"); From 5d8406285b7d973477037135f3226fef0fb74db5 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Apr 2026 01:04:03 +0300 Subject: [PATCH 301/555] daemon: Limit the number of crashes before exiting to 64 In case someone is intentionally crashing the daemon and brute-force ASLR or get some other kind of info leak about the address-space layout, we'd like to to limit the blast radius by dying and (maybe) being restarted to get a fresh address space layout. --- .../include/nix/cmd/unix-socket-server.hh | 3 ++ src/libcmd/unix/unix-socket-server.cc | 3 ++ src/nix/unix/daemon.cc | 35 +++++++++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh index 7a0d9fa79317..48202cf18293 100644 --- a/src/libcmd/include/nix/cmd/unix-socket-server.hh +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -68,6 +68,8 @@ struct ServeUnixSocketOptions #endif }; +MakeError(AbortServeSocket, BaseError); + /** * Run a server loop that accepts connections and calls the handler for each. * @@ -83,6 +85,7 @@ struct ServeUnixSocketOptions * * This function never returns normally. It runs until interrupted * (e.g., via SIGINT), at which point it throws `Interrupted`. + * Can be explicitly exited by throwing AbortServeSocket. * * @param options Configuration for the server. * @param handler Callback invoked for each accepted connection. diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index 5d1fba462207..c0348fa3b444 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -122,6 +122,9 @@ PeerInfo getPeerInfo(Descriptor remote) handler(std::move(remote), [&]() { listeningSockets.clear(); }); } + } catch (AbortServeSocket &) { + /* Explicitly aborted, bail out. */ + throw; } catch (Error & error) { auto ei = error.info(); // FIXME: add to trace? diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 05e47f79c36b..4bc7a512d9e4 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -280,6 +280,21 @@ static void daemonLoop( } #endif + /* Check for anything that might be a crash. Too many crashes aren't + supposed to happen and we should limit the amount if someone is + intentionally triggering those as an ASLR bypass attempt (each forked + daemon worker has the same address space layout as we do). TODO: Ideally + we'd re-exec the daemon worker so that it gets a fresh address space + for each connection. Alternatively, we could make the daemon socket use + Accept=yes systemd.socket(5). */ + unsigned crashCount = 0; + + /* For now we are just limiting the number of crashes experienced by this + daemon instance. systemd (e.g.) would restart us, which would get us + a fresh address space layout - which is exactly what we want in case + someone is intentionally crashing the daemon to brute-force ASLR. */ + static constexpr unsigned crashLimit = 64; + try { unix::serveUnixSocket( { @@ -287,13 +302,29 @@ static void daemonLoop( .socketMode = 0666, .auxiliaryFd = sigChldPipe.pipe.readSide.get(), .onAuxiliaryFdPollin = - []() { + [&crashCount]() { sigChldPipe.drain(); /* Reap all dead children. */ pid_t pid = -1; int status; - while (pid = ::waitpid(/*pid (any child process)=*/-1, &status, WNOHANG), pid > 0) + while (pid = ::waitpid(/*pid (any child process)=*/-1, &status, WNOHANG), pid > 0) { printInfo("reaped child process %1%, status = %2%", pid, statusToString(status)); + + if (!WIFSIGNALED(status)) + continue; + + int sig = WTERMSIG(status); + for (auto i : {SIGILL, SIGSEGV, SIGBUS, SIGABRT, SIGSYS, SIGFPE}) { + if (sig == i) { + printInfo("daemon worker %1% crashed", pid); + ++crashCount; + break; + } + } + + if (crashCount >= crashLimit) + throw unix::AbortServeSocket("too many daemon worker crashes (%1%)", crashLimit); + } }, }, [&](AutoCloseFD remote, std::function closeListeners) { From 88b5bcd07751de3b5ca174d5192cf4d160434748 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Apr 2026 01:06:24 +0300 Subject: [PATCH 302/555] libutil: Make formatter happy --- src/libutil/serialise.cc | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 542cee2b0278..4e6daa84c9b4 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -329,20 +329,21 @@ std::unique_ptr sourceToSink(fun reader) cur = in; if (!coro) { - coro = coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { - LambdaSource source([&](char * out, size_t out_len) { - if (cur.empty()) { - yield(); - if (yield.get()) - throw EndOfFile("coroutine has finished"); - } - - size_t n = cur.copy(out, out_len); - cur.remove_prefix(n); - return n; + coro = + coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { + LambdaSource source([&](char * out, size_t out_len) { + if (cur.empty()) { + yield(); + if (yield.get()) + throw EndOfFile("coroutine has finished"); + } + + size_t n = cur.copy(out, out_len); + cur.remove_prefix(n); + return n; + }); + reader(source); }); - reader(source); - }); } if (!*coro) { @@ -386,14 +387,15 @@ std::unique_ptr sinkToSource(fun writer, fun eof) { bool hasCoro = coro.has_value(); if (!hasCoro) { - coro = coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { - LambdaSink sink([&](std::string_view data) { - if (!data.empty()) { - yield(data); - } + coro = + coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { + LambdaSink sink([&](std::string_view data) { + if (!data.empty()) { + yield(data); + } + }); + writer(sink); }); - writer(sink); - }); } if (cur.empty()) { From 7cdf3bad750b2af7c91e973f86ab8833312fa7ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 23 Apr 2026 00:23:28 +0200 Subject: [PATCH 303/555] packaging: build the Rust nix-installer with embedded Nix Builds NixOS/nix-installer via buildRustPackage with this revision's Nix closure baked in, exposed as hydraJobs.rustInstaller.. Replaces the dropped --nix-package-url knob (#15728). Source pinned via fetchFromGitHub to avoid a circular flake input. --- flake.nix | 3 ++ packaging/hydra.nix | 26 +++++++++++ packaging/rust-installer/default.nix | 64 ++++++++++++++++++++++++++++ packaging/rust-installer/tarball.nix | 50 ++++++++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 packaging/rust-installer/default.nix create mode 100644 packaging/rust-installer/tarball.nix diff --git a/flake.nix b/flake.nix index 89cbe93cff7f..c9be5e1dd58c 100644 --- a/flake.nix +++ b/flake.nix @@ -466,6 +466,9 @@ ) ) ) + // lib.optionalAttrs (self.hydraJobs.rustInstaller ? ${system}) { + rustInstaller = self.hydraJobs.rustInstaller.${system}; + } // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 5558b0309efd..e3f8f1c1f1cb 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -298,6 +298,32 @@ rec { } ); + # `NixOS/nix-installer` with this revision's Nix closure embedded. + rustInstaller = + lib.genAttrs + ( + linux64BitSystems + ++ [ + "x86_64-darwin" + "aarch64-darwin" + ] + ) + ( + system: + let + pkgs = nixpkgsFor.${system}.native; + # Embed the native (glibc) Nix even though the Linux installer + # binary is static/musl. + tarball = pkgs.callPackage ./rust-installer/tarball.nix { + nix = pkgs.nixComponents2.nix-everything; + }; + builder = if pkgs.stdenv.hostPlatform.isLinux then pkgs.pkgsStatic else pkgs; + in + builder.callPackage ./rust-installer { + inherit tarball; + } + ); + # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); diff --git a/packaging/rust-installer/default.nix b/packaging/rust-installer/default.nix new file mode 100644 index 000000000000..dcfbf4badc31 --- /dev/null +++ b/packaging/rust-installer/default.nix @@ -0,0 +1,64 @@ +# `NixOS/nix-installer` built with *this* Nix closure embedded, so +# Hydra/CI can dogfood the Rust installer without the (removed) +# `--nix-package-url` knob. +{ + lib, + stdenv, + rustPlatform, + fetchFromGitHub, + tarball, +}: + +let + installerVersion = "2.34.5"; + src = fetchFromGitHub { + owner = "NixOS"; + repo = "nix-installer"; + tag = installerVersion; + hash = "sha256-+gM241qQOzQlOnP0a7d47z3iRf9+yNjbBJCLIWWNX+c="; + }; +in + +rustPlatform.buildRustPackage { + pname = "nix-installer"; + version = tarball.passthru.nixVersion; + + inherit src; + + cargoHash = "sha256-6pt2f7wznH672L5+SkbA5GA6Sxvk1KamAf3erGqZlLU="; + + doCheck = false; + + env = { + NIX_TARBALL_PATH = "${tarball}/nix.tar.zst"; + NIX_STORE_PATH = tarball.passthru.nixStorePath; + NSS_CACERT_STORE_PATH = tarball.passthru.cacertStorePath; + NIX_VERSION = tarball.passthru.nixVersion; + } + // lib.optionalAttrs stdenv.hostPlatform.isDarwin { + # Drop the unused libiconv dylib the darwin stdenv injects; the + # binary must run before `/nix/store` exists. + NIX_LDFLAGS = "-dead_strip_dylibs"; + }; + + postInstall = '' + install -m755 nix-installer.sh $out/bin/nix-installer.sh + + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix-installer" >> $out/nix-support/hydra-build-products + echo "file binary-dist $out/bin/nix-installer.sh" >> $out/nix-support/hydra-build-products + ''; + + # The binary embeds store-path strings (`NIX_STORE_PATH`, …) on + # purpose; don't let the reference scanner pull the whole Nix + # closure into this derivation's runtime closure. + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + + meta = { + description = "Rust-based Nix installer with an embedded Nix ${tarball.passthru.nixVersion}"; + homepage = "https://github.com/NixOS/nix-installer"; + license = lib.licenses.lgpl21Only; + mainProgram = "nix-installer"; + }; +} diff --git a/packaging/rust-installer/tarball.nix b/packaging/rust-installer/tarball.nix new file mode 100644 index 000000000000..4f236e0215bb --- /dev/null +++ b/packaging/rust-installer/tarball.nix @@ -0,0 +1,50 @@ +# Zstd-compressed Nix closure in the layout expected by +# `NixOS/nix-installer` (`include_bytes!` at build time). +{ + lib, + stdenv, + runCommand, + buildPackages, + zstd, + nix, + cacert, +}: + +let + installerClosureInfo = buildPackages.closureInfo { + rootPaths = [ + nix + cacert + ]; + }; +in + +runCommand "nix-installer-tarball-${nix.version}" + { + nativeBuildInputs = [ zstd ]; + + passthru = { + nixStorePath = nix.outPath; + cacertStorePath = cacert.outPath; + nixVersion = nix.version; + }; + } + '' + mkdir -p $out + + dir=nix-${nix.version}-${stdenv.hostPlatform.system} + + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + + tar cf - \ + --sort=name \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --mtime='1970-01-01' \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) \ + | zstd -19 -T1 -o $out/nix.tar.zst + '' From b12be692c8c1f73ea13bde46c00100fbe4a851c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 23 Apr 2026 00:23:28 +0200 Subject: [PATCH 304/555] ci: run the Rust installer in installer_test Build .#rustInstaller alongside the bash installer, ship it in the same artifact and run the binary directly; drop the now-dead experimental-installer plumbing from install-nix-action. --- .../actions/install-nix-action/action.yaml | 70 ------------------- .github/workflows/ci.yml | 33 ++++----- .../prepare-installer-for-github-actions | 6 +- 3 files changed, 20 insertions(+), 89 deletions(-) diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml index 535ae9d08fd9..f889944b2d03 100644 --- a/.github/actions/install-nix-action/action.yaml +++ b/.github/actions/install-nix-action/action.yaml @@ -4,22 +4,12 @@ inputs: dogfood: description: "Whether to use Nix installed from the latest artifact from master branch" required: true # Be explicit about the fact that we are using unreleased artifacts - experimental-installer: - description: "Whether to use the experimental installer to install Nix" - default: false - experimental-installer-version: - description: "Version of the experimental installer to use. If `latest`, the newest artifact from the default branch is used." - # TODO: This should probably be pinned to a release after https://github.com/NixOS/experimental-nix-installer/pull/49 lands in one - default: "latest" extra_nix_config: description: "Gets appended to `/etc/nix/nix.conf` if passed." install_url: description: "URL of the Nix installer" required: false default: "https://releases.nixos.org/nix/nix-2.32.1/install" - tarball_url: - description: "URL of the Nix tarball to use with the experimental installer" - required: false github_token: description: "Github token" required: true @@ -51,74 +41,14 @@ runs: gh run download "$RUN_ID" --repo "$DOGFOOD_REPO" -n "$INSTALLER_ARTIFACT" -D "$INSTALLER_DOWNLOAD_DIR" echo "installer-path=file://$INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - TARBALL_PATH="$(find "$INSTALLER_DOWNLOAD_DIR" -name 'nix*.tar.xz' -print | head -n 1)" - echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" echo "::notice ::Dogfooding Nix installer from master (https://github.com/$DOGFOOD_REPO/actions/runs/$RUN_ID)" env: GH_TOKEN: ${{ inputs.github_token }} DOGFOOD_REPO: "NixOS/nix" - - name: "Gather system info for experimental installer" - shell: bash - if: ${{ inputs.experimental-installer == 'true' }} - run: | - echo "::notice Using experimental installer from $EXPERIMENTAL_INSTALLER_REPO (https://github.com/$EXPERIMENTAL_INSTALLER_REPO)" - - if [ "$RUNNER_OS" == "Linux" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="linux" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - elif [ "$RUNNER_OS" == "macOS" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="darwin" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_OS: $RUNNER_OS" - exit 1 - fi - - if [ "$RUNNER_ARCH" == "X64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=x86_64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - elif [ "$RUNNER_ARCH" == "ARM64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=aarch64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_ARCH: $RUNNER_ARCH" - exit 1 - fi - - echo "EXPERIMENTAL_INSTALLER_ARTIFACT=nix-installer-$EXPERIMENTAL_INSTALLER_ARCH-$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - env: - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - - name: "Download latest experimental installer" - shell: bash - id: download-latest-experimental-installer - if: ${{ inputs.experimental-installer == 'true' && inputs.experimental-installer-version == 'latest' }} - run: | - RUN_ID=$(gh run list --repo "$EXPERIMENTAL_INSTALLER_REPO" --workflow ci.yml --branch main --status success --json databaseId --jq ".[0].databaseId") - - EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR="$GITHUB_WORKSPACE/$EXPERIMENTAL_INSTALLER_ARTIFACT" - mkdir -p "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - - gh run download "$RUN_ID" --repo "$EXPERIMENTAL_INSTALLER_REPO" -n "$EXPERIMENTAL_INSTALLER_ARTIFACT" -D "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - # Executable permissions are lost in artifacts - find $EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR -type f -exec chmod +x {} + - echo "installer-path=$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ inputs.github_token }} - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 - if: ${{ inputs.experimental-installer != 'true' }} with: # Ternary operator in GHA: https://www.github.com/actions/runner/issues/409#issuecomment-752775072 install_url: ${{ inputs.dogfood == 'true' && format('{0}/install', steps.download-nix-installer.outputs.installer-path) || inputs.install_url }} install_options: ${{ inputs.dogfood == 'true' && format('--tarball-url-prefix {0}', steps.download-nix-installer.outputs.installer-path) || '' }} extra_nix_config: ${{ inputs.extra_nix_config }} - - uses: DeterminateSystems/nix-installer-action@786fff0690178f1234e4e1fe9b536e94f5433196 # v20 - if: ${{ inputs.experimental-installer == 'true' }} - with: - diagnostic-endpoint: "" - # TODO: It'd be nice to use `artifacts.nixos.org` for both of these, maybe through an `/experimental-installer/latest` endpoint? or `/commit/`? - local-root: ${{ inputs.experimental-installer-version == 'latest' && steps.download-latest-experimental-installer.outputs.installer-path || '' }} - source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }} - nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }} - extra-conf: ${{ inputs.extra_nix_config }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be8cb0f0c4fd..07329d16a202 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,19 +164,19 @@ jobs: - scenario: on ubuntu runs-on: ubuntu-24.04 os: linux - experimental-installer: false + rust-installer: false - scenario: on macos runs-on: macos-14 os: darwin - experimental-installer: false - - scenario: on ubuntu (experimental) + rust-installer: false + - scenario: on ubuntu (rust) runs-on: ubuntu-24.04 os: linux - experimental-installer: true - - scenario: on macos (experimental) + rust-installer: true + - scenario: on macos (rust) runs-on: macos-14 os: darwin - experimental-installer: true + rust-installer: true name: installer test ${{ matrix.scenario }} runs-on: ${{ matrix.runs-on }} steps: @@ -188,22 +188,19 @@ jobs: path: out - name: Looking up the installer tarball URL id: installer-tarball-url - run: | - echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" - echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" + run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 - if: ${{ !matrix.experimental-installer }} + if: ${{ !matrix.rust-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} - - uses: ./.github/actions/install-nix-action - if: ${{ matrix.experimental-installer }} - with: - dogfood: false - experimental-installer: true - tarball_url: ${{ steps.installer-tarball-url.outputs.tarball-path }} - github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Run rust installer + if: ${{ matrix.rust-installer }} + run: | + chmod +x out/nix-installer + ./out/nix-installer install --no-confirm + env: + RUST_BACKTRACE: full - run: sudo apt install fish zsh if: matrix.os == 'linux' - run: brew install fish diff --git a/ci/gha/tests/prepare-installer-for-github-actions b/ci/gha/tests/prepare-installer-for-github-actions index 0fbecf25c2aa..e240e56fca0a 100755 --- a/ci/gha/tests/prepare-installer-for-github-actions +++ b/ci/gha/tests/prepare-installer-for-github-actions @@ -2,10 +2,14 @@ set -euo pipefail -nix build -L ".#installerScriptForGHA" ".#binaryTarball" +nix build -L \ + ".#installerScriptForGHA" \ + ".#binaryTarball" \ + ".#rustInstaller" mkdir -p out cp ./result/install "out/install" name="$(basename "$(realpath ./result-1)")" # everything before the first dash cp -r ./result-1 "out/${name%%-*}" +cp ./result-2/bin/nix-installer "out/nix-installer" From 23d53de80a631bc2f5fefe2ed1d2f5b865755bc6 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 02:09:52 +0300 Subject: [PATCH 305/555] PosixDirectorySourceAccessor: Correct the SymlinkNotAllowed error message with cached parent dirfds This wouldn't display the prefix that's already cached. --- src/libutil/posix-source-accessor.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index da18d398e83f..1fe3bf134666 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -333,10 +333,14 @@ std::pair> PosixDirectorySourceAccessor } } - AutoCloseFD parentFdOwning = - openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); - - return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; + try { + AutoCloseFD parentFdOwning = + openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); + return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; + } catch (SymlinkNotAllowed & e) { + /* Need to fixup the error message to include the actual path relative to the (possibly) cached fd. */ + throw SymlinkNotAllowed(anchor / e.path, "path '%s' is a symlink", showPath(anchor / e.path)); + } } std::optional PosixDirectorySourceAccessor::maybeLstat(const CanonPath & path) From 3e458a7af4544925624d7b00513571e178d78c5a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 02:35:06 +0300 Subject: [PATCH 306/555] PosixDirectorySourceAccessor: Improve dirFd caching for readFile/openSubdirectory Those didn't do caching without openat2 and did a bit more syscalls that ideal. With openat2 this is a slight regression, but overall not too meaningful. --- src/libutil/posix-source-accessor.cc | 93 +++++++++++++++++----------- 1 file changed, 57 insertions(+), 36 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 1fe3bf134666..661922cd39cc 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -207,6 +207,13 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase globalDirFdCount.fetch_add(cache->size() - before, std::memory_order_relaxed); } + /** + * Helper for opening the parent directory used for other FD-relative operations down the line. + * Also caches the resulting dirFd. Throws FileNotFound if some intermediate directory or the parent + * doesn't exist. + */ + std::pair> openParentAndUpsert(const CanonPath & path, bool ignoreMissing); + /** * Get the parent directory of path. The second pair element might be an owning file descriptor * if path.parent().isRoot() is false. @@ -343,25 +350,39 @@ std::pair> PosixDirectorySourceAccessor } } +std::pair> +PosixDirectorySourceAccessor::openParentAndUpsert(const CanonPath & path, bool ignoreMissing) +{ + auto [parentFd, parentFdOwning] = openParent(path); + if (parentFd == INVALID_DESCRIPTOR) { + if (errno == ENOENT || errno == ENOTDIR) /* Intermediate component might not exist. */ { + if (ignoreMissing) + return {INVALID_DESCRIPTOR, {}}; + else + throw FileNotFound("path '%s' does not exist", showPath(path)); + } + throw SysError("opening directory '%1%'", showPath(path.parent().value())); + } + + if (dirFdCache && parentFdOwning) { + assert(*parentFdOwning); + insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); + } + + return {parentFd, parentFdOwning}; +} + std::optional PosixDirectorySourceAccessor::maybeLstat(const CanonPath & path) -try { +{ PosixStat st; if (path.isRoot()) { /* Must never fail - we already have the file descriptor for the directory. */ st = nix::fstat(dirFd.get()); } else { - auto [parentFd, parentFdOwning] = openParent(path); - if (parentFd == INVALID_DESCRIPTOR) { - if (errno == ENOENT || errno == ENOTDIR) - return std::nullopt; - throw SysError("opening directory '%1%'", showPath(path.parent().value())); - } - - if (dirFdCache && parentFdOwning) { - assert(*parentFdOwning); - insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); - } + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/true); + if (parentFd == INVALID_DESCRIPTOR) + return std::nullopt; /* We know that CanonPath returns a NUL-terminated string_view, so the use of ->data() here is safe. */ if (::fstatat(parentFd, path.baseName()->data(), &st, AT_SYMLINK_NOFOLLOW) == -1) { @@ -373,20 +394,26 @@ try { maybeUpdateMtime(st.st_mtime); return sourceAccessorStatFromPosixStat(st); -} catch (SymlinkNotAllowed & e) { - throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } void PosixDirectorySourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) -try { +{ if (path.isRoot()) throw NotARegularFile("'%s' is not a regular file", showPath(path)); - AutoCloseFD fileFd = - openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + /* TODO: We can do better when we have openat2. */ + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/false); + AutoCloseFD fileFd; + + try { + fileFd = openFileEnsureBeneathNoSymlinks(parentFd, path.baseName().value(), O_RDONLY | O_CLOEXEC); + } catch (SymlinkNotAllowed & e) { + auto parent = path.parent().value(); + throw SymlinkNotAllowed(parent / e.path, "path '%s' is a symlink", showPath(parent / e.path)); + } if (!fileFd) { - if (errno == ENOENT || errno == ENOTDIR) /* Intermediate component might not exist. */ + if (errno == ENOENT) throw FileNotFound("file '%s' does not exist", showPath(path)); throw SysError("opening '%s'", showPath(path)); } @@ -397,12 +424,10 @@ try { PosixFileSourceAccessor fileAccessor(std::move(fileFd), fsPath / path.rel(), trackLastModified, st); maybeUpdateMtime(st.st_mtime); fileAccessor.readFile(CanonPath::root, sink, sizeCallback); -} catch (SymlinkNotAllowed & e) { - throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } AutoCloseFD PosixDirectorySourceAccessor::openSubdirectory(const CanonPath & path) -try { +{ AutoCloseFD dirFdOwning; if (path.isRoot()) { @@ -411,8 +436,16 @@ try { if (!dirFdOwning) throw SysError("opening directory '%s'", showPath(path)); } else { - dirFdOwning = openFileEnsureBeneathNoSymlinks( - dirFd.get(), path, O_DIRECTORY | O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + /* TODO: We can do better when we have openat2. */ + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/false); + + try { + dirFdOwning = + openFileEnsureBeneathNoSymlinks(parentFd, path.baseName().value(), O_DIRECTORY | O_RDONLY | O_CLOEXEC); + } catch (SymlinkNotAllowed & e) { + auto parent = path.parent().value(); + throw SymlinkNotAllowed(parent / e.path, "path '%s' is a symlink", showPath(parent / e.path)); + } if (!dirFdOwning) { if (errno == ENOTDIR) @@ -422,8 +455,6 @@ try { } return dirFdOwning; -} catch (SymlinkNotAllowed & e) { - throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) @@ -497,17 +528,7 @@ try { if (path.isRoot()) throw NotASymlink("file '%s' is not a symlink", showPath(path)); - auto [parentFd, parentFdOwning] = openParent(path); - if (parentFd == INVALID_DESCRIPTOR) { - if (errno == ENOENT || errno == ENOTDIR) - throw FileNotFound("path '%s' does not exist", showPath(path)); - throw SysError("opening directory '%1%'", showPath(path.parent().value())); - } - - if (dirFdCache && parentFdOwning) { - assert(*parentFdOwning); - insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); - } + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/false); try { return readLinkAt(parentFd, CanonPath(path.baseName().value())); From 794f00a5776c03160147a3a58df5abffd9d9f2ee Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 03:01:40 +0300 Subject: [PATCH 307/555] Add CachingSourceAccessor for the sake of the evaluator, cache positive lstat/readlinks This avoids the footgun and issues around cache invalidation, while also keeping lookups more efficient. The evaluator already assumes a mostly immutable view of the filesystem (and we do nuke the caches during :r repl command). We used to cache lstat results because we do symlink resolution manually. Also adds the readDirectory override to the LocalStoreAccessor. --- src/libexpr/eval.cc | 2 + src/libstore/local-fs-store.cc | 8 ++ src/libutil/caching-source-accessor.cc | 102 ++++++++++++++++++ .../include/nix/util/source-accessor.hh | 6 ++ src/libutil/meson.build | 1 + 5 files changed, 119 insertions(+) create mode 100644 src/libutil/caching-source-accessor.cc diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index fb5d8d7a5ecb..7eb27f45b87d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -277,6 +277,8 @@ EvalState::EvalState( mounted fetchTree. */ auto accessor = settings.pureEval ? storeFS.cast() : makeUnionSourceAccessor({getFSSourceAccessor(), storeFS}); + /* Cache positive lstat/readlink results to speed up resolveSymlinks. */ + accessor = makeCachingSourceAccessor(accessor); /* Apply access control if needed. */ if (settings.restrictEval || settings.pureEval) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 12818f32a447..3fc724f5fe74 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -74,6 +74,14 @@ struct LocalStoreAccessor : SourceAccessor return accessor->readDirectory(path); } + void readDirectory( + const CanonPath & dirPath, + std::function callback) override + { + requireStoreObject(dirPath); + return accessor->readDirectory(dirPath, std::move(callback)); + } + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { requireStoreObject(path); diff --git a/src/libutil/caching-source-accessor.cc b/src/libutil/caching-source-accessor.cc new file mode 100644 index 000000000000..041fab0d621a --- /dev/null +++ b/src/libutil/caching-source-accessor.cc @@ -0,0 +1,102 @@ +#include "nix/util/source-accessor.hh" + +#include + +namespace nix { + +class CachingSourceAccessor : public SourceAccessor +{ + ref next; + + boost::concurrent_flat_map lstatCache; + boost::concurrent_flat_map readLinkCache; + +public: + CachingSourceAccessor(ref next_) + : next(std::move(next_)) + { + displayPrefix.clear(); + } + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override + { + next->readFile(path, sink, sizeCallback); + } + + std::optional maybeLstat(const CanonPath & path) override + { + if (auto res = getConcurrent(lstatCache, path)) + return *res; + + auto st = next->maybeLstat(path); + if (!st) + return std::nullopt; + + /* Never evict, the evaluator better keep positive lookups cached. */ + lstatCache.emplace(path, *st); + return st; + } + + Stat lstat(const CanonPath & path) override + { + if (auto res = getConcurrent(lstatCache, path)) + return *res; + + auto st = next->lstat(path); + /* Never evict, the evaluator better keep positive lookups cached. */ + lstatCache.emplace(path, st); + return st; + } + + DirEntries readDirectory(const CanonPath & path) override + { + return next->readDirectory(path); + } + + void readDirectory( + const CanonPath & dirPath, + std::function callback) override + { + return next->readDirectory(dirPath, std::move(callback)); + } + + std::string readLink(const CanonPath & path) override + { + if (auto res = getConcurrent(readLinkCache, path)) + return *res; + + auto target = next->readLink(path); + /* Never evict, the evaluator better keep positive lookups cached. */ + readLinkCache.emplace(path, target); + return target; + } + + std::string showPath(const CanonPath & path) override + { + return next->showPath(path); + } + + void invalidateCache() override + { + lstatCache.clear(); + readLinkCache.clear(); + next->invalidateCache(); + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return next->getPhysicalPath(path); + } + + std::pair> getFingerprint(const CanonPath & path) override + { + return next->getFingerprint(path); + } +}; + +ref makeCachingSourceAccessor(ref next) +{ + return make_ref(std::move(next)); +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index a2fbaac34993..cc076054e73e 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -288,4 +288,10 @@ ref makeFSSourceAccessor( */ ref makeUnionSourceAccessor(std::vector> && accessors); +/** + * Make a wrapper source accessor that caches positive lookup results. + * Useful for the evaluator which already assumes a mostly immutable view of the filesystem. + */ +ref makeCachingSourceAccessor(ref next); + } // namespace nix diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 50ade6688726..d132ce67c748 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -136,6 +136,7 @@ sources = [ config_priv_h ] + files( 'base-n.cc', 'base-nix-32.cc', 'bump-memory-resource.cc', + 'caching-source-accessor.cc', 'canon-path.cc', 'compression-algo.cc', 'compression-settings.cc', From 6c2f1b446c9f6d0acf09703d28ba10b895de0c6c Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 22:48:01 +0300 Subject: [PATCH 308/555] libexpr: Cache findFile lookups better We were doing a bunch of pointless I/O (even before dirfd accessor) for non-existent paths in nixPath for stuff like and repeated lookups. This caches positive and negative lookups and nukes the cache in resetFileCache for the purposes of repl. --- src/libexpr/eval.cc | 34 ++++++++++++++++++++++------ src/libexpr/include/nix/expr/eval.hh | 15 +++++++++--- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7eb27f45b87d..700d9a1ebe6b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1173,6 +1173,7 @@ void EvalState::resetFileCache() importResolutionCache->clear(); fileEvalCache->clear(); inputCache->clear(); + lookupPathResolved->clear(); positions.clear(); rootFS->invalidateCache(); } @@ -3269,14 +3270,28 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ continue; auto r = *rOpt; - auto res = (r / CanonPath(suffix)).resolveSymlinks(); - if (res.pathExists()) + auto suffixPath = CanonPath(suffix); + if (auto cachedRes = getConcurrent(*rOpt->resolvedPaths, suffixPath)) { + if (*cachedRes) + return **cachedRes; + else + // Cached negative lookup. + continue; + } + + auto res = (r.path / suffixPath).resolveSymlinks(); + if (res.pathExists()) { + r.resolvedPaths->emplace(suffixPath, res); return res; + } // Backward compatibility hack: throw an exception if access // to this path is not allowed. if (auto accessor = res.accessor.dynamic_pointer_cast()) accessor->checkAccess(res.path); + + // Cache negative lookups too. + r.resolvedPaths->emplace(suffixPath, std::nullopt); } if (hasPrefix(path, "nix/")) @@ -3290,17 +3305,22 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ .debugThrow(); } -std::optional EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) +std::shared_ptr +EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) { auto & value = value0.s; if (auto cached = getConcurrent(*lookupPathResolved, value)) return *cached; - auto finish = [&](std::optional res) { - if (res) - debug("resolved search path element '%s' to '%s'", value, *res); - else + auto finish = [&](std::optional maybePath) { + std::shared_ptr res; + if (maybePath) { + debug("resolved search path element '%s' to '%s'", value, *maybePath); + res = std::make_shared( + *maybePath, make_ref()); + } else { debug("failed to resolve search path element '%s'", value); + } lookupPathResolved->emplace(std::string(value), res); return res; }; diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 89e2d5099da2..20765f85546b 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -489,7 +489,15 @@ private: LookupPath lookupPath; - const ref, StringViewHash, std::equal_to<>>> + struct LookupPathResolvedState + { + SourcePath path; + const ref>> resolvedPaths; + }; + + const ref< + boost:: + concurrent_flat_map, StringViewHash, std::equal_to<>>> lookupPathResolved; /** @@ -626,9 +634,10 @@ public: * * If the specified search path element is a URI, download it. * - * If it is not found, return `std::nullopt`. + * If it is not found, return `nullptr`. */ - std::optional resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); + std::shared_ptr + resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); /** * Evaluate an expression to normal form From bae48a5ba2ac21d4936f2a8d68aba9c603746a00 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Apr 2026 15:47:56 +0200 Subject: [PATCH 309/555] Input::getAccessorUnchecked(): Wrap fetches in a path lock This prevents multiple processes (like nix-eval-jobs instances) from fetching the same input at the same time. That doesn't matter for correctness, but it can cause a lot of redundant downloads. --- src/libfetchers/fetchers.cc | 16 ++++++++++++++++ tests/functional/tarball.sh | 14 ++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index fb87f9b94506..c47b07c43b47 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -7,6 +7,9 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/url.hh" +#include "nix/util/users.hh" +#include "nix/store/pathlocks.hh" +#include "nix/util/environment-variables.hh" #include @@ -301,6 +304,19 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); + /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up + * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. */ + auto lockFilePath = + getCacheDir() / "fetcher-locks" + / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); + std::filesystem::create_directories(lockFilePath.parent_path()); + PathLocks lock( + {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); + + static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; + if (inTest) + std::this_thread::sleep_for(std::chrono::seconds(1)); + /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. We only do this for final diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index e7d9b96fda33..6deb7b96b7c6 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -116,3 +116,17 @@ path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" # Test that unpacking an empty file does not segfault (see https://github.com/NixOS/nix/issues/15116). touch "$TEST_ROOT/empty" expectStderr 1 nix store prefetch-file --unpack "file://$TEST_ROOT/empty" | grepQuiet "archive.*is empty" + +# Test that concurrent invocations of Nix will fetch the tarball only once. +rm -rf "$TEST_HOME/.cache" +store="$TEST_ROOT/prefetch-store" +nix-store --store "$store" --init # needed because concurrent creation of the store can give SQLite errors +_NIX_TEST_CONCURRENT_FETCHES=1 _NIX_FORCE_HTTP=1 nix flake prefetch --store "$store" -v "tarball+file://$TEST_ROOT/tar.tar" 2> "$TEST_ROOT/log1" & +pid1="$!" +_NIX_TEST_CONCURRENT_FETCHES=1 _NIX_FORCE_HTTP=1 nix flake prefetch --store "$store" -v "tarball+file://$TEST_ROOT/tar.tar" 2> "$TEST_ROOT/log2" & +pid2="$!" +wait "$pid1" +wait "$pid2" +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "Download.*to") -eq 2 ]] +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "downloading.*tar.tar") -eq 1 ]] +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "waiting for another Nix process to finish fetching input") -eq 1 ]] From eb5c6775512df79d557687fe259f9223fbc5e348 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 23 Apr 2026 23:35:26 +0300 Subject: [PATCH 310/555] libfetchers: Acquire fetcher-locks only when input is not substituted When substituting an input, the store in question already acquires internal PathLocks on the to-be-substituted store path. Acquiring a lock eagerly is pessimising the case where multiple stores are substituted to concurrenty (but with the same cache dir). Also changes std::filesystem::create_directories to createDirs which wraps exceptions nicely and adds a setDeletion() call to clean up lock files. --- src/libfetchers/fetchers.cc | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c47b07c43b47..4eedafa22a24 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -304,19 +304,6 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); - /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up - * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. */ - auto lockFilePath = - getCacheDir() / "fetcher-locks" - / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); - std::filesystem::create_directories(lockFilePath.parent_path()); - PathLocks lock( - {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); - - static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; - if (inTest) - std::this_thread::sleep_for(std::chrono::seconds(1)); - /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. We only do this for final @@ -358,6 +345,21 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings } } + /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up + * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. Note that we don't need + * this when substituting the input. */ + auto lockFilePath = + getCacheDir() / "fetcher-locks" + / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); + createDirs(lockFilePath.parent_path()); + PathLocks lock( + {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); + lock.setDeletion(true); + + static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; + if (inTest) + std::this_thread::sleep_for(std::chrono::seconds(1)); + auto [accessor, result] = scheme->getAccessor(settings, store, *this); if (!accessor->fingerprint) From 5f5fb7a3a538525200505d65762d44bc6df7ffad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Apr 2026 18:14:10 +0200 Subject: [PATCH 311/555] RemoteStore::addToStore(): Fix version comparison --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 2def1bf87aa7..88c3847bb3cf 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -460,7 +460,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, Repair void RemoteStore::addMultipleToStore( PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) { - if (getConnection()->protoVersion < WorkerProto::Version{.number = {1, 32}}) { + if (getConnection()->protoVersion.number < WorkerProto::Version::Number{1, 32}) { Store::addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs); return; } From f63e1383143ac15e4e47b975c629f4298a8d9e0a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 24 Apr 2026 22:51:07 +0300 Subject: [PATCH 312/555] Tune error messages --- src/libutil/posix-source-accessor.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 661922cd39cc..b5921381d313 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -346,7 +346,7 @@ std::pair> PosixDirectorySourceAccessor return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; } catch (SymlinkNotAllowed & e) { /* Need to fixup the error message to include the actual path relative to the (possibly) cached fd. */ - throw SymlinkNotAllowed(anchor / e.path, "path '%s' is a symlink", showPath(anchor / e.path)); + throw SymlinkNotAllowed(anchor / e.path, "path '%s' (or its ancestor) is a symlink", showPath(anchor / e.path)); } } @@ -420,7 +420,7 @@ void PosixDirectorySourceAccessor::readFile(const CanonPath & path, Sink & sink, auto st = nix::fstat(fileFd.get()); if (!S_ISREG(st.st_mode)) - throw Error("file '%s' has an unsupported type", showPath(path)); + throw NotARegularFile("file '%s' is not a regular file", showPath(path)); PosixFileSourceAccessor fileAccessor(std::move(fileFd), fsPath / path.rel(), trackLastModified, st); maybeUpdateMtime(st.st_mtime); fileAccessor.readFile(CanonPath::root, sink, sizeCallback); From fb4d488dd55119703411aced25385a7346791ff2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 14:26:17 +0300 Subject: [PATCH 313/555] libflake: Drop unused NixStringContext in getFlake This was some leftovers from detnix cherry-pick, we ban string contexts in getFlake and have always. --- src/libflake/flake-primops.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 3ad66611726b..1fc4f7ccbde9 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -42,7 +42,6 @@ PrimOp getFlake(const Settings & settings) auto path = state.realisePath(pos, *args[0]); callFlake(state, lockFlake(settings, state, path, lockFlags), v); } else { - NixStringContext context; std::string flakeRefS( state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); From 569ee752c341db5de5fa63962d01f6a5d4cb995e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:20:34 +0300 Subject: [PATCH 314/555] libexpr: Add a way to collect string context from ValuePrinter --- src/libexpr/include/nix/expr/print.hh | 13 +++++++++++-- src/libexpr/print.cc | 15 ++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/libexpr/include/nix/expr/print.hh b/src/libexpr/include/nix/expr/print.hh index 229f7159d15a..8e6d0f9bf09a 100644 --- a/src/libexpr/include/nix/expr/print.hh +++ b/src/libexpr/include/nix/expr/print.hh @@ -10,6 +10,7 @@ #include #include "nix/util/fmt.hh" +#include "nix/expr/value/context.hh" #include "nix/expr/print-options.hh" namespace nix { @@ -64,7 +65,12 @@ bool isReservedKeyword(const std::string_view str); */ std::ostream & printIdentifier(std::ostream & o, std::string_view s); -void printValue(EvalState & state, std::ostream & str, Value & v, PrintOptions options = PrintOptions{}); +void printValue( + EvalState & state, + std::ostream & str, + Value & v, + PrintOptions options = PrintOptions{}, + NixStringContext * context = nullptr); /** * A partially-applied form of `printValue` which can be formatted using `<<` @@ -77,12 +83,15 @@ private: EvalState & state; Value & value; PrintOptions options; + NixStringContext * context; public: - ValuePrinter(EvalState & state, Value & value, PrintOptions options = PrintOptions{}) + ValuePrinter( + EvalState & state, Value & value, PrintOptions options = PrintOptions{}, NixStringContext * context = nullptr) : state(state) , value(value) , options(options) + , context(context) { } }; diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index f2f62a636982..0c95ae60c5b6 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -162,6 +162,7 @@ class Printer std::ostream & output; EvalState & state; PrintOptions options; + NixStringContext * context; std::optional seen; size_t totalAttrsPrinted = 0; size_t totalListItemsPrinted = 0; @@ -577,9 +578,12 @@ class Printer printBool(v); break; - case nString: + case nString: { printString(v); + if (context) + copyContext(v, *context); break; + } case nPath: printPath(v); @@ -632,10 +636,11 @@ class Printer } public: - Printer(std::ostream & output, EvalState & state, PrintOptions options) + Printer(std::ostream & output, EvalState & state, PrintOptions options, NixStringContext * context) : output(output) , state(state) , options(options) + , context(context) { } @@ -656,14 +661,14 @@ class Printer } }; -void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options) +void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options, NixStringContext * context) { - Printer(output, state, options).print(v); + Printer(output, state, options, context).print(v); } std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer) { - printValue(printer.state, output, printer.value, printer.options); + printValue(printer.state, output, printer.value, printer.options, printer.context); return output; } From a8e2e0022732edc2602ad6849dd333213def2621 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Sat, 25 Apr 2026 13:26:37 -0700 Subject: [PATCH 315/555] linux-derivation-builder: Also block *listxattr Not blocking *listxattr not only leaks information from the host filesystem (e.g. leaks the fact that SELinux is enabled through the presence of security.selinux in the xattrs) but can confuse applications that assume that if *listxattr succeeds, xattrs are supported. For example, mkfs.ubifs will call llistxattr to get a list of attributes with a check for errno == EOPNOTSUPP to detect xattrs being unsupported. If that succeeds and it finds xattrs (which it will on an SELinux system), it calls lgetxattr on the individual attributes, which fails, causing mkfs.ubifs to fail. --- src/libstore/unix/build/linux-derivation-builder.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 0bc32a91e648..1f636ec56f2f 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -116,7 +116,10 @@ static void setupSeccomp(const LocalSettings & localSettings) /* Prevent builders from using EAs or ACLs. Not all filesystems support these, and they're not allowed in the Nix store because they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(listxattr), 0) != 0 + || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(llistxattr), 0) != 0 + || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(flistxattr), 0) != 0 + || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 From 891ef140b8564a7848a3d75976e172c3e15ec14b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:46:58 +0300 Subject: [PATCH 316/555] Don't copy flakes to the store unnecessarily This builds on top of https://github.com/NixOS/nix/pull/14050 to actually make flakes get lazily copied to the store. This repurposes a slightly less lazy (but also more deterministic) approach than determinate nix has taken. We do still pay to cost of hashing an input once to compute the store path and narHash daemon-client-side. This could be improved in follow-ups in case we don't actually need to check the narHash (like during local development). We need certain backwards compatibility hacks for getFlake with a discarded string context, those are similar to what detnix does. See: https://github.com/DeterminateSystems/nix-src/pull/422 See: https://github.com/DeterminateSystems/nix-src/pull/402 Co-authored-by: Eelco Dolstra --- src/libcmd/installable-value.cc | 5 +- src/libexpr/include/nix/expr/eval.hh | 27 ++++++++++- .../include/nix/expr/print-ambiguous.hh | 8 +++- src/libexpr/paths.cc | 47 ++++++++++++++++++- src/libexpr/primops.cc | 36 ++++++++++---- src/libexpr/print-ambiguous.cc | 14 ++++-- src/libflake/flake-primops.cc | 17 +++++++ src/nix/app.cc | 2 + src/nix/eval.cc | 16 ++++--- src/nix/flake.cc | 11 +++-- src/nix/nix-instantiate/nix-instantiate.cc | 4 +- tests/functional/flakes/flakes.sh | 1 - 12 files changed, 159 insertions(+), 29 deletions(-) diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 3a167af3db49..92811c1d01da 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -54,8 +54,11 @@ InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std:: } else if (v.type() == nString) { + auto path = state->coerceToSingleDerivedPath(pos, v, errorCtx); + if (auto o = std::get_if(&path.raw())) + state->ensureLazyPathCopied(o->path); return {{ - .path = DerivedPath::fromSingle(state->coerceToSingleDerivedPath(pos, v, errorCtx)), + .path = DerivedPath::fromSingle(path), .info = make_ref(), }}; } diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 20765f85546b..1283b27cf822 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -738,6 +738,26 @@ public: std::optional tryAttrsToString( const PosIdx pos, Value & v, NixStringContext & context, bool coerceMore = false, bool copyToStore = true); + enum class CopyLazyPaths : bool { + PreserveLazy = false, + Copy = true, + }; + + /** + * For efficiency reasons, some store paths (as seen by the evaluator) in + * the storeFS at their content-addressed locations don't get copied to the + * store eagerly. This saves on needless I/O and possibly IPC if all the + * evaluator does is just evaluate nix expressions from those locations. + * This function copies such store objects to the store if they aren't already valid. + */ + void ensureLazyPathCopied(const StorePath & path); + + /** + * Ensure that all NixStringContextElem::Opaque context elements get fetched + * to the store. + */ + void ensureLazyPathsCopied(const NixStringContext & context); + /** * String coercion. * @@ -1044,9 +1064,14 @@ public: /** * Coerce `v` to a path and realise it, i.e. build anything in the value's string context using `realiseContext()`. + * @param copyLazyPaths When encountering a lazy path (i.e. a string with Opaque context that's also "mounted" on + * the storeFS), fetch the store path to the store. */ SourcePath realisePath( - const PosIdx pos, Value & v, std::optional resolveSymlinks = SymlinkResolution::Full); + const PosIdx pos, + Value & v, + std::optional resolveSymlinks = SymlinkResolution::Full, + CopyLazyPaths copyLazyPaths = CopyLazyPaths::PreserveLazy); /** * Realise the given string with context, and return the string with outputs instead of downstream output diff --git a/src/libexpr/include/nix/expr/print-ambiguous.hh b/src/libexpr/include/nix/expr/print-ambiguous.hh index 7e44a6b66ebc..07fcc337e355 100644 --- a/src/libexpr/include/nix/expr/print-ambiguous.hh +++ b/src/libexpr/include/nix/expr/print-ambiguous.hh @@ -17,6 +17,12 @@ class EvalState; * * See: https://github.com/NixOS/nix/issues/9730 */ -void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, size_t depth = 0); +void printAmbiguous( + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, + NixStringContext * context = nullptr, + size_t depth = 0); } // namespace nix diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index ca303208173e..ff09b7aaa5e0 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -22,10 +22,55 @@ SourcePath EvalState::storePath(const StorePath & path) return {rootFS, CanonPath{store->printStorePath(path)}}; } +void EvalState::ensureLazyPathCopied(const StorePath & path) +{ + if (settings.isReadOnly()) + return; + + auto mount = storeFS->getMount(CanonPath(store->printStorePath(path))); + if (!mount) + return; + + /* TODO: We could memoise this in-memory if necessary. */ + auto storePath = fetchToStore( + fetchSettings, + *store, + SourcePath{ref(mount)}, + /* Force a copy. mountInput does a dryRun to just calculate the storePath and narHash. */ + FetchMode::Copy, + path.name()); + + /* Catch hash mismatches more loudly. This is more likely caused by unsound + caching of different accessor types that fetch the same repo with + the same git revision, but with different kinds of accessors (think + tarball-based fetchers vs local/remote git accessors). */ + if (storePath != path) { + panic(fmt( + "hashed store path computed by the evaluator ('%1%') does not match what was computed when copying to the store ('%2%'), this is a bug", + store->printStorePath(path), + store->printStorePath(storePath))); + } +} + +void EvalState::ensureLazyPathsCopied(const NixStringContext & context) +{ + for (const auto & c : context) + if (auto * o = std::get_if(&c.raw)) + /* TODO: This could be done in parallel. */ + ensureLazyPathCopied(o->path); +} + StorePath EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref accessor) { - auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); + /* To mount the input, dryRun is sufficient. We still compute the narHash (to check for mismatches) and the store + path to figure out where to mount it. TODO: This could be relaxed in the future by making outPath and narHash + lazier. Good code that doesn't do `toString ./.` or otherwise inspects the outPath string and only uses it for + doing relative imports does not even require computing the store path. That is a big invasive change though and + would require having a special "LazyStorePathString" thunk. narHash also doesn't need to be computed eagerly in + case it's not actually specified (like during local development with a dirty tree) - in that case narHash could + also become a lazy app/thunk that shares the state with the storePath delayed computation. */ + auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::DryRun, input.getName()); allowPath(storePath); // FIXME: should just whitelist the entire virtual store diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4b7680be1ff6..0c472fb10217 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "nix/store/names.hh" #include "nix/store/path-references.hh" #include "nix/store/store-api.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/util/util.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" @@ -63,7 +64,7 @@ std::string EvalState::realiseString(Value & s, StorePathSet * storePathsOutMayb nix::NixStringContext stringContext; auto rawStr = coerceToString(pos, s, stringContext, "while realising a string").toOwned(); auto rewrites = realiseContext(stringContext, storePathsOutMaybe, isIFD); - + ensureLazyPathsCopied(stringContext); return nix::rewriteStrings(rawStr, rewrites); } @@ -88,7 +89,11 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { - ensureValid(o.path); + /* If the path happens to be mounted on the storeFS, that means it's lazy path string and would get + copied to the store on-demand (when referenced in a derivation). The string is equal to final + store path where the store object would end up (the path is hashed before mounting). */ + if (!storeFS->getMount(CanonPath(store->printStorePath(o.path)))) + ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); }, @@ -158,7 +163,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS return res; } -SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optional resolveSymlinks) +SourcePath EvalState::realisePath( + const PosIdx pos, Value & v, std::optional resolveSymlinks, CopyLazyPaths copyLazyPaths) { NixStringContext context; @@ -167,6 +173,8 @@ SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optional(&c.raw)) + if (auto p = std::get_if(&c.raw)) { + state.ensureLazyPathCopied(p->path); refs.insert(p->path); - else + } else state .error( "files created by %1% may not reference derivations, but %2% references %3%", diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index ed91cad85a47..b0a5224f26e6 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -7,7 +7,13 @@ namespace nix { // See: https://github.com/NixOS/nix/issues/9730 -void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, size_t depth) +void printAmbiguous( + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, + NixStringContext * context, + size_t depth) { checkInterrupt(); @@ -22,6 +28,8 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::setlexicographicOrder(state.symbols)) { str << state.symbols[i->name] << " = "; - printAmbiguous(state, *i->value, str, seen, depth + 1); + printAmbiguous(state, *i->value, str, seen, context, depth + 1); str << "; "; } str << "}"; @@ -52,7 +60,7 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::setisInStore(sourcePath->string())) { + auto [storePath, subPath] = state.store->toStorePath(sourcePath->string()); + if (auto mount = state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath)))) { + auto path = state.storePath(storePath) / CanonPath(subPath); + if (!flakeRef.subdir.empty()) + path = path / flakeRef.subdir; + return callFlake(state, lockFlake(settings, state, path, lockFlags), v); + } + } + callFlake(state, lockFlake(settings, state, flakeRef, lockFlags), v); } }; diff --git a/src/nix/app.cc b/src/nix/app.cc index 634db04f3fe1..167ffd8d86f8 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -95,6 +95,8 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) c.raw)); } + state.ensureLazyPathsCopied(context); + return UnresolvedApp{App{ .context = std::move(context2), .program = program, diff --git a/src/nix/eval.cc b/src/nix/eval.cc index ed6fe41be655..6b8c0ba12234 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -83,10 +83,10 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption [&](this const auto & recurse, Value & v, const PosIdx pos, const std::filesystem::path & path) -> void { state->forceValue(v, pos); - if (v.type() == nString) - // FIXME: disallow strings with contexts? + if (v.type() == nString) { + copyContext(v, context); writeFile(path, v.string_view()); - else if (v.type() == nAttrs) { + } else if (v.type() == nAttrs) { [[maybe_unused]] bool directoryCreated = std::filesystem::create_directory(path); // Directory should not already exist assert(directoryCreated); @@ -110,9 +110,8 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption else if (raw) { logger->stop(); - writeFull( - getStandardOutput(), - *state->coerceToString(noPos, *v, context, "while generating the eval command output")); + auto string = state->coerceToString(noPos, *v, context, "while generating the eval command output"); + writeFull(getStandardOutput(), *string); } else if (json) { @@ -120,8 +119,11 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption } else { - logger->cout("%s", ValuePrinter(*state, *v, PrintOptions{.force = true, .derivationPaths = true})); + ValuePrinter printer(*state, *v, PrintOptions{.force = true, .derivationPaths = true}, &context); + logger->cout("%s", printer); } + + state->ensureLazyPathsCopied(context); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2a5239549e0f..53719f5ec7f7 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,6 +7,7 @@ #include "nix/expr/get-drvs.hh" #include "nix/util/os-string.hh" #include "nix/util/signals.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/store/store-open.hh" #include "nix/store/derivations.hh" #include "nix/store/outputs-spec.hh" @@ -215,8 +216,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON auto lockedFlake = lockFlake(); auto & flake = lockedFlake.flake; - // Currently, all flakes are in the Nix store via the rootFS accessor. - auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first); + /* Flakes do not get copied to the store, but are instead mounted at + their expected store paths in storeFS. Querying metadata does not + force copying to the store, as one would expect. */ + auto storePath = store->toStorePath(flake.path.path.abs()).first; if (json) { nlohmann::json j; @@ -238,7 +241,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = storePath; + j["path"] = store->printStorePath(storePath); j["locks"] = lockedFlake.lockFile.toJSON().first; if (auto fingerprint = lockedFlake.getFingerprint(*store, fetchSettings)) j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); @@ -249,7 +252,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout(ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", flake.lockedRef.to_string()); if (flake.description) logger->cout(ANSI_BOLD "Description:" ANSI_NORMAL " %s", *flake.description); - logger->cout(ANSI_BOLD "Path:" ANSI_NORMAL " %s", storePath); + logger->cout(ANSI_BOLD "Path:" ANSI_NORMAL " %s", store->printStorePath(storePath)); if (auto rev = flake.lockedRef.input.getRev()) logger->cout(ANSI_BOLD "Revision:" ANSI_NORMAL " %s", rev->to_string(HashFormat::Base16, false)); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) diff --git a/src/nix/nix-instantiate/nix-instantiate.cc b/src/nix/nix-instantiate/nix-instantiate.cc index 27a11767b004..dee79bcbfc2c 100644 --- a/src/nix/nix-instantiate/nix-instantiate.cc +++ b/src/nix/nix-instantiate/nix-instantiate.cc @@ -66,7 +66,7 @@ void processExpr( if (strict) state.forceValueDeep(vRes); std::set seen; - printAmbiguous(state, vRes, std::cout, &seen); + printAmbiguous(state, vRes, std::cout, &seen, &context); std::cout << std::endl; } } else { @@ -94,6 +94,8 @@ void processExpr( std::cout << fmt("%s%s\n", drvPathS, (outputName != "out" ? "!" + outputName : "")); } } + + state.ensureLazyPathsCopied(context); } } diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 8d95f61240bd..fec80cc89529 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -69,7 +69,6 @@ nix flake metadata "$flake1Dir" | grepQuiet 'URL:.*flake1.*' # Test 'nix flake metadata --json'. json=$(nix flake metadata flake1 --json | jq .) [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] -[[ -d $(echo "$json" | jq -r .path) ]] [[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]] hash1=$(echo "$json" | jq -r .revision) [[ -n $(echo "$json" | jq -r .fingerprint) ]] From d4ea1a07d2c14042e9e2cb359fa4efb39a244127 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Thu, 23 Apr 2026 22:21:48 -0400 Subject: [PATCH 317/555] Enable more clang-tidy checks, related fixes Signed-off-by: Lisanna Dettwyler --- .../common/clang-tidy/.clang-tidy | 17 +++++++++++------ src/libexpr-tests/nix_api_expr.cc | 4 ++-- src/libexpr/eval-cache.cc | 2 +- src/libexpr/eval-error.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/get-drvs.cc | 6 +++--- src/libexpr/include/nix/expr/diagnose.hh | 4 ++-- src/libexpr/include/nix/expr/nixexpr.hh | 2 +- src/libexpr/include/nix/expr/parser-state.hh | 2 +- src/libexpr/primops.cc | 2 +- src/libstore/build/derivation-goal.cc | 4 ++++ src/libstore/build/goal.cc | 4 ++-- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/build/worker.cc | 2 +- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/include/nix/store/build/goal.hh | 4 ++-- src/libstore/include/nix/store/sqlite.hh | 2 +- src/libstore/misc.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/sqlite.cc | 2 +- src/libstore/store-api.cc | 18 ++++++++---------- src/libstore/unix/build/child.cc | 2 +- src/libstore/unix/build/derivation-builder.cc | 4 ++-- src/libutil-tests/file-descriptor.cc | 4 ++-- src/libutil/file-descriptor.cc | 1 + src/libutil/include/nix/util/async.hh | 4 ++-- src/libutil/include/nix/util/closure.hh | 2 +- src/libutil/include/nix/util/configuration.hh | 2 +- src/libutil/include/nix/util/error.hh | 2 +- .../include/nix/util/file-descriptor.hh | 1 + src/libutil/include/nix/util/serialise.hh | 1 + src/libutil/include/nix/util/sort.hh | 12 ++++++------ src/libutil/include/nix/util/topo-sort.hh | 2 +- src/libutil/include/nix/util/util.hh | 2 +- src/libutil/linux/linux-namespaces.cc | 4 ++-- src/libutil/logging.cc | 12 +++++++++--- src/libutil/unix/file-system.cc | 2 +- 37 files changed, 80 insertions(+), 64 deletions(-) diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy index daf4c7f7c4b8..b0cc92429c4e 100644 --- a/nix-meson-build-support/common/clang-tidy/.clang-tidy +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -51,11 +51,6 @@ Checks: - -bugprone-macro-parentheses # 1 warning - increment/decrement in conditions - -bugprone-inc-dec-in-conditions - # 2 warnings - std::move on forwarding reference (auto&&) in ranges lambdas - - -bugprone-move-forwarding-reference - # 2 warnings - coroutine pattern: co_await await(std::move(waitees)) then reuse. - # Relies on moved-from containers being empty (holds for libstdc++/libc++). - - -bugprone-use-after-move # 2 warnings - sorts Value* by ->string_view(), not by pointer value (false positive) - -bugprone-nondeterministic-pointer-iteration-order # 9 warnings - intentional std::bit_cast/memcpy on Value* arrays (evaluator hot path) @@ -68,7 +63,7 @@ Checks: # template; fires when T=unsigned char but that instantiation is correct. - -bugprone-unintended-char-ostream-output # - # Non-bugprone checks (also disabled to pass on current codebase): + # Non-bugprone checks (some disabled to pass on current codebase): # # 4 warnings - exceptions not derived from std::exception # All thrown exceptions must derive from std::exception @@ -77,6 +72,14 @@ Checks: # - cppcoreguidelines-pro-type-cstyle-cast # 11 warnings - coroutine lambdas with captures (intentional pattern in async goal/store code) # - cppcoreguidelines-avoid-capturing-lambda-coroutines + - performance-noexcept-swap + - performance-noexcept-move-constructor + - performance-noexcept-destructor + - performance-use-std-move + - misc-throw-by-value-catch-by-reference + - cppcoreguidelines-missing-std-forward + - android-cloexec-open + - android-cloexec-pipe2 # Custom nix checks (when added) - nix-* @@ -93,3 +96,5 @@ CheckOptions: bugprone-unsafe-functions.CustomFunctions: > ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; ::std::filesystem::remove_all, nix::deletePath, "Use nix::deletePath (remove_all is not TOCTOU safe)"; + +ExtraArgs: ["-Werror=unnecessary-virtual-specifier"] diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index c3a3f2dd53b1..8362e6850892 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -20,8 +20,8 @@ TEST_F(nix_api_expr_test, nix_eval_state_lookup_path) auto delTmpDir = std::make_unique(tmpDir, true); auto nixpkgs = tmpDir / "pkgs"; auto nixos = tmpDir / "cfg"; - std::filesystem::create_directories(nixpkgs); - std::filesystem::create_directories(nixos); + nix::createDirs(nixpkgs); + nix::createDirs(nixos); std::string nixpkgsEntry = "nixpkgs=" + nixpkgs.string(); std::string nixosEntry = "nixos-config=" + nixos.string(); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index a419530c6dd8..63f0c148870c 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -106,7 +106,7 @@ struct AttrDb } template - AttrId doSQLite(F && fun) + AttrId doSQLite(const F & fun) { if (failed) return 0; diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 67e8d5ea1ded..22c622526c46 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -97,7 +97,7 @@ void EvalErrorBuilder::debugThrow() auto error = std::move(this->error); delete this; - throw error; + throw std::move(error); } template diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 700d9a1ebe6b..4cd90a2a5f84 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3465,7 +3465,7 @@ void forceNoNullByte(std::string_view s, std::function pos) if (pos) { error.atPos(pos()); } - throw error; + throw std::move(error); } } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 693b1946ee46..1acc591b05b7 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -162,14 +162,14 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) - throw errMsg; + throw std::move(errMsg); Outputs result; for (auto elem : outTI->listView()) { if (elem->type() != nString) - throw errMsg; + throw std::move(errMsg); auto out = outputs.find(elem->string_view()); if (out == outputs.end()) - throw errMsg; + throw std::move(errMsg); result.insert(*out); } return result; diff --git a/src/libexpr/include/nix/expr/diagnose.hh b/src/libexpr/include/nix/expr/diagnose.hh index 68c8f6543f59..4a360970ba21 100644 --- a/src/libexpr/include/nix/expr/diagnose.hh +++ b/src/libexpr/include/nix/expr/diagnose.hh @@ -44,7 +44,7 @@ NIX_DECLARE_CONFIG_SERIALISER(Diagnose) * @throws The error returned by mkError if level is `Fatal` and mkError returns a value */ template -void diagnose(const Setting & setting, F && mkError) +void diagnose(const Setting & setting, const F & mkError) { auto withError = [&](bool fatal, auto && handler) { auto maybeError = mkError(fatal); @@ -64,7 +64,7 @@ void diagnose(const Setting & setting, F && mkError) withError(false, [](auto && error) { logWarning(error.info()); }); return; case Diagnose::Fatal: - withError(true, [](auto && error) { throw std::move(error); }); + withError(true, [](auto && error) { throw std::forward(error); }); return; } } diff --git a/src/libexpr/include/nix/expr/nixexpr.hh b/src/libexpr/include/nix/expr/nixexpr.hh index 07fbed403c2f..b13c00ad541e 100644 --- a/src/libexpr/include/nix/expr/nixexpr.hh +++ b/src/libexpr/include/nix/expr/nixexpr.hh @@ -557,7 +557,7 @@ public: std::numeric_limits::max()); if (pos) err.atPos(positions[pos]); - throw err; + throw std::move(err); } std::uninitialized_copy_n(formals.formals.begin(), nFormals, formalsStart); }; diff --git a/src/libexpr/include/nix/expr/parser-state.hh b/src/libexpr/include/nix/expr/parser-state.hh index f9bd06589e42..2482d53ea041 100644 --- a/src/libexpr/include/nix/expr/parser-state.hh +++ b/src/libexpr/include/nix/expr/parser-state.hh @@ -89,7 +89,7 @@ public: * @see https://github.com/NixOS/nix/issues/14642 */ template - void visit(F && f) + void visit(const F & f) { std::visit( overloaded{ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4b7680be1ff6..5683d13d5c4b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -694,7 +694,7 @@ static RegisterPrimOp primop_isPath({ }); template -static inline void withExceptionContext(Trace trace, Callable && func) +static inline void withExceptionContext(Trace trace, const Callable & func) { try { func(); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6d76cd9d275e..fc4c8a5bb507 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -92,6 +92,8 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) co_await await(std::move(waitees)); if (nrFailed == 0) { + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath))); co_await await(std::move(waitees)); @@ -111,6 +113,8 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) } } + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) co_await await(std::move(waitees)); trace("all outputs substituted (maybe)"); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 7651282e3e4f..af245d90187c 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -58,13 +58,13 @@ Goal::ChildEvent Goal::ChildEvents::popChildEvent() using handle_type = nix::Goal::handle_type; using Suspend = nix::Goal::Suspend; -Co::Co(Co && rhs) +Co::Co(Co && rhs) noexcept { this->handle = rhs.handle; rhs.handle = nullptr; } -Co & Co::operator=(Co && rhs) +Co & Co::operator=(Co && rhs) noexcept { if (handle) { handle.promise().alive = false; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 4cb42975fe29..90273493e288 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -146,7 +146,7 @@ Goal::Co PathSubstitutionGoal::init() } if (lastStoresException.has_value()) { if (!worker.settings.tryFallback) { - throw *lastStoresException; + throw std::move(*lastStoresException); } else logError(lastStoresException->info()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 002bb1a30d79..3fe196d08005 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -61,7 +61,7 @@ std::shared_ptr Worker::initGoalIfNeeded(std::weak_ptr & goal_weak, Args & if (auto goal = goal_weak.lock()) return goal; - auto goal = std::make_shared(args...); + auto goal = std::make_shared(std::forward(args)...); goal_weak = goal; wakeUp(goal); return goal; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index ff5a89f135bb..713de4969a94 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -214,7 +214,7 @@ void HttpBinaryCacheStore::upsertFile( } catch (FileTransferError & e) { UploadToHTTP err(e.message()); err.addTrace({}, "while uploading to HTTP binary cache at '%s'", config->cacheUri.to_string()); - throw err; + throw std::move(err); } } diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 6ddc73250d34..3e5a3139d5ff 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -255,8 +255,8 @@ public: explicit Co(handle_type handle) : handle(handle) {}; - Co & operator=(Co &&); - Co(Co && rhs); + Co & operator=(Co &&) noexcept; + Co(Co && rhs) noexcept; Co & operator=(const Co &) = delete; Co(const Co & rhs) = delete; ~Co(); diff --git a/src/libstore/include/nix/store/sqlite.hh b/src/libstore/include/nix/store/sqlite.hh index 789e82174627..0a2d21b12849 100644 --- a/src/libstore/include/nix/store/sqlite.hh +++ b/src/libstore/include/nix/store/sqlite.hh @@ -207,7 +207,7 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning); * database is busy. */ template -T retrySQLite(F && fun) +T retrySQLite(const F & fun) { time_t nextWarning = time(nullptr) + 1; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 51708a2cbce4..8db977b6fce0 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -149,7 +149,7 @@ querySubstitutablePathInfosAsync(Store & store, const StorePathCAMap & paths, Su } if (lastStoresException.has_value()) { if (!settings.getWorkerSettings().tryFallback) { - throw *lastStoresException; + throw std::move(*lastStoresException); } else logError(lastStoresException->info()); } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 733157524176..f6d300c0936f 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -173,7 +173,7 @@ void S3BinaryCacheStore::upsertFile( } catch (FileTransferError & e) { UploadToS3 err(e.message()); err.addTrace({}, "while uploading to S3 binary cache at '%s'", config->cacheUri.to_string()); - throw err; + throw std::move(err); } } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 3fd1f798ea74..c65fc240c66f 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -47,7 +47,7 @@ SQLiteError::SQLiteError( exp.err.msg = HintFmt( err == SQLITE_PROTOCOL ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" : "SQLite database '%s' is busy", path ? path : "(in-memory)"); - throw exp; + throw std::move(exp); } else throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf)); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3f367b19a16c..3c23156769b7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -55,21 +55,19 @@ StoreConfigBase::StoreDirSetting::StoreDirSetting(Config * options, FilePathType switch (pathType) { case FilePathType::Unix: - return canonStoreDir( - envOverrides.transform([](auto && s) { return os_string_to_string(std::move(s)); }) - .value_or(NIX_STORE_DIR)); + return canonStoreDir(envOverrides.transform([](const auto & s) { return os_string_to_string(s); }) + .value_or(NIX_STORE_DIR)); case FilePathType::Native: - return canonStoreDir( - envOverrides.transform([](auto && s) { return std::filesystem::path(std::move(s)); }) - .or_else([]() -> std::optional { + return canonStoreDir(envOverrides.transform([](const auto & s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { #ifdef _WIN32 - return windows::known_folders::getProgramData() / "nix" / "store"; + return windows::known_folders::getProgramData() / "nix" / "store"; #else - return std::filesystem::path{NIX_STORE_DIR}; + return std::filesystem::path{NIX_STORE_DIR}; #endif - }) - .value()); + }) + .value()); } assert(false); }(), diff --git a/src/libstore/unix/build/child.cc b/src/libstore/unix/build/child.cc index 3a704e6edf2c..f603b671e440 100644 --- a/src/libstore/unix/build/child.cc +++ b/src/libstore/unix/build/child.cc @@ -26,7 +26,7 @@ void commonChildInit() throw SysError("cannot dup stderr into stdout"); /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR | O_CLOEXEC); if (fdDevNull == -1) throw SysError("cannot open '%1%'", pathNullDevice); if (dup2(fdDevNull, STDIN_FILENO) == -1) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 7d348af31975..3e9ad5434754 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -974,7 +974,7 @@ void DerivationBuilderImpl::openSlave() { std::string slaveName = getPtsName(builderOut.get()); - AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY | O_CLOEXEC); if (!builderOut) throw SysError("opening pseudoterminal slave"); @@ -1056,7 +1056,7 @@ void DerivationBuilderImpl::processSandboxSetupMessages() FdSource source(builderOut.get()); auto ex = readError(source); ex.addTrace({}, "while setting up the build environment"); - throw ex; + throw std::move(ex); } debug("sandbox setup: " + msg); msgs.push_back(std::move(msg)); diff --git a/src/libutil-tests/file-descriptor.cc b/src/libutil-tests/file-descriptor.cc index 808bb31d8e99..c385d2960f01 100644 --- a/src/libutil-tests/file-descriptor.cc +++ b/src/libutil-tests/file-descriptor.cc @@ -130,7 +130,7 @@ TEST(ReadLine, TreatsEioAsEof) ASSERT_EQ(unlockpt(master), 0); // Open and immediately close the slave to trigger EIO on the master. - int slave = open(ptsname(master), O_RDWR | O_NOCTTY); + int slave = open(ptsname(master), O_RDWR | O_NOCTTY | O_CLOEXEC); ASSERT_NE(slave, -1); close(slave); @@ -153,7 +153,7 @@ TEST(ReadLine, PartialLineBeforeEio) ASSERT_EQ(grantpt(master), 0); ASSERT_EQ(unlockpt(master), 0); - int slave = open(ptsname(master), O_RDWR | O_NOCTTY); + int slave = open(ptsname(master), O_RDWR | O_NOCTTY | O_CLOEXEC); ASSERT_NE(slave, -1); // Write a partial line (no terminator) from the slave, then close it. diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index acd273ca79d9..eff35721b2a1 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -237,6 +237,7 @@ AutoCloseFD::AutoCloseFD(AutoCloseFD && that) noexcept that.fd = INVALID_DESCRIPTOR; } +// NOLINTNEXTLINE(performance-noexcept-move-constructor) - technically can throw AutoCloseFD & AutoCloseFD::operator=(AutoCloseFD && that) { close(); diff --git a/src/libutil/include/nix/util/async.hh b/src/libutil/include/nix/util/async.hh index 7abd446f7347..3f6be9b48cc7 100644 --- a/src/libutil/include/nix/util/async.hh +++ b/src/libutil/include/nix/util/async.hh @@ -88,7 +88,7 @@ asio::awaitable callbackToAwaitable(F && initiate) } template -asio::awaitable forEachAsync(Range && range, F && f) +asio::awaitable forEachAsync(Range && range, const F & f) { /* This code only runs on a strand - we don't do multithreaded executors, so no need for synchronisation. */ @@ -102,7 +102,7 @@ asio::awaitable forEachAsync(Range && range, F && f) co_await asio::async_initiate( [&](auto handler) { auto h = std::make_shared(std::move(handler)); - for (auto && elt : range) { + for (auto && elt : std::forward(range)) { asio::co_spawn(executor, f(elt), [executor, h, &err, &pending](std::exception_ptr ex) { if (ex && !err) err = ex; diff --git a/src/libutil/include/nix/util/closure.hh b/src/libutil/include/nix/util/closure.hh index 586acfff7e70..5078a02d0795 100644 --- a/src/libutil/include/nix/util/closure.hh +++ b/src/libutil/include/nix/util/closure.hh @@ -26,7 +26,7 @@ template using GetEdgesAsync = fun>(const T & elt)>; template -auto computeClosure(std::set startElts, std::set & res, GetEdgesAsync getEdges, CompletionToken && token) +auto computeClosure(std::set startElts, std::set & res, GetEdgesAsync getEdges, CompletionToken token) { auto initiator = [&res, startElts = std::move(startElts), getEdges = std::move(getEdges)](auto handler) { auto executor = asio::make_strand(asio::get_associated_executor(handler)); diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 19d678601b1e..5dc98904a4d0 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -578,7 +578,7 @@ struct ExperimentalFeatureSettings : Config */ template requires std::invocable && std::convertible_to, std::string> - void require(const ExperimentalFeature & feature, GetReason && getReason) const + void require(const ExperimentalFeature & feature, const GetReason & getReason) const { if (isEnabled(feature)) return; diff --git a/src/libutil/include/nix/util/error.hh b/src/libutil/include/nix/util/error.hh index b01c18e58da4..2a846fed17cf 100644 --- a/src/libutil/include/nix/util/error.hh +++ b/src/libutil/include/nix/util/error.hh @@ -126,7 +126,7 @@ protected: public: BaseError(const BaseError &) = default; BaseError & operator=(const BaseError &) = default; - BaseError & operator=(BaseError &&) = default; + BaseError & operator=(BaseError &&) noexcept = default; template BaseError(unsigned int status, Args &&... args) diff --git a/src/libutil/include/nix/util/file-descriptor.hh b/src/libutil/include/nix/util/file-descriptor.hh index f92e737fcda5..a6796f22691e 100644 --- a/src/libutil/include/nix/util/file-descriptor.hh +++ b/src/libutil/include/nix/util/file-descriptor.hh @@ -262,6 +262,7 @@ public: AutoCloseFD(AutoCloseFD && fd) noexcept; ~AutoCloseFD(); AutoCloseFD & operator=(const AutoCloseFD & fd) = delete; + // NOLINTNEXTLINE(performance-noexcept-move-constructor) - technically can throw because of close() AutoCloseFD & operator=(AutoCloseFD && fd); Descriptor get() const; explicit operator bool() const; diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index 761a0fe6ed88..ed84c767e8b3 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -177,6 +177,7 @@ struct FdSink : BufferedSink FdSink(const FdSink &) = delete; FdSink & operator=(const FdSink &) = delete; + // NOLINTNEXTLINE(performance-noexcept-move-constructor) - can throw FdSink & operator=(FdSink && s) { flush(); diff --git a/src/libutil/include/nix/util/sort.hh b/src/libutil/include/nix/util/sort.hh index 2a4eb6e7c98e..77a3ea59a50d 100644 --- a/src/libutil/include/nix/util/sort.hh +++ b/src/libutil/include/nix/util/sort.hh @@ -123,7 +123,7 @@ void insertionsort(Iter begin, Iter end, Comparator comp = {}) * to the specified comparator. */ template>> -Iter strictlyDecreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) +Iter strictlyDecreasingPrefix(Iter begin, Iter end, const Comparator & comp = {}) { if (begin == end) return begin; @@ -138,7 +138,7 @@ Iter strictlyDecreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) * to the specified comparator. */ template>> -Iter strictlyDecreasingSuffix(Iter begin, Iter end, Comparator && comp = {}) +Iter strictlyDecreasingSuffix(Iter begin, Iter end, const Comparator & comp = {}) { if (begin == end) return end; @@ -153,9 +153,9 @@ Iter strictlyDecreasingSuffix(Iter begin, Iter end, Comparator && comp = {}) * to the specified comparator. */ template>> -Iter weaklyIncreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) +Iter weaklyIncreasingPrefix(Iter begin, Iter end, const Comparator & comp = {}) { - return strictlyDecreasingPrefix(begin, end, std::not_fn(std::forward(comp))); + return strictlyDecreasingPrefix(begin, end, std::not_fn(comp)); } /** @@ -163,9 +163,9 @@ Iter weaklyIncreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) * to the specified comparator. */ template>> -Iter weaklyIncreasingSuffix(Iter begin, Iter end, Comparator && comp = {}) +Iter weaklyIncreasingSuffix(Iter begin, Iter end, const Comparator & comp = {}) { - return strictlyDecreasingSuffix(begin, end, std::not_fn(std::forward(comp))); + return strictlyDecreasingSuffix(begin, end, std::not_fn(comp)); } /** diff --git a/src/libutil/include/nix/util/topo-sort.hh b/src/libutil/include/nix/util/topo-sort.hh index 6218b66a5023..12031e071535 100644 --- a/src/libutil/include/nix/util/topo-sort.hh +++ b/src/libutil/include/nix/util/topo-sort.hh @@ -20,7 +20,7 @@ using TopoSortResult = std::variant, Cycle>; template F> requires std::same_as>, std::set> -TopoSortResult topoSort(std::set items, F && getChildren) +TopoSortResult topoSort(std::set items, const F & getChildren) { std::vector sorted; decltype(items) visited, parents; diff --git a/src/libutil/include/nix/util/util.hh b/src/libutil/include/nix/util/util.hh index 144c83cc0c9f..8d26ab1a1efd 100644 --- a/src/libutil/include/nix/util/util.hh +++ b/src/libutil/include/nix/util/util.hh @@ -31,7 +31,7 @@ template auto concatStrings(Parts &&... parts) -> std::enable_if_t<(... && std::is_convertible_v), std::string> { - std::string_view views[sizeof...(parts)] = {parts...}; + std::string_view views[sizeof...(parts)] = {std::forward(parts)...}; return concatStringsSep({}, views); } diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index 9c96c5bcb768..26a7479050ad 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -95,11 +95,11 @@ void saveMountNamespace() { static std::once_flag done; std::call_once(done, []() { - fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY | O_CLOEXEC); if (!fdSavedMountNamespace) throw SysError("saving parent mount namespace"); - fdSavedRoot = open("/proc/self/root", O_RDONLY); + fdSavedRoot = open("/proc/self/root", O_RDONLY | O_CLOEXEC); }); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 58c78df34986..66038e25e27f 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -359,9 +359,15 @@ std::unique_ptr makeJSONLogger(const std::filesystem::path & path, bool } }; - AutoCloseFD fd = std::filesystem::is_socket(path) - ? connect(path) - : toDescriptor(open(path.string().c_str(), O_CREAT | O_APPEND | O_WRONLY, 0644)); + AutoCloseFD fd = std::filesystem::is_socket(path) ? connect(path) + : toDescriptor(open( + path.string().c_str(), + O_CREAT | O_APPEND | O_WRONLY +#ifndef _WIN32 + | O_CLOEXEC +#endif + , + 0644)); if (!fd) throw SysError("opening log file %1%", PathFmt(path)); diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index 9181f4466e1b..f9417bb7e9d4 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -211,7 +211,7 @@ static void _deletePath( throw; } - int fd = openat(parentfd, name.rel_c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + int fd = openat(parentfd, name.rel_c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); if (fd == -1) throw SysError("opening directory %1%", PathFmt(path)); AutoCloseDir dir(fdopendir(fd)); From 8b974a35bddd047b8893d432ed951790765c4377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 26 Apr 2026 20:03:10 +0200 Subject: [PATCH 318/555] tests/functional/json: fix script(1) invocation for util-linux 2.42 util-linux 2.42 (commit 7268e79b) added "+" to the getopt string of script(1), so option parsing stops at the first non-option argument. The previous `script -e -q /dev/null -c CMD` ordering therefore treats `-c CMD` as extra positional arguments and fails with "unexpected number of arguments". Place all options before the argument, which works on both old and new util-linux. --- tests/functional/json.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/json.sh b/tests/functional/json.sh index 49992e0d9324..157395e11373 100644 --- a/tests/functional/json.sh +++ b/tests/functional/json.sh @@ -51,7 +51,7 @@ if type script &>/dev/null; then if [[ $acceptsCommandFlag -eq 0 ]]; then script -e -q /dev/null "$@" else - script -e -q /dev/null -c "$(shellEscapeArray "$@")" + script -e -q -c "$(shellEscapeArray "$@")" /dev/null fi } runScript nix eval --json --expr "{ a.b.c = true; }" > "$TEST_HOME/actual.json" From 456764609d0c696ce78758514cbeabab9917459d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 26 Apr 2026 20:53:13 +0200 Subject: [PATCH 319/555] doc/rl-next: add release note for blocking *listxattr in sandbox The seccomp filter change is user-visible: builds on SELinux hosts may now behave differently (correctly) and tools probing xattr support via listxattr will see ENOTSUP. Document this so users can trace behavior changes back to this PR. --- doc/manual/rl-next/seccomp-block-listxattr.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/manual/rl-next/seccomp-block-listxattr.md diff --git a/doc/manual/rl-next/seccomp-block-listxattr.md b/doc/manual/rl-next/seccomp-block-listxattr.md new file mode 100644 index 000000000000..2455ed4f825f --- /dev/null +++ b/doc/manual/rl-next/seccomp-block-listxattr.md @@ -0,0 +1,10 @@ +--- +synopsis: "Linux sandbox: also block `listxattr` syscalls" +prs: [15743] +--- + +The Linux sandbox now also returns `ENOTSUP` for `listxattr`, +`llistxattr` and `flistxattr`, matching the existing treatment of +`getxattr`/`setxattr`/`removexattr`. This prevents host xattrs (e.g. +`security.selinux`) from leaking into builds and fixes tools such as +`mkfs.ubifs` that probe xattr support via `listxattr`. From 02df27e1cff20c140ea216d0f7ff50325c4a9c40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 27 Apr 2026 18:54:01 +0200 Subject: [PATCH 320/555] rust-installer: 2.34.5 -> 2.34.6 2.34.6 dropped build.rs/env!() embedding in favour of appending the closure via scripts/pack post-build, so the Rust compile is independent of the embedded Nix and stays cacheable across revisions. Adapt the packaging accordingly. --- packaging/rust-installer/default.nix | 98 +++++++++++++++++----------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/packaging/rust-installer/default.nix b/packaging/rust-installer/default.nix index dcfbf4badc31..98aec45e9e53 100644 --- a/packaging/rust-installer/default.nix +++ b/packaging/rust-installer/default.nix @@ -4,61 +4,85 @@ { lib, stdenv, + buildPackages, + runCommand, rustPlatform, fetchFromGitHub, tarball, }: let - installerVersion = "2.34.5"; + installerVersion = "2.34.6"; src = fetchFromGitHub { owner = "NixOS"; repo = "nix-installer"; tag = installerVersion; - hash = "sha256-+gM241qQOzQlOnP0a7d47z3iRf9+yNjbBJCLIWWNX+c="; + hash = "sha256-aTaz8EtHexvke7tGr5MfeKy9g7AraIAFN+dPApm+fds="; }; -in -rustPlatform.buildRustPackage { - pname = "nix-installer"; - version = tarball.passthru.nixVersion; + # Bare binary: no Nix closure yet. Appended below via `pack`, so the + # (expensive) Rust compile is independent of the embedded Nix and + # stays cacheable across Nix revisions. + bare = rustPlatform.buildRustPackage { + pname = "nix-installer-bare"; + version = installerVersion; - inherit src; + inherit src; - cargoHash = "sha256-6pt2f7wznH672L5+SkbA5GA6Sxvk1KamAf3erGqZlLU="; + cargoHash = "sha256-/mNXkeZVuYsqd0TiUa7bzSP4xpKh0Fqga9EpasPbrzU="; - doCheck = false; + doCheck = false; - env = { - NIX_TARBALL_PATH = "${tarball}/nix.tar.zst"; - NIX_STORE_PATH = tarball.passthru.nixStorePath; - NSS_CACERT_STORE_PATH = tarball.passthru.cacertStorePath; - NIX_VERSION = tarball.passthru.nixVersion; - } - // lib.optionalAttrs stdenv.hostPlatform.isDarwin { - # Drop the unused libiconv dylib the darwin stdenv injects; the - # binary must run before `/nix/store` exists. - NIX_LDFLAGS = "-dead_strip_dylibs"; + env = lib.optionalAttrs stdenv.hostPlatform.isDarwin { + # Drop the unused libiconv dylib the darwin stdenv injects; the + # binary must run before `/nix/store` exists. + NIX_LDFLAGS = "-dead_strip_dylibs"; + }; + + postInstall = '' + install -m755 nix-installer.sh $out/bin/nix-installer.sh + ''; }; +in - postInstall = '' - install -m755 nix-installer.sh $out/bin/nix-installer.sh +runCommand "nix-installer-${tarball.passthru.nixVersion}" + { + nativeBuildInputs = [ + buildPackages.python3 + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + buildPackages.darwin.sigtool + buildPackages.darwin.cctools + ]; - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix-installer" >> $out/nix-support/hydra-build-products - echo "file binary-dist $out/bin/nix-installer.sh" >> $out/nix-support/hydra-build-products - ''; + # The appended payload contains store-path strings on purpose; don't + # let the reference scanner pull the whole Nix closure into this + # derivation's runtime closure. + __structuredAttrs = true; + unsafeDiscardReferences.out = true; - # The binary embeds store-path strings (`NIX_STORE_PATH`, …) on - # purpose; don't let the reference scanner pull the whole Nix - # closure into this derivation's runtime closure. - __structuredAttrs = true; - unsafeDiscardReferences.out = true; + passthru = { inherit bare; }; - meta = { - description = "Rust-based Nix installer with an embedded Nix ${tarball.passthru.nixVersion}"; - homepage = "https://github.com/NixOS/nix-installer"; - license = lib.licenses.lgpl21Only; - mainProgram = "nix-installer"; - }; -} + meta = { + description = "Rust-based Nix installer with an embedded Nix ${tarball.passthru.nixVersion}"; + homepage = "https://github.com/NixOS/nix-installer"; + license = lib.licenses.lgpl21Only; + mainProgram = "nix-installer"; + }; + } + '' + mkdir -p $out/bin $out/nix-support + + python3 ${src}/scripts/pack \ + --input ${bare}/bin/nix-installer \ + --tarball ${tarball}/nix.tar.zst \ + --nix-store-path ${tarball.passthru.nixStorePath} \ + --cacert-store-path ${tarball.passthru.cacertStorePath} \ + --nix-version ${tarball.passthru.nixVersion} \ + --output $out/bin/nix-installer + + install -m755 ${bare}/bin/nix-installer.sh $out/bin/nix-installer.sh + + echo "file binary-dist $out/bin/nix-installer" >> $out/nix-support/hydra-build-products + echo "file binary-dist $out/bin/nix-installer.sh" >> $out/nix-support/hydra-build-products + '' From 498f96d1457de6d96b7e2b6aeeaee64c34c3a1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 27 Apr 2026 21:35:04 +0200 Subject: [PATCH 321/555] libutil: Use poll() in FdSource::hasData() to avoid fd_set overflow FD_SET writes past the stack fd_set when fd >= FD_SETSIZE. hasData() runs before every frame in withFramedSink(), so clients with many open fds would corrupt the stack (or abort under glibc _FORTIFY_SOURCE) during addToStore(). poll() has no such limit; Windows keeps select() since its fd_set is a bounded handle array. --- src/libutil/serialise.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 6c77c15fe584..4ff2d63f2f86 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -224,6 +224,9 @@ bool FdSource::hasData() return true; while (true) { +#ifdef _WIN32 + /* Windows' fd_set is a bounded handle array, so FD_SET can't + overflow; on Unix use poll() since fd may exceed FD_SETSIZE. */ fd_set fds; FD_ZERO(&fds); Socket sock = toSocket(fd); @@ -240,6 +243,20 @@ bool FdSource::hasData() throw SysError("polling file descriptor"); } return FD_ISSET(sock, &fds); +#else + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + pfd.revents = 0; + + auto n = poll(&pfd, 1, 0); + if (n < 0) { + if (errno == EINTR) + continue; + throw SysError("polling file descriptor"); + } + return n > 0 && (pfd.revents & (POLLIN | POLLHUP | POLLERR)) != 0; +#endif } } From 934a7af2877ac60207453bdf353092cf267a6ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 27 Apr 2026 23:27:08 +0200 Subject: [PATCH 322/555] dependabot: enable Nix ecosystem for flake.lock updates Dependabot recently gained support for updating Nix flake inputs. Enable it so we get automated weekly bumps of flake.lock instead of having to update inputs manually. Group all flake inputs into a single pull request to avoid a flood of one-PR-per-input updates and to keep CI load and review effort low. --- .github/dependabot.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5ace4600a1f2..1880a89da556 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,11 @@ updates: directory: "/" schedule: interval: "weekly" + - package-ecosystem: "nix" + directory: "/" + schedule: + interval: "weekly" + groups: + flake-inputs: + patterns: + - "*" From f2720c2037a2fdb241699c855d8d1cd387032bf3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 22:54:30 +0000 Subject: [PATCH 323/555] build(deps): bump cachix/install-nix-action from 31.10.4 to 31.10.5 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.10.4 to 31.10.5. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/616559265b40713947b9c190a8ff4b507b5df49b...ab739621df7a23f52766f9ccc97f38da6b7af14f) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.10.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07329d16a202..49fff49b9737 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,7 +189,7 @@ jobs: - name: Looking up the installer tarball URL id: installer-tarball-url run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 + - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 if: ${{ !matrix.rust-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} @@ -255,7 +255,7 @@ jobs: id: installer-tarball-url run: | echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 + - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} From 741de5d7481207eb2d761c17c3642711235d589e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 22:55:24 +0000 Subject: [PATCH 324/555] build(deps): bump the flake-inputs group with 2 updates Bumps the flake-inputs group with 2 updates: [flake-parts](https://github.com/hercules-ci/flake-parts) and [git-hooks-nix](https://github.com/cachix/git-hooks.nix). Updates `flake-parts` from `205b12d` to `3107b77` - [Commits](https://github.com/hercules-ci/flake-parts/compare/205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9...3107b77cd68437b9a76194f0f7f9c55f2329ca5b) Updates `git-hooks-nix` from `aa9f40c` to `3cfd774` - [Commits](https://github.com/cachix/git-hooks.nix/compare/aa9f40c906904ebd83da78e7f328cd8aeaeae785...3cfd774b0a530725a077e17354fbdb87ea1c4aad) --- updated-dependencies: - dependency-name: flake-parts dependency-version: 3107b77cd68437b9a76194f0f7f9c55f2329ca5b dependency-type: direct:production dependency-group: flake-inputs - dependency-name: git-hooks-nix dependency-version: 3cfd774b0a530725a077e17354fbdb87ea1c4aad dependency-type: direct:production dependency-group: flake-inputs ... Signed-off-by: dependabot[bot] --- flake.lock | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 4c0bf91927a4..1212049c3bea 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1733312601, - "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", + "lastModified": 1775087534, + "narHash": "sha256-91qqW8lhL7TLwgQWijoGBbiD4t7/q75KTi8NxjVmSmA=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", + "rev": "3107b77cd68437b9a76194f0f7f9c55f2329ca5b", "type": "github" }, "original": { @@ -42,17 +42,14 @@ "gitignore": [], "nixpkgs": [ "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" ] }, "locked": { - "lastModified": 1734279981, - "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", + "lastModified": 1776796298, + "narHash": "sha256-PcRvlWayisPSjd0UcRQbhG8Oqw78AcPE6x872cPRHN8=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", + "rev": "3cfd774b0a530725a077e17354fbdb87ea1c4aad", "type": "github" }, "original": { From f63f6037c921af7da9cf4eaa1bb12f9a30981251 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 22 Apr 2026 15:45:23 -0400 Subject: [PATCH 325/555] Misc gc/deletion improvements / fixes - Improve deletion logging - Fix informational messages from being duplicated - Add some debugging messages useful for diagnosing why a path wasn't deleted - Use `.contains` instead of `.count` - Add --skip-live alias to match Lix (be forgiving to people used to Lix flags and support --skip-live as an alias for --skip-alive. - Exclude outputs and drvs from deletion if passed a StorePathSet. - gcDeleteSpecific is no longer an indicator of operating on specific paths, rather the presence of a StorePathSet in pathsToDelete is. - Clarify gc comments around keep-derivations and keep-outputs Signed-off-by: Lisanna Dettwyler --- src/libstore/gc.cc | 56 ++++++++++++++++++++++++----------------- src/nix/store-delete.cc | 1 + 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 835d06864671..701fc66e69d0 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -385,8 +385,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Using `--ignore-liveness' with `--delete' can have unintended consequences if `keep-outputs' or `keep-derivations' are true (the garbage collector will recurse into deleting the outputs - or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + or derivers, respectively, even if they aren't in the + pathsToDelete). So disable them. */ + if (std::holds_alternative(options.pathsToDelete) && options.ignoreLiveness) { keepOutputs = false; keepDerivations = false; } @@ -608,14 +609,15 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Bail out if we've previously discovered that this path is alive. */ - if (alive.count(*path)) { + if (alive.contains(*path)) { + debug("cannot delete '%s' because '%s' is alive", printStorePath(start), printStorePath(*path)); alive.insert(start); return; } /* If we've previously deleted this path, we don't have to handle it again. */ - if (dead.count(*path)) + if (dead.contains(*path)) continue; auto markAlive = [&]() { @@ -636,19 +638,24 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) }; /* If this is a root, bail out. */ - if (roots.count(*path)) { + if (roots.contains(*path)) { debug("cannot delete '%s' because it's a root", printStorePath(*path)); return markAlive(); } if (std::holds_alternative(options.pathsToDelete) - && !std::get(options.pathsToDelete).contains(*path)) + && !std::get(options.pathsToDelete).contains(*path)) { + debug( + "cannot delete '%s' because '%s' is not in the specified paths to delete", + printStorePath(start), + printStorePath(*path)); return; + } { auto hashPart = path->hashPart(); auto shared(_shared.lock()); - if (shared->tempRoots.count(hashPart)) { + if (shared->tempRoots.contains(hashPart)) { debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); return markAlive(); } @@ -668,8 +675,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) for (auto & p : i->second) enqueue(p); - /* If keep-derivations is set and this is a - derivation, then visit the derivation outputs. */ + /* If keep-derivations is set and this is a derivation, then we only want to delete this derivation if + * we can also delete all its outputs, so visit the derivation outputs. */ if (keepDerivations && path->isDerivation()) { for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) if (maybeOutPath && isValidPath(*maybeOutPath) @@ -677,7 +684,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) enqueue(*maybeOutPath); } - /* If keep-outputs is set, then visit the derivers. */ + /* If keep-outputs is set, we only want to delete this path if we + * can also delete its derivers, so visit the derivers. */ if (keepOutputs) { auto derivers = queryValidDerivers(*path); for (auto & i : derivers) @@ -707,27 +715,29 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) std::visit( overloaded{ [&](const StorePathSet & paths) { - for (auto & i : paths) { - switch (options.action) { - case GCOptions::gcDeleteDead: - printInfo("deleting garbage within specified paths..."); - break; - case GCOptions::gcDeleteSpecific: - printInfo("deleting specified paths..."); - break; - case GCOptions::gcReturnDead: - case GCOptions::gcReturnLive: - printInfo("determining live/dead paths..."); - } + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage within specified paths..."); + break; + case GCOptions::gcDeleteSpecific: + printInfo("deleting specified paths..."); + break; + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } + for (auto & i : paths) { maybeDeleteReferrersClosure(i); - if (options.action == GCOptions::gcDeleteSpecific && !dead.count(i)) + if (options.action == GCOptions::gcDeleteSpecific && !dead.contains(i)) throw Error( "Cannot delete path '%1%' since it is still alive. " "To find out why, use: " "nix-store --query --roots and nix-store --query --referrers", printStorePath(i)); + else if (!dead.contains(i)) + debug("cannot delete '%s' because it's still alive", printStorePath(i)); } }, [&](const GCOptions::WholeStore & _) { diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 5eef2d1ad235..a1a387898491 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -20,6 +20,7 @@ struct CmdStoreDelete : StorePathsCommand addFlag({ .longName = "skip-alive", + .aliases = {"skip-live"}, .description = "Do not emit errors when attempting to delete something that is still alive, useful with --recursive.", .handler = {&options.action, GCOptions::gcDeleteDead}, From 87032c6dc4b539a0fc9bb50964965bc6c3554532 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 28 Apr 2026 23:52:19 +0300 Subject: [PATCH 326/555] libstore: Drop queryMissing from Worker::run This is no longer needed, since PathSubstitutionGoal now queries ValidPathInfo asynchronously and we can start the builds right away without querying the whole closure at once. This can significantly speed up the build startup. This was added in back in bbdf08bc0facb5157a10c794712dae7e5902be03 when pathinfo queries were done sequentially and in a blocking manner - this is not the case anymore. The asynchronous queries don't wait for a build slot to start, so the concurrency there is not limited and all path queries can complete as the worker loop naturally. This also acts as natural rate limiting to avoid allocating too many coroutines up-front and blowing up the memory usage (which also slows down the whole build to a crawl because fork() starts taking increasingly longer to complete due large anonymous memory mappings). --- src/libstore/build/worker.cc | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 3fe196d08005..208bf5ca3ebb 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -295,28 +295,11 @@ void Worker::waitForCompletion(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - std::vector topPaths; - - for (auto & i : _topGoals) { - topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back( - DerivedPath::Built{ - .drvPath = goal->drvReq, - .outputs = goal->wantedOutputs, - }); - } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Opaque{goal->storePath}); - } - } - - /* Call queryMissing() to efficiently query substitutes. */ - store.queryMissing(topPaths); - debug("entered goal loop"); + for (std::shared_ptr goal : _topGoals) + topGoals.insert(std::move(goal)); while (1) { - checkInterrupt(); // TODO GC interface? From 8c03190cea1b479246514006a55eae4bdfd844c3 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Sun, 26 Apr 2026 18:47:53 +0200 Subject: [PATCH 327/555] include in src/libfetchers/fetchers.cc `sleep_for` is defined in --- src/libfetchers/fetchers.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 4eedafa22a24..310373a19adf 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -11,6 +11,7 @@ #include "nix/store/pathlocks.hh" #include "nix/util/environment-variables.hh" +#include #include namespace nix::fetchers { From 53a90079c1293fc07293b7f2e5df2c69c5df79a3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 29 Apr 2026 20:52:06 +0300 Subject: [PATCH 328/555] libstore: Anchor all Store and StoreConfig vtables Classes with virtual functions defined in headers must have at least one out-of-line definition of a virtual function (also called "key function" [3]) to prevent having the vtable from having weak (vague [2]) linkage and causing a bunch of headaches on Darwin. For now this just limits to fixes for what would be required for nix-heuristic-gc, which compiles with pybind11 which uses hidden visibility for the dylib. In the future we'd certainly want to enable -Werror=weak-vtables to prevent such footguns. The pattern used here is basically the same as what LLVM does [1]. [1]: https://github.com/llvm/llvm-project/blob/710c297289b751857bdc0a8677437d575b403922/clang/lib/ExtractAPI/API.cpp#L177-L200 [2]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#vague [3]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#vague-vtable --- src/libstore/binary-cache-store.cc | 4 ++++ src/libstore/common-ssh-store-config.cc | 2 ++ src/libstore/dummy-store.cc | 10 ++++++++++ src/libstore/http-binary-cache-store.cc | 4 ++++ .../include/nix/store/binary-cache-store.hh | 6 ++++++ .../include/nix/store/common-ssh-store-config.hh | 4 ++++ .../include/nix/store/dummy-store-impl.hh | 4 ++++ src/libstore/include/nix/store/dummy-store.hh | 4 ++++ src/libstore/include/nix/store/gc-store.hh | 4 ++++ .../include/nix/store/http-binary-cache-store.hh | 6 ++++++ .../include/nix/store/indirect-root-store.hh | 4 ++++ .../include/nix/store/legacy-ssh-store.hh | 8 ++++++++ .../nix/store/local-binary-cache-store.hh | 4 ++++ src/libstore/include/nix/store/local-fs-store.hh | 6 ++++++ .../include/nix/store/local-overlay-store.hh | 6 ++++++ src/libstore/include/nix/store/local-store.hh | 6 +++++- src/libstore/include/nix/store/log-store.hh | 4 ++++ src/libstore/include/nix/store/remote-store.hh | 8 ++++++++ src/libstore/include/nix/store/ssh-store.hh | 8 ++++++++ src/libstore/include/nix/store/store-api.hh | 10 ++++++++++ .../include/nix/store/uds-remote-store.hh | 8 ++++++++ src/libstore/indirect-root-store.cc | 2 ++ src/libstore/legacy-ssh-store.cc | 4 ++++ src/libstore/local-binary-cache-store.cc | 8 ++++++++ src/libstore/local-fs-store.cc | 4 ++++ src/libstore/local-overlay-store.cc | 4 ++++ src/libstore/local-store.cc | 8 ++++++++ src/libstore/log-store.cc | 2 ++ src/libstore/remote-store.cc | 4 ++++ src/libstore/restricted-store.cc | 6 ++++++ src/libstore/ssh-store.cc | 16 ++++++++++++++++ src/libstore/uds-remote-store.cc | 4 ++++ 32 files changed, 181 insertions(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 64fe33536bbd..8ceb8f2151af 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -22,6 +22,10 @@ namespace nix { +void BinaryCacheStoreConfig::anchor() {} + +void BinaryCacheStore::anchor() {} + BinaryCacheStore::BinaryCacheStore(Config & config) : config{config} { diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index ee1d3bf8acde..db3677151416 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -9,6 +9,8 @@ CommonSSHStoreConfig::CommonSSHStoreConfig(const ParsedURL::Authority & authorit { } +void CommonSSHStoreConfig::anchor() {} + SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const { return { diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 052ec9b1283e..2ee093d85be8 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -10,6 +10,10 @@ namespace nix { +void DummyStoreConfig::anchor() {} + +void DummyStore::anchor() {} + std::string DummyStoreConfig::doc() { return @@ -126,6 +130,10 @@ bool DummyStoreConfig::getReadOnly() const struct DummyStoreImpl : DummyStore { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; /** @@ -378,6 +386,8 @@ struct DummyStoreImpl : DummyStore } }; +void DummyStoreImpl::anchor() {} + ref DummyStore::Config::openDummyStore() const { return make_ref(ref{shared_from_this()}); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 713de4969a94..8f0c22f99856 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -23,6 +23,10 @@ StringSet HttpBinaryCacheStoreConfig::uriSchemes() return ret; } +void HttpBinaryCacheStoreConfig::anchor() {} + +void HttpBinaryCacheStore::anchor() {} + HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig(ParsedURL _cacheUri, const Params & params) : StoreConfig(params, FilePathType::Unix) , BinaryCacheStoreConfig(params) diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 7871ad03c884..3e8cf886c897 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -16,6 +16,10 @@ class RemoteFSAccessor; struct BinaryCacheStoreConfig : virtual StoreConfig { +private: + void anchor() override; + +public: BinaryCacheStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { @@ -92,6 +96,8 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ Config & config; private: + void anchor() override; + std::vector> signers; protected: diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index 1e90c94afcbe..b622e82e26bf 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -10,6 +10,10 @@ class SSHMaster; struct CommonSSHStoreConfig : virtual StoreConfig { +private: + void anchor() override; + +public: CommonSSHStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 8fdeeb362515..bec77a6bee68 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -15,6 +15,10 @@ struct MemorySourceAccessor; */ struct DummyStore : virtual Store { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; ref config; diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index f76fb3d5c2b0..c8a212c75603 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -12,6 +12,10 @@ struct DummyStore; struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig { +private: + void anchor() override; + +public: DummyStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 5e23f2052472..015478e79fd3 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -106,6 +106,10 @@ struct GCResults */ struct GcStore : public virtual Store { +private: + void anchor() override; + +public: inline static std::string operationName = "Garbage collection"; /** diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index 748daec646b9..12465261caef 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -14,6 +14,10 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, virtual CommonSSHStoreConfig { +private: + void anchor() override; + +public: LegacySSHStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) , CommonSSHStoreConfig(params) @@ -63,6 +67,10 @@ struct LegacySSHStoreConfig : std::enable_shared_from_this struct LegacySSHStore : public virtual Store { +private: + void anchor() override; + +public: using Config = LegacySSHStoreConfig; ref config; diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index 69a4bac1c8a9..181b33e4bdf8 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -9,6 +9,10 @@ struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this> makeRootDirSetting(LocalFSStoreConfig & self, std::optional defaultValue) { @@ -87,6 +89,10 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ virtual GcStore, virtual LogStore { +private: + void anchor() override; + +public: using Config = LocalFSStoreConfig; const Config & config; diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index dfb1fb184a55..10b04937c3aa 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -7,6 +7,10 @@ namespace nix { */ struct LocalOverlayStoreConfig : virtual LocalStoreConfig { +private: + void anchor() override; + +public: LocalOverlayStoreConfig(const StringMap & params) : LocalOverlayStoreConfig("", params) { @@ -119,6 +123,8 @@ struct LocalOverlayStore : virtual LocalStore LocalOverlayStore(ref); private: + void anchor() override; + /** * The store beneath us. * diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index bf3437e95771..e5ecaf8a59de 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -35,8 +35,9 @@ struct LocalSettings; struct LocalBuildStoreConfig : virtual LocalFSStoreConfig { - private: + void anchor() override; + /** Input for computing the build directory. See `getBuildDir()`. */ @@ -89,6 +90,7 @@ struct LocalStoreConfig : std::enable_shared_from_this, LocalStoreConfig(const std::filesystem::path & path, const Params & params); private: + void anchor() override; /** * An indirection so that we don't need to refer to global settings @@ -177,6 +179,8 @@ public: class LocalStore : public virtual IndirectRootStore, public virtual GcStore { + void anchor() override; + public: using Config = LocalStoreConfig; diff --git a/src/libstore/include/nix/store/log-store.hh b/src/libstore/include/nix/store/log-store.hh index 2d81d02b10cc..e0acd9a04f62 100644 --- a/src/libstore/include/nix/store/log-store.hh +++ b/src/libstore/include/nix/store/log-store.hh @@ -7,6 +7,10 @@ namespace nix { struct LogStore : public virtual Store { +private: + void anchor() override; + +public: inline static std::string operationName = "Build log storage and retrieval"; /** diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index 57beb9135f7a..144b4b8e4355 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -23,6 +23,10 @@ class RemoteFSAccessor; struct RemoteStoreConfig : virtual StoreConfig { +private: + void anchor() override; + +public: RemoteStoreConfig(const Params & params, FilePathType pathType) : StoreConfig(params, pathType) { @@ -44,6 +48,10 @@ struct RemoteStoreConfig : virtual StoreConfig */ struct RemoteStore : public virtual Store, public virtual GcStore, public virtual LogStore { +private: + void anchor() override; + +public: using Config = RemoteStoreConfig; const Config & config; diff --git a/src/libstore/include/nix/store/ssh-store.hh b/src/libstore/include/nix/store/ssh-store.hh index 4ab88ca74cb7..324e85eb60f4 100644 --- a/src/libstore/include/nix/store/ssh-store.hh +++ b/src/libstore/include/nix/store/ssh-store.hh @@ -12,6 +12,10 @@ struct SSHStoreConfig : std::enable_shared_from_this, virtual RemoteStoreConfig, virtual CommonSSHStoreConfig { +private: + void anchor() override; + +public: SSHStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) , RemoteStoreConfig(params, FilePathType::Unix) @@ -43,6 +47,10 @@ struct SSHStoreConfig : std::enable_shared_from_this, struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig { +private: + void anchor() override; + +public: MountedSSHStoreConfig(StringMap params); MountedSSHStoreConfig(const ParsedURL::Authority & authority, StringMap params); diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 97251eaee9c2..bfd4ffce2181 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -229,6 +229,12 @@ public: */ struct StoreConfig : public StoreConfigBase, public StoreDirConfig { +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor() = 0; + +public: using Params = StoreReference::Params; StoreConfig(const Params & params, FilePathType pathType); @@ -380,6 +386,10 @@ struct StoreConfig : public StoreConfigBase, public StoreDirConfig */ class Store : public std::enable_shared_from_this, public StoreDirConfig { + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor() = 0; + public: using Config = StoreConfig; diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index 43931c628af8..8359f8fdf45b 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -27,6 +27,10 @@ struct UDSRemoteStoreConfig : std::enable_shared_from_this virtual LocalFSStoreConfig, virtual RemoteStoreConfig { +private: + void anchor() override; + +public: UDSRemoteStoreConfig(const std::filesystem::path & path, const Params & params); UDSRemoteStoreConfig(const Params & params); @@ -57,6 +61,10 @@ struct UDSRemoteStoreConfig : std::enable_shared_from_this struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore { +private: + void anchor() override; + +public: using Config = UDSRemoteStoreConfig; ref config; diff --git a/src/libstore/indirect-root-store.cc b/src/libstore/indirect-root-store.cc index 7384456e286d..b203afb63dcb 100644 --- a/src/libstore/indirect-root-store.cc +++ b/src/libstore/indirect-root-store.cc @@ -2,6 +2,8 @@ namespace nix { +void IndirectRootStore::anchor() {} + void IndirectRootStore::makeSymlink(const std::filesystem::path & link, const std::filesystem::path & target) { /* Create directories up to `gcRoot'. */ diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a00914c4493d..cbcc42dbed75 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -24,6 +24,8 @@ LegacySSHStoreConfig::LegacySSHStoreConfig(const ParsedURL::Authority & authorit { } +void LegacySSHStoreConfig::anchor() {} + std::string LegacySSHStoreConfig::doc() { return @@ -37,6 +39,8 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; +void LegacySSHStore::anchor() {} + LegacySSHStore::LegacySSHStore(ref config) : Store{*config} , config{config} diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 60298efdbbfd..4d79f613e4a5 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -52,6 +52,10 @@ StoreReference LocalBinaryCacheStoreConfig::getReference() const struct LocalBinaryCacheStore : virtual BinaryCacheStore { +private: + void anchor() override; + +public: using Config = LocalBinaryCacheStoreConfig; ref config; @@ -139,6 +143,10 @@ StringSet LocalBinaryCacheStoreConfig::uriSchemes() return {"file"}; } +void LocalBinaryCacheStoreConfig::anchor() {} + +void LocalBinaryCacheStore::anchor() {} + ref LocalBinaryCacheStoreConfig::openStore() const { auto store = make_ref( diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 3fc724f5fe74..77525d5416da 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -5,6 +5,10 @@ namespace nix { +void LocalFSStoreConfig::anchor() {} + +void LocalFSStore::anchor() {} + LocalFSStoreConfig::LocalFSStoreConfig(const std::filesystem::path & rootDir, const Params & params) : StoreConfig(params, FilePathType::Native) /* Default `?root` from `rootDir` if non set diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 8d1a16f91281..c0fee83d7d18 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -10,6 +10,10 @@ namespace nix { +void LocalOverlayStoreConfig::anchor() {} + +void LocalOverlayStore::anchor() {} + std::string LocalOverlayStoreConfig::doc() { return diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 75f485c31f22..53d7456f94f6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -56,6 +56,14 @@ namespace nix { +void LocalStoreConfig::anchor() {} + +void LocalBuildStoreConfig::anchor() {} + +void LocalStore::anchor() {} + +void GcStore::anchor() {} + LocalStoreConfig::LocalStoreConfig(const std::filesystem::path & path, const Params & params) : StoreConfig(params, FilePathType::Native) , LocalFSStoreConfig(path, params) diff --git a/src/libstore/log-store.cc b/src/libstore/log-store.cc index fd03bb30ea02..23e6563991d4 100644 --- a/src/libstore/log-store.cc +++ b/src/libstore/log-store.cc @@ -2,6 +2,8 @@ namespace nix { +void LogStore::anchor() {} + std::optional LogStore::getBuildLog(const StorePath & path) { auto maybePath = getBuildDerivationPath(path); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 88c3847bb3cf..a2fb3a10d251 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -30,6 +30,8 @@ namespace nix { +void RemoteStoreConfig::anchor() {} + /* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Config & config) : Store{config} @@ -59,6 +61,8 @@ RemoteStore::RemoteStore(const Config & config) { } +void RemoteStore::anchor() {} + ref RemoteStore::openConnectionWrapper() { if (failed) { diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 03c130da9653..eaeca59a2bf4 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -38,6 +38,10 @@ bool RestrictionContext::isAllowed(const DerivedPath & req) */ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStore { +private: + void anchor() override; + +public: ref config; ref next; @@ -157,6 +161,8 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor } }; +void RestrictedStore::anchor() {} + ref makeRestrictedStore(ref config, ref next, RestrictionContext & context) { return make_ref(config, next, context); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 945271b1a929..4138be6e383a 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -17,6 +17,10 @@ SSHStoreConfig::SSHStoreConfig(const ParsedURL::Authority & authority, const Par { } +void SSHStoreConfig::anchor() {} + +void MountedSSHStoreConfig::anchor() {} + std::string SSHStoreConfig::doc() { return @@ -39,6 +43,10 @@ StoreReference SSHStoreConfig::getReference() const struct alignas(8) /* Work around ASAN failures on i686-linux. */ SSHStore : virtual RemoteStore { +private: + void anchor() override; + +public: using Config = SSHStoreConfig; ref config; @@ -87,6 +95,8 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ }; }; +void SSHStore::anchor() {} + MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) : StoreConfig(params, FilePathType::Native) , RemoteStoreConfig(params, FilePathType::Native) @@ -128,6 +138,10 @@ std::string MountedSSHStoreConfig::doc() */ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore { +private: + void anchor() override; + +public: using Config = MountedSSHStoreConfig; MountedSSHStore(ref config) @@ -187,6 +201,8 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore } }; +void MountedSSHStore::anchor() {} + ref SSHStore::Config::openStore() const { return make_ref(ref{shared_from_this()}); diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index e0b2d68ec37c..c4137df7c6aa 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -26,6 +26,10 @@ std::filesystem::path getDaemonSocketPath(const Store::Config & config) .value_or(config.getStateDir() / "daemon-socket" / "socket"); } +void UDSRemoteStoreConfig::anchor() {} + +void UDSRemoteStore::anchor() {} + UDSRemoteStoreConfig::UDSRemoteStoreConfig(const std::filesystem::path & path, const StoreReference::Params & params) : Store::Config{params, FilePathType::Native} , LocalFSStore::Config{params} From 80416579477c30db201b6f9f0e0d19428eab839b Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Tue, 28 Apr 2026 19:36:45 +0200 Subject: [PATCH 329/555] Migrate C++ error trace tests into functional/lang tests. Migrated 138 language-level error trace tests from C++ unit tests to functional characterization tests in tests/functional/lang/. Some tests were commented out due to insufficiant MACROs, but they work well as functional lang tests. Some tests had duplicates, only one copy is kept. The remaning C++ tests (TraceBuilder and NestedThrows) were kept as is, while ~1200 lines were removed from src/libexpr-tests/error_traces.cc. --- src/libexpr-tests/error_traces.cc | 1266 ----------------- tests/functional/lang/eval-fail-add-1.err.exp | 10 + tests/functional/lang/eval-fail-add-1.nix | 1 + tests/functional/lang/eval-fail-add-2.err.exp | 10 + tests/functional/lang/eval-fail-add-2.nix | 1 + tests/functional/lang/eval-fail-all-1.err.exp | 10 + tests/functional/lang/eval-fail-all-1.nix | 1 + tests/functional/lang/eval-fail-all-2.err.exp | 10 + tests/functional/lang/eval-fail-all-2.nix | 1 + tests/functional/lang/eval-fail-all-3.err.exp | 10 + tests/functional/lang/eval-fail-all-3.nix | 1 + tests/functional/lang/eval-fail-any-1.err.exp | 10 + tests/functional/lang/eval-fail-any-1.nix | 1 + tests/functional/lang/eval-fail-any-2.err.exp | 10 + tests/functional/lang/eval-fail-any-2.nix | 1 + tests/functional/lang/eval-fail-any-3.err.exp | 10 + tests/functional/lang/eval-fail-any-3.nix | 1 + .../lang/eval-fail-attrNames-1.err.exp | 10 + .../functional/lang/eval-fail-attrNames-1.nix | 1 + .../lang/eval-fail-attrValues-1.err.exp | 10 + .../lang/eval-fail-attrValues-1.nix | 1 + .../lang/eval-fail-baseNameOf-1.err.exp | 10 + .../lang/eval-fail-baseNameOf-1.nix | 1 + .../lang/eval-fail-bitAnd-1.err.exp | 10 + tests/functional/lang/eval-fail-bitAnd-1.nix | 1 + .../lang/eval-fail-bitAnd-2.err.exp | 10 + tests/functional/lang/eval-fail-bitAnd-2.nix | 1 + .../functional/lang/eval-fail-bitOr-1.err.exp | 10 + tests/functional/lang/eval-fail-bitOr-1.nix | 1 + .../functional/lang/eval-fail-bitOr-2.err.exp | 10 + tests/functional/lang/eval-fail-bitOr-2.nix | 1 + .../lang/eval-fail-bitXor-1.err.exp | 10 + tests/functional/lang/eval-fail-bitXor-1.nix | 1 + .../lang/eval-fail-bitXor-2.err.exp | 10 + tests/functional/lang/eval-fail-bitXor-2.nix | 1 + .../lang/eval-fail-catAttrs-1.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-1.nix | 1 + .../lang/eval-fail-catAttrs-2.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-2.nix | 1 + .../lang/eval-fail-catAttrs-3.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-3.nix | 1 + .../lang/eval-fail-catAttrs-4.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-4.nix | 5 + .../functional/lang/eval-fail-ceil-1.err.exp | 10 + tests/functional/lang/eval-fail-ceil-1.nix | 1 + .../lang/eval-fail-compareVersions-1.err.exp | 10 + .../lang/eval-fail-compareVersions-1.nix | 1 + .../lang/eval-fail-compareVersions-2.err.exp | 10 + .../lang/eval-fail-compareVersions-2.nix | 1 + .../lang/eval-fail-concatLists-1.err.exp | 10 + .../lang/eval-fail-concatLists-1.nix | 1 + .../lang/eval-fail-concatLists-2.err.exp | 10 + .../lang/eval-fail-concatLists-2.nix | 1 + .../lang/eval-fail-concatLists-3.err.exp | 10 + .../lang/eval-fail-concatLists-3.nix | 4 + .../lang/eval-fail-concatMap-1.err.exp | 10 + .../functional/lang/eval-fail-concatMap-1.nix | 1 + .../lang/eval-fail-concatMap-2.err.exp | 10 + .../functional/lang/eval-fail-concatMap-2.nix | 1 + .../lang/eval-fail-concatMap-3.err.exp | 14 + .../functional/lang/eval-fail-concatMap-3.nix | 1 + .../lang/eval-fail-concatMap-4.err.exp | 14 + .../functional/lang/eval-fail-concatMap-4.nix | 4 + .../lang/eval-fail-concatStringsSep-1.err.exp | 10 + .../lang/eval-fail-concatStringsSep-1.nix | 1 + .../lang/eval-fail-concatStringsSep-2.err.exp | 10 + .../lang/eval-fail-concatStringsSep-2.nix | 1 + .../lang/eval-fail-concatStringsSep-3.err.exp | 10 + .../lang/eval-fail-concatStringsSep-3.nix | 5 + .../lang/eval-fail-derivationStrict-1.err.exp | 10 + .../lang/eval-fail-derivationStrict-1.nix | 1 + .../eval-fail-derivationStrict-10.err.exp | 18 + .../lang/eval-fail-derivationStrict-10.nix | 6 + .../eval-fail-derivationStrict-11.err.exp | 18 + .../lang/eval-fail-derivationStrict-11.nix | 6 + .../eval-fail-derivationStrict-12.err.exp | 18 + .../lang/eval-fail-derivationStrict-12.nix | 5 + .../eval-fail-derivationStrict-13.err.exp | 18 + .../lang/eval-fail-derivationStrict-13.nix | 6 + .../eval-fail-derivationStrict-14.err.exp | 18 + .../lang/eval-fail-derivationStrict-14.nix | 6 + .../eval-fail-derivationStrict-15.err.exp | 18 + .../lang/eval-fail-derivationStrict-15.nix | 9 + .../eval-fail-derivationStrict-16.err.exp | 18 + .../lang/eval-fail-derivationStrict-16.nix | 7 + .../eval-fail-derivationStrict-17.err.exp | 18 + .../lang/eval-fail-derivationStrict-17.nix | 7 + .../eval-fail-derivationStrict-19.err.exp | 18 + .../lang/eval-fail-derivationStrict-19.nix | 7 + .../lang/eval-fail-derivationStrict-2.err.exp | 10 + .../lang/eval-fail-derivationStrict-2.nix | 1 + .../eval-fail-derivationStrict-20.err.exp | 20 + .../lang/eval-fail-derivationStrict-20.nix | 7 + .../eval-fail-derivationStrict-21.err.exp | 20 + .../lang/eval-fail-derivationStrict-21.nix | 10 + .../eval-fail-derivationStrict-22.err.exp | 18 + .../lang/eval-fail-derivationStrict-22.nix | 7 + .../lang/eval-fail-derivationStrict-3.err.exp | 16 + .../lang/eval-fail-derivationStrict-3.nix | 1 + .../lang/eval-fail-derivationStrict-4.err.exp | 11 + .../lang/eval-fail-derivationStrict-4.nix | 1 + .../lang/eval-fail-derivationStrict-5.err.exp | 13 + .../lang/eval-fail-derivationStrict-5.nix | 5 + .../lang/eval-fail-derivationStrict-6.err.exp | 13 + .../lang/eval-fail-derivationStrict-6.nix | 5 + .../lang/eval-fail-derivationStrict-7.err.exp | 18 + .../lang/eval-fail-derivationStrict-7.nix | 5 + .../lang/eval-fail-derivationStrict-8.err.exp | 18 + .../lang/eval-fail-derivationStrict-8.nix | 5 + .../lang/eval-fail-derivationStrict-9.err.exp | 18 + .../lang/eval-fail-derivationStrict-9.nix | 5 + tests/functional/lang/eval-fail-div-1.err.exp | 10 + tests/functional/lang/eval-fail-div-1.nix | 1 + tests/functional/lang/eval-fail-div-2.err.exp | 10 + tests/functional/lang/eval-fail-div-2.nix | 1 + tests/functional/lang/eval-fail-div-3.err.exp | 8 + tests/functional/lang/eval-fail-div-3.nix | 1 + .../functional/lang/eval-fail-elem-1.err.exp | 10 + tests/functional/lang/eval-fail-elem-1.nix | 1 + .../lang/eval-fail-elemAt-1.err.exp | 10 + tests/functional/lang/eval-fail-elemAt-1.nix | 1 + .../lang/eval-fail-elemAt-2.err.exp | 8 + tests/functional/lang/eval-fail-elemAt-2.nix | 1 + .../lang/eval-fail-elemAt-3.err.exp | 8 + tests/functional/lang/eval-fail-elemAt-3.nix | 1 + .../lang/eval-fail-filter-1.err.exp | 10 + tests/functional/lang/eval-fail-filter-1.nix | 1 + .../lang/eval-fail-filter-2.err.exp | 10 + tests/functional/lang/eval-fail-filter-2.nix | 1 + .../lang/eval-fail-filter-3.err.exp | 10 + tests/functional/lang/eval-fail-filter-3.nix | 1 + .../lang/eval-fail-filterSource-1.err.exp | 10 + .../lang/eval-fail-filterSource-1.nix | 1 + .../lang/eval-fail-filterSource-2.err.exp | 10 + .../lang/eval-fail-filterSource-2.nix | 1 + .../lang/eval-fail-filterSource-3.err.exp | 10 + .../lang/eval-fail-filterSource-3.nix | 1 + .../lang/eval-fail-filterSource-4.err.exp | 10 + .../lang/eval-fail-filterSource-4.nix | 1 + .../lang/eval-fail-filterSource-5.err.exp | 12 + .../lang/eval-fail-filterSource-5.nix | 1 + .../functional/lang/eval-fail-floor-1.err.exp | 10 + tests/functional/lang/eval-fail-floor-1.nix | 1 + .../lang/eval-fail-foldlPrime-1.err.exp | 10 + .../lang/eval-fail-foldlPrime-1.nix | 1 + .../lang/eval-fail-foldlPrime-2.err.exp | 10 + .../lang/eval-fail-foldlPrime-2.nix | 1 + .../lang/eval-fail-foldlPrime-3.err.exp | 8 + .../lang/eval-fail-foldlPrime-3.nix | 1 + .../lang/eval-fail-foldlPrime-4.err.exp | 24 + .../lang/eval-fail-foldlPrime-4.nix | 1 + .../lang/eval-fail-functionArgs-1.err.exp | 8 + .../lang/eval-fail-functionArgs-1.nix | 1 + .../lang/eval-fail-genList-1.err.exp | 10 + tests/functional/lang/eval-fail-genList-1.nix | 1 + .../lang/eval-fail-genList-2.err.exp | 10 + tests/functional/lang/eval-fail-genList-2.nix | 1 + .../lang/eval-fail-genList-3.err.exp | 20 + tests/functional/lang/eval-fail-genList-3.nix | 1 + .../lang/eval-fail-genList-4.err.exp | 8 + tests/functional/lang/eval-fail-genList-4.nix | 1 + .../lang/eval-fail-getAttr-1.err.exp | 10 + tests/functional/lang/eval-fail-getAttr-1.nix | 1 + .../lang/eval-fail-getAttr-2.err.exp | 10 + tests/functional/lang/eval-fail-getAttr-2.nix | 1 + .../lang/eval-fail-getAttr-3.err.exp | 10 + tests/functional/lang/eval-fail-getAttr-3.nix | 1 + .../lang/eval-fail-getEnv-1.err.exp | 10 + tests/functional/lang/eval-fail-getEnv-1.nix | 1 + .../lang/eval-fail-groupBy-1.err.exp | 10 + tests/functional/lang/eval-fail-groupBy-1.nix | 1 + .../lang/eval-fail-groupBy-2.err.exp | 10 + tests/functional/lang/eval-fail-groupBy-2.nix | 1 + .../lang/eval-fail-groupBy-3.err.exp | 10 + tests/functional/lang/eval-fail-groupBy-3.nix | 5 + .../lang/eval-fail-hasAttr-1.err.exp | 10 + tests/functional/lang/eval-fail-hasAttr-1.nix | 1 + .../lang/eval-fail-hasAttr-2.err.exp | 10 + tests/functional/lang/eval-fail-hasAttr-2.nix | 1 + .../lang/eval-fail-hashString-1.err.exp | 10 + .../lang/eval-fail-hashString-1.nix | 1 + .../lang/eval-fail-hashString-2.err.exp | 9 + .../lang/eval-fail-hashString-2.nix | 1 + .../lang/eval-fail-hashString-3.err.exp | 10 + .../lang/eval-fail-hashString-3.nix | 1 + .../functional/lang/eval-fail-head-1.err.exp | 10 + tests/functional/lang/eval-fail-head-1.nix | 1 + .../functional/lang/eval-fail-head-2.err.exp | 8 + tests/functional/lang/eval-fail-head-2.nix | 1 + .../lang/eval-fail-intersectAttrs-1.err.exp | 10 + .../lang/eval-fail-intersectAttrs-1.nix | 1 + .../lang/eval-fail-intersectAttrs-2.err.exp | 10 + .../lang/eval-fail-intersectAttrs-2.nix | 1 + .../lang/eval-fail-length-1.err.exp | 10 + tests/functional/lang/eval-fail-length-1.nix | 1 + .../lang/eval-fail-length-2.err.exp | 10 + tests/functional/lang/eval-fail-length-2.nix | 1 + .../lang/eval-fail-lessThan-1.err.exp | 8 + .../functional/lang/eval-fail-lessThan-1.nix | 1 + .../lang/eval-fail-lessThan-2.err.exp | 8 + .../functional/lang/eval-fail-lessThan-2.nix | 1 + .../lang/eval-fail-lessThan-3.err.exp | 10 + .../functional/lang/eval-fail-lessThan-3.nix | 1 + .../lang/eval-fail-listToAttrs-1.err.exp | 10 + .../lang/eval-fail-listToAttrs-1.nix | 1 + .../lang/eval-fail-listToAttrs-2.err.exp | 10 + .../lang/eval-fail-listToAttrs-2.nix | 1 + .../lang/eval-fail-listToAttrs-3.err.exp | 10 + .../lang/eval-fail-listToAttrs-3.nix | 1 + .../lang/eval-fail-listToAttrs-4.err.exp | 18 + .../lang/eval-fail-listToAttrs-4.nix | 1 + .../lang/eval-fail-listToAttrs-5.err.exp | 10 + .../lang/eval-fail-listToAttrs-5.nix | 1 + tests/functional/lang/eval-fail-map-1.err.exp | 10 + tests/functional/lang/eval-fail-map-1.nix | 1 + tests/functional/lang/eval-fail-map-2.err.exp | 10 + tests/functional/lang/eval-fail-map-2.nix | 1 + .../lang/eval-fail-mapAttrs-1.err.exp | 10 + .../functional/lang/eval-fail-mapAttrs-1.nix | 1 + .../lang/eval-fail-mapAttrs-2.err.exp | 4 + .../functional/lang/eval-fail-mapAttrs-2.nix | 1 + .../lang/eval-fail-mapAttrs-3.err.exp | 4 + .../functional/lang/eval-fail-mapAttrs-3.nix | 1 + .../lang/eval-fail-mapAttrs-4.err.exp | 16 + .../functional/lang/eval-fail-mapAttrs-4.nix | 1 + .../functional/lang/eval-fail-match-1.err.exp | 10 + tests/functional/lang/eval-fail-match-1.nix | 1 + .../functional/lang/eval-fail-match-2.err.exp | 10 + tests/functional/lang/eval-fail-match-2.nix | 1 + .../functional/lang/eval-fail-match-3.err.exp | 8 + tests/functional/lang/eval-fail-match-3.nix | 1 + tests/functional/lang/eval-fail-mul-1.err.exp | 10 + tests/functional/lang/eval-fail-mul-1.nix | 1 + tests/functional/lang/eval-fail-mul-2.err.exp | 10 + tests/functional/lang/eval-fail-mul-2.nix | 1 + .../lang/eval-fail-parseDrvName-1.err.exp | 10 + .../lang/eval-fail-parseDrvName-1.nix | 1 + .../lang/eval-fail-partition-1.err.exp | 10 + .../functional/lang/eval-fail-partition-1.nix | 1 + .../lang/eval-fail-partition-2.err.exp | 10 + .../functional/lang/eval-fail-partition-2.nix | 1 + .../lang/eval-fail-partition-3.err.exp | 10 + .../functional/lang/eval-fail-partition-3.nix | 1 + .../lang/eval-fail-pathExists-1.err.exp | 10 + .../lang/eval-fail-pathExists-1.nix | 1 + .../lang/eval-fail-pathExists-2.err.exp | 10 + .../lang/eval-fail-pathExists-2.nix | 1 + .../lang/eval-fail-placeholder-1.err.exp | 10 + .../lang/eval-fail-placeholder-1.nix | 1 + .../lang/eval-fail-removeAttrs-1.err.exp | 10 + .../lang/eval-fail-removeAttrs-1.nix | 1 + .../lang/eval-fail-removeAttrs-2.err.exp | 10 + .../lang/eval-fail-removeAttrs-2.nix | 1 + .../lang/eval-fail-removeAttrs-3.err.exp | 10 + .../lang/eval-fail-removeAttrs-3.nix | 1 + .../lang/eval-fail-replaceStrings-1.err.exp | 10 + .../lang/eval-fail-replaceStrings-1.nix | 1 + .../lang/eval-fail-replaceStrings-2.err.exp | 10 + .../lang/eval-fail-replaceStrings-2.nix | 1 + .../lang/eval-fail-replaceStrings-3.err.exp | 8 + .../lang/eval-fail-replaceStrings-3.nix | 1 + .../lang/eval-fail-replaceStrings-4.err.exp | 10 + .../lang/eval-fail-replaceStrings-4.nix | 1 + .../lang/eval-fail-replaceStrings-5.err.exp | 10 + .../lang/eval-fail-replaceStrings-5.nix | 1 + .../lang/eval-fail-replaceStrings-6.err.exp | 10 + .../lang/eval-fail-replaceStrings-6.nix | 1 + .../functional/lang/eval-fail-sort-1.err.exp | 10 + tests/functional/lang/eval-fail-sort-1.nix | 1 + .../functional/lang/eval-fail-sort-2.err.exp | 10 + tests/functional/lang/eval-fail-sort-2.nix | 1 + .../functional/lang/eval-fail-sort-3.err.exp | 8 + tests/functional/lang/eval-fail-sort-3.nix | 4 + .../functional/lang/eval-fail-sort-4.err.exp | 10 + tests/functional/lang/eval-fail-sort-4.nix | 4 + .../functional/lang/eval-fail-sort-5.err.exp | 26 + tests/functional/lang/eval-fail-sort-5.nix | 4 + .../functional/lang/eval-fail-sort-6.err.exp | 26 + tests/functional/lang/eval-fail-sort-6.nix | 4 + .../functional/lang/eval-fail-split-1.err.exp | 10 + tests/functional/lang/eval-fail-split-1.nix | 1 + .../functional/lang/eval-fail-split-2.err.exp | 10 + tests/functional/lang/eval-fail-split-2.nix | 1 + .../functional/lang/eval-fail-split-3.err.exp | 8 + tests/functional/lang/eval-fail-split-3.nix | 1 + .../lang/eval-fail-splitVersion-1.err.exp | 10 + .../lang/eval-fail-splitVersion-1.nix | 1 + .../lang/eval-fail-storePath-1.err.exp | 10 + .../functional/lang/eval-fail-storePath-1.nix | 1 + .../lang/eval-fail-stringLength-1.err.exp | 10 + .../lang/eval-fail-stringLength-1.nix | 1 + tests/functional/lang/eval-fail-sub-1.err.exp | 10 + tests/functional/lang/eval-fail-sub-1.nix | 1 + tests/functional/lang/eval-fail-sub-2.err.exp | 10 + tests/functional/lang/eval-fail-sub-2.nix | 1 + .../lang/eval-fail-substring-1.err.exp | 10 + .../functional/lang/eval-fail-substring-1.nix | 1 + .../lang/eval-fail-substring-2.err.exp | 10 + .../functional/lang/eval-fail-substring-2.nix | 1 + .../lang/eval-fail-substring-3.err.exp | 10 + .../functional/lang/eval-fail-substring-3.nix | 1 + .../lang/eval-fail-substring-4.err.exp | 8 + .../functional/lang/eval-fail-substring-4.nix | 1 + .../functional/lang/eval-fail-tail-1.err.exp | 10 + tests/functional/lang/eval-fail-tail-1.nix | 1 + .../functional/lang/eval-fail-tail-2.err.exp | 8 + tests/functional/lang/eval-fail-tail-2.nix | 1 + .../lang/eval-fail-toPath-1.err.exp | 10 + tests/functional/lang/eval-fail-toPath-1.nix | 1 + .../lang/eval-fail-toPath-2.err.exp | 10 + tests/functional/lang/eval-fail-toPath-2.nix | 1 + .../lang/eval-fail-toString-1.err.exp | 10 + .../functional/lang/eval-fail-toString-1.nix | 1 + .../lang/eval-fail-zipAttrsWith-1.err.exp | 10 + .../lang/eval-fail-zipAttrsWith-1.nix | 1 + .../lang/eval-fail-zipAttrsWith-2.err.exp | 10 + .../lang/eval-fail-zipAttrsWith-2.nix | 1 + .../lang/eval-fail-zipAttrsWith-3.err.exp | 8 + .../lang/eval-fail-zipAttrsWith-3.nix | 1 + .../lang/eval-fail-zipAttrsWith-4.err.exp | 22 + .../lang/eval-fail-zipAttrsWith-4.nix | 4 + 321 files changed, 2068 insertions(+), 1266 deletions(-) create mode 100644 tests/functional/lang/eval-fail-add-1.err.exp create mode 100644 tests/functional/lang/eval-fail-add-1.nix create mode 100644 tests/functional/lang/eval-fail-add-2.err.exp create mode 100644 tests/functional/lang/eval-fail-add-2.nix create mode 100644 tests/functional/lang/eval-fail-all-1.err.exp create mode 100644 tests/functional/lang/eval-fail-all-1.nix create mode 100644 tests/functional/lang/eval-fail-all-2.err.exp create mode 100644 tests/functional/lang/eval-fail-all-2.nix create mode 100644 tests/functional/lang/eval-fail-all-3.err.exp create mode 100644 tests/functional/lang/eval-fail-all-3.nix create mode 100644 tests/functional/lang/eval-fail-any-1.err.exp create mode 100644 tests/functional/lang/eval-fail-any-1.nix create mode 100644 tests/functional/lang/eval-fail-any-2.err.exp create mode 100644 tests/functional/lang/eval-fail-any-2.nix create mode 100644 tests/functional/lang/eval-fail-any-3.err.exp create mode 100644 tests/functional/lang/eval-fail-any-3.nix create mode 100644 tests/functional/lang/eval-fail-attrNames-1.err.exp create mode 100644 tests/functional/lang/eval-fail-attrNames-1.nix create mode 100644 tests/functional/lang/eval-fail-attrValues-1.err.exp create mode 100644 tests/functional/lang/eval-fail-attrValues-1.nix create mode 100644 tests/functional/lang/eval-fail-baseNameOf-1.err.exp create mode 100644 tests/functional/lang/eval-fail-baseNameOf-1.nix create mode 100644 tests/functional/lang/eval-fail-bitAnd-1.err.exp create mode 100644 tests/functional/lang/eval-fail-bitAnd-1.nix create mode 100644 tests/functional/lang/eval-fail-bitAnd-2.err.exp create mode 100644 tests/functional/lang/eval-fail-bitAnd-2.nix create mode 100644 tests/functional/lang/eval-fail-bitOr-1.err.exp create mode 100644 tests/functional/lang/eval-fail-bitOr-1.nix create mode 100644 tests/functional/lang/eval-fail-bitOr-2.err.exp create mode 100644 tests/functional/lang/eval-fail-bitOr-2.nix create mode 100644 tests/functional/lang/eval-fail-bitXor-1.err.exp create mode 100644 tests/functional/lang/eval-fail-bitXor-1.nix create mode 100644 tests/functional/lang/eval-fail-bitXor-2.err.exp create mode 100644 tests/functional/lang/eval-fail-bitXor-2.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-4.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-4.nix create mode 100644 tests/functional/lang/eval-fail-ceil-1.err.exp create mode 100644 tests/functional/lang/eval-fail-ceil-1.nix create mode 100644 tests/functional/lang/eval-fail-compareVersions-1.err.exp create mode 100644 tests/functional/lang/eval-fail-compareVersions-1.nix create mode 100644 tests/functional/lang/eval-fail-compareVersions-2.err.exp create mode 100644 tests/functional/lang/eval-fail-compareVersions-2.nix create mode 100644 tests/functional/lang/eval-fail-concatLists-1.err.exp create mode 100644 tests/functional/lang/eval-fail-concatLists-1.nix create mode 100644 tests/functional/lang/eval-fail-concatLists-2.err.exp create mode 100644 tests/functional/lang/eval-fail-concatLists-2.nix create mode 100644 tests/functional/lang/eval-fail-concatLists-3.err.exp create mode 100644 tests/functional/lang/eval-fail-concatLists-3.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-1.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-1.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-2.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-2.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-3.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-3.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-4.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-4.nix create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-1.err.exp create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-1.nix create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-2.err.exp create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-2.nix create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-3.err.exp create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-3.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-1.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-1.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-10.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-10.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-11.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-11.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-12.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-12.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-13.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-13.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-14.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-14.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-15.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-15.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-16.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-16.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-17.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-17.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-19.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-19.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-2.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-2.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-20.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-20.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-21.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-21.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-22.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-22.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-3.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-3.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-4.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-4.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-5.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-5.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-6.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-6.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-7.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-7.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-8.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-8.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-9.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-9.nix create mode 100644 tests/functional/lang/eval-fail-div-1.err.exp create mode 100644 tests/functional/lang/eval-fail-div-1.nix create mode 100644 tests/functional/lang/eval-fail-div-2.err.exp create mode 100644 tests/functional/lang/eval-fail-div-2.nix create mode 100644 tests/functional/lang/eval-fail-div-3.err.exp create mode 100644 tests/functional/lang/eval-fail-div-3.nix create mode 100644 tests/functional/lang/eval-fail-elem-1.err.exp create mode 100644 tests/functional/lang/eval-fail-elem-1.nix create mode 100644 tests/functional/lang/eval-fail-elemAt-1.err.exp create mode 100644 tests/functional/lang/eval-fail-elemAt-1.nix create mode 100644 tests/functional/lang/eval-fail-elemAt-2.err.exp create mode 100644 tests/functional/lang/eval-fail-elemAt-2.nix create mode 100644 tests/functional/lang/eval-fail-elemAt-3.err.exp create mode 100644 tests/functional/lang/eval-fail-elemAt-3.nix create mode 100644 tests/functional/lang/eval-fail-filter-1.err.exp create mode 100644 tests/functional/lang/eval-fail-filter-1.nix create mode 100644 tests/functional/lang/eval-fail-filter-2.err.exp create mode 100644 tests/functional/lang/eval-fail-filter-2.nix create mode 100644 tests/functional/lang/eval-fail-filter-3.err.exp create mode 100644 tests/functional/lang/eval-fail-filter-3.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-1.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-1.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-2.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-2.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-3.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-3.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-4.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-4.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-5.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-5.nix create mode 100644 tests/functional/lang/eval-fail-floor-1.err.exp create mode 100644 tests/functional/lang/eval-fail-floor-1.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-1.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-1.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-2.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-2.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-3.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-3.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-4.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-4.nix create mode 100644 tests/functional/lang/eval-fail-functionArgs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-functionArgs-1.nix create mode 100644 tests/functional/lang/eval-fail-genList-1.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-1.nix create mode 100644 tests/functional/lang/eval-fail-genList-2.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-2.nix create mode 100644 tests/functional/lang/eval-fail-genList-3.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-3.nix create mode 100644 tests/functional/lang/eval-fail-genList-4.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-4.nix create mode 100644 tests/functional/lang/eval-fail-getAttr-1.err.exp create mode 100644 tests/functional/lang/eval-fail-getAttr-1.nix create mode 100644 tests/functional/lang/eval-fail-getAttr-2.err.exp create mode 100644 tests/functional/lang/eval-fail-getAttr-2.nix create mode 100644 tests/functional/lang/eval-fail-getAttr-3.err.exp create mode 100644 tests/functional/lang/eval-fail-getAttr-3.nix create mode 100644 tests/functional/lang/eval-fail-getEnv-1.err.exp create mode 100644 tests/functional/lang/eval-fail-getEnv-1.nix create mode 100644 tests/functional/lang/eval-fail-groupBy-1.err.exp create mode 100644 tests/functional/lang/eval-fail-groupBy-1.nix create mode 100644 tests/functional/lang/eval-fail-groupBy-2.err.exp create mode 100644 tests/functional/lang/eval-fail-groupBy-2.nix create mode 100644 tests/functional/lang/eval-fail-groupBy-3.err.exp create mode 100644 tests/functional/lang/eval-fail-groupBy-3.nix create mode 100644 tests/functional/lang/eval-fail-hasAttr-1.err.exp create mode 100644 tests/functional/lang/eval-fail-hasAttr-1.nix create mode 100644 tests/functional/lang/eval-fail-hasAttr-2.err.exp create mode 100644 tests/functional/lang/eval-fail-hasAttr-2.nix create mode 100644 tests/functional/lang/eval-fail-hashString-1.err.exp create mode 100644 tests/functional/lang/eval-fail-hashString-1.nix create mode 100644 tests/functional/lang/eval-fail-hashString-2.err.exp create mode 100644 tests/functional/lang/eval-fail-hashString-2.nix create mode 100644 tests/functional/lang/eval-fail-hashString-3.err.exp create mode 100644 tests/functional/lang/eval-fail-hashString-3.nix create mode 100644 tests/functional/lang/eval-fail-head-1.err.exp create mode 100644 tests/functional/lang/eval-fail-head-1.nix create mode 100644 tests/functional/lang/eval-fail-head-2.err.exp create mode 100644 tests/functional/lang/eval-fail-head-2.nix create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-length-1.err.exp create mode 100644 tests/functional/lang/eval-fail-length-1.nix create mode 100644 tests/functional/lang/eval-fail-length-2.err.exp create mode 100644 tests/functional/lang/eval-fail-length-2.nix create mode 100644 tests/functional/lang/eval-fail-lessThan-1.err.exp create mode 100644 tests/functional/lang/eval-fail-lessThan-1.nix create mode 100644 tests/functional/lang/eval-fail-lessThan-2.err.exp create mode 100644 tests/functional/lang/eval-fail-lessThan-2.nix create mode 100644 tests/functional/lang/eval-fail-lessThan-3.err.exp create mode 100644 tests/functional/lang/eval-fail-lessThan-3.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-4.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-4.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-5.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-5.nix create mode 100644 tests/functional/lang/eval-fail-map-1.err.exp create mode 100644 tests/functional/lang/eval-fail-map-1.nix create mode 100644 tests/functional/lang/eval-fail-map-2.err.exp create mode 100644 tests/functional/lang/eval-fail-map-2.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-4.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-4.nix create mode 100644 tests/functional/lang/eval-fail-match-1.err.exp create mode 100644 tests/functional/lang/eval-fail-match-1.nix create mode 100644 tests/functional/lang/eval-fail-match-2.err.exp create mode 100644 tests/functional/lang/eval-fail-match-2.nix create mode 100644 tests/functional/lang/eval-fail-match-3.err.exp create mode 100644 tests/functional/lang/eval-fail-match-3.nix create mode 100644 tests/functional/lang/eval-fail-mul-1.err.exp create mode 100644 tests/functional/lang/eval-fail-mul-1.nix create mode 100644 tests/functional/lang/eval-fail-mul-2.err.exp create mode 100644 tests/functional/lang/eval-fail-mul-2.nix create mode 100644 tests/functional/lang/eval-fail-parseDrvName-1.err.exp create mode 100644 tests/functional/lang/eval-fail-parseDrvName-1.nix create mode 100644 tests/functional/lang/eval-fail-partition-1.err.exp create mode 100644 tests/functional/lang/eval-fail-partition-1.nix create mode 100644 tests/functional/lang/eval-fail-partition-2.err.exp create mode 100644 tests/functional/lang/eval-fail-partition-2.nix create mode 100644 tests/functional/lang/eval-fail-partition-3.err.exp create mode 100644 tests/functional/lang/eval-fail-partition-3.nix create mode 100644 tests/functional/lang/eval-fail-pathExists-1.err.exp create mode 100644 tests/functional/lang/eval-fail-pathExists-1.nix create mode 100644 tests/functional/lang/eval-fail-pathExists-2.err.exp create mode 100644 tests/functional/lang/eval-fail-pathExists-2.nix create mode 100644 tests/functional/lang/eval-fail-placeholder-1.err.exp create mode 100644 tests/functional/lang/eval-fail-placeholder-1.nix create mode 100644 tests/functional/lang/eval-fail-removeAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-removeAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-removeAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-removeAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-removeAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-removeAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-1.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-1.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-2.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-2.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-3.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-3.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-4.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-4.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-5.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-5.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-6.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-6.nix create mode 100644 tests/functional/lang/eval-fail-sort-1.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-1.nix create mode 100644 tests/functional/lang/eval-fail-sort-2.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-2.nix create mode 100644 tests/functional/lang/eval-fail-sort-3.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-3.nix create mode 100644 tests/functional/lang/eval-fail-sort-4.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-4.nix create mode 100644 tests/functional/lang/eval-fail-sort-5.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-5.nix create mode 100644 tests/functional/lang/eval-fail-sort-6.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-6.nix create mode 100644 tests/functional/lang/eval-fail-split-1.err.exp create mode 100644 tests/functional/lang/eval-fail-split-1.nix create mode 100644 tests/functional/lang/eval-fail-split-2.err.exp create mode 100644 tests/functional/lang/eval-fail-split-2.nix create mode 100644 tests/functional/lang/eval-fail-split-3.err.exp create mode 100644 tests/functional/lang/eval-fail-split-3.nix create mode 100644 tests/functional/lang/eval-fail-splitVersion-1.err.exp create mode 100644 tests/functional/lang/eval-fail-splitVersion-1.nix create mode 100644 tests/functional/lang/eval-fail-storePath-1.err.exp create mode 100644 tests/functional/lang/eval-fail-storePath-1.nix create mode 100644 tests/functional/lang/eval-fail-stringLength-1.err.exp create mode 100644 tests/functional/lang/eval-fail-stringLength-1.nix create mode 100644 tests/functional/lang/eval-fail-sub-1.err.exp create mode 100644 tests/functional/lang/eval-fail-sub-1.nix create mode 100644 tests/functional/lang/eval-fail-sub-2.err.exp create mode 100644 tests/functional/lang/eval-fail-sub-2.nix create mode 100644 tests/functional/lang/eval-fail-substring-1.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-1.nix create mode 100644 tests/functional/lang/eval-fail-substring-2.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-2.nix create mode 100644 tests/functional/lang/eval-fail-substring-3.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-3.nix create mode 100644 tests/functional/lang/eval-fail-substring-4.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-4.nix create mode 100644 tests/functional/lang/eval-fail-tail-1.err.exp create mode 100644 tests/functional/lang/eval-fail-tail-1.nix create mode 100644 tests/functional/lang/eval-fail-tail-2.err.exp create mode 100644 tests/functional/lang/eval-fail-tail-2.nix create mode 100644 tests/functional/lang/eval-fail-toPath-1.err.exp create mode 100644 tests/functional/lang/eval-fail-toPath-1.nix create mode 100644 tests/functional/lang/eval-fail-toPath-2.err.exp create mode 100644 tests/functional/lang/eval-fail-toPath-2.nix create mode 100644 tests/functional/lang/eval-fail-toString-1.err.exp create mode 100644 tests/functional/lang/eval-fail-toString-1.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-1.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-2.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-3.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-4.nix diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index e722cc48499a..9f2d1f92fa3a 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -54,1270 +54,4 @@ TEST_F(ErrorTraceTest, NestedThrows) } } -#define ASSERT_TRACE1(args, type, message) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 1u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE2(args, type, message, context) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 2u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE3(args, type, message, context1, context2) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 3u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE4(args, type, message, context1, context2, context3) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 4u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context3)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -// We assume that expr starts with "builtins.derivationStrict { name =", -// otherwise the name attribute position (1, 29) would be invalid. -#define DERIVATION_TRACE_HINTFMT(name) \ - HintFmt( \ - "while evaluating derivation '%s'\n" \ - " whose name attribute is located at %s", \ - name, \ - Pos(1, 29, Pos::String{.source = make_ref(expr)})) - -// To keep things simple, we also assume that derivation name is "foo". -#define ASSERT_DERIVATION_TRACE1(args, type, message) \ - ASSERT_TRACE2(args, type, message, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE2(args, type, message, context) \ - ASSERT_TRACE3(args, type, message, context, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE3(args, type, message, context1, context2) \ - ASSERT_TRACE4(args, type, message, context1, context2, DERIVATION_TRACE_HINTFMT("foo")) - -TEST_F(ErrorTraceTest, replaceStrings) -{ - ASSERT_TRACE2( - "replaceStrings 0 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [] 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.replaceStrings")); - - ASSERT_TRACE1( - "replaceStrings [ 0 ] [] {}", - EvalError, - HintFmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); - - ASSERT_TRACE2( - "replaceStrings [ 1 ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"oo\" ] [ true ] \"foo\"", - TypeError, - HintFmt("expected a string but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"old\" ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument passed to builtins.replaceStrings")); -} - -TEST_F(ErrorTraceTest, scopedImport) {} - -TEST_F(ErrorTraceTest, import) {} - -TEST_F(ErrorTraceTest, typeOf) {} - -TEST_F(ErrorTraceTest, isNull) {} - -TEST_F(ErrorTraceTest, isFunction) {} - -TEST_F(ErrorTraceTest, isInt) {} - -TEST_F(ErrorTraceTest, isFloat) {} - -TEST_F(ErrorTraceTest, isString) {} - -TEST_F(ErrorTraceTest, isBool) {} - -TEST_F(ErrorTraceTest, isPath) {} - -TEST_F(ErrorTraceTest, break) {} - -TEST_F(ErrorTraceTest, abort) {} - -TEST_F(ErrorTraceTest, throw) {} - -TEST_F(ErrorTraceTest, addErrorContext) {} - -TEST_F(ErrorTraceTest, ceil) -{ - ASSERT_TRACE2( - "ceil \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.ceil")); -} - -TEST_F(ErrorTraceTest, floor) -{ - ASSERT_TRACE2( - "floor \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.floor")); -} - -TEST_F(ErrorTraceTest, tryEval) {} - -TEST_F(ErrorTraceTest, getEnv) -{ - ASSERT_TRACE2( - "getEnv [ ]", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getEnv")); -} - -TEST_F(ErrorTraceTest, seq) {} - -TEST_F(ErrorTraceTest, deepSeq) {} - -TEST_F(ErrorTraceTest, trace) {} - -TEST_F(ErrorTraceTest, placeholder) -{ - ASSERT_TRACE2( - "placeholder []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.placeholder")); -} - -TEST_F(ErrorTraceTest, toPath) -{ - ASSERT_TRACE2( - "toPath []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.toPath")); - - ASSERT_TRACE2( - "toPath \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the first argument passed to builtins.toPath")); -} - -TEST_F(ErrorTraceTest, storePath) -{ - ASSERT_TRACE2( - "storePath true", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.storePath'")); -} - -TEST_F(ErrorTraceTest, pathExists) -{ - ASSERT_TRACE2( - "pathExists []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while realising the context of a path")); - - ASSERT_TRACE2( - "pathExists \"zorglub\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "zorglub"), - HintFmt("while realising the context of a path")); -} - -TEST_F(ErrorTraceTest, baseNameOf) -{ - ASSERT_TRACE2( - "baseNameOf []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.baseNameOf")); -} - -TEST_F(ErrorTraceTest, dirOf) {} - -TEST_F(ErrorTraceTest, readFile) {} - -TEST_F(ErrorTraceTest, findFile) {} - -TEST_F(ErrorTraceTest, hashFile) {} - -TEST_F(ErrorTraceTest, readDir) {} - -TEST_F(ErrorTraceTest, toXML) {} - -TEST_F(ErrorTraceTest, toJSON) {} - -TEST_F(ErrorTraceTest, fromJSON) {} - -TEST_F(ErrorTraceTest, toFile) {} - -TEST_F(ErrorTraceTest, filterSource) -{ - ASSERT_TRACE2( - "filterSource [] []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] ./.", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.filterSource")); - - // Unsupported by store "dummy" - - // ASSERT_TRACE2("filterSource (_: 1) ./.", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while adding path '/home/layus/projects/nix'")); - - // ASSERT_TRACE2("filterSource (_: _: 1) ./.", - // TypeError, - // HintFmt("expected a Boolean but found %s: %s", "an integer", "1"), - // HintFmt("while evaluating the return value of the path filter function")); -} - -TEST_F(ErrorTraceTest, path) {} - -TEST_F(ErrorTraceTest, attrNames) -{ - ASSERT_TRACE2( - "attrNames []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrNames")); -} - -TEST_F(ErrorTraceTest, attrValues) -{ - ASSERT_TRACE2( - "attrValues []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrValues")); -} - -TEST_F(ErrorTraceTest, getAttr) -{ - ASSERT_TRACE2( - "getAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" {}", - TypeError, - HintFmt("attribute '%s' missing", "foo"), - HintFmt("in the attribute set under consideration")); -} - -TEST_F(ErrorTraceTest, unsafeGetAttrPos) {} - -TEST_F(ErrorTraceTest, hasAttr) -{ - ASSERT_TRACE2( - "hasAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.hasAttr")); - - ASSERT_TRACE2( - "hasAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.hasAttr")); -} - -TEST_F(ErrorTraceTest, isAttrs) {} - -TEST_F(ErrorTraceTest, removeAttrs) -{ - ASSERT_TRACE2( - "removeAttrs \"\" \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ \"1\" ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); -} - -TEST_F(ErrorTraceTest, listToAttrs) -{ - ASSERT_TRACE2( - "listToAttrs 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the argument passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ {} ]", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in a {name=...; value=...;} pair")); - - ASSERT_TRACE2( - "listToAttrs [ { name = 1; } ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ { name = \"foo\"; } ]", - TypeError, - HintFmt("attribute '%s' missing", "value"), - HintFmt("in a {name=...; value=...;} pair")); -} - -TEST_F(ErrorTraceTest, intersectAttrs) -{ - ASSERT_TRACE2( - "intersectAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.intersectAttrs")); - - ASSERT_TRACE2( - "intersectAttrs {} []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.intersectAttrs")); -} - -TEST_F(ErrorTraceTest, catAttrs) -{ - ASSERT_TRACE2( - "catAttrs [] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ { foo = 1; } 1 { bar = 5;} ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); -} - -TEST_F(ErrorTraceTest, functionArgs) -{ - ASSERT_TRACE1("functionArgs {}", TypeError, HintFmt("'functionArgs' requires a function")); -} - -TEST_F(ErrorTraceTest, mapAttrs) -{ - ASSERT_TRACE2( - "mapAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.mapAttrs")); - - // XXX: deferred - // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: x + \"1\") { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: y: x + 1) { foo.bar = 1; }", - // TypeError, - // HintFmt("cannot coerce %s to a string", "an integer"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, zipAttrsWith) -{ - ASSERT_TRACE2( - "zipAttrsWith [] [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.zipAttrsWith")); - - ASSERT_TRACE2( - "zipAttrsWith (_: 1) [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); - - // XXX: How to properly tell that the function takes two arguments ? - // The same question also applies to sort, and maybe others. - // Due to laziness, we only create a thunk, and it fails later on. - // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while evaluating the attribute 'foo'")); - - // XXX: Also deferred deeply - // ASSERT_TRACE2("zipAttrsWith (a: b: a + b) [ { foo = 1; } { foo = 2; } ]", - // TypeError, - // HintFmt("cannot coerce %s to a string", "a list"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, isList) {} - -TEST_F(ErrorTraceTest, elemAt) -{ - ASSERT_TRACE2( - "elemAt \"foo\" (-1)", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.elemAt'")); - - ASSERT_TRACE1( - "elemAt [] (-1)", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", -1, 0)); - - ASSERT_TRACE1( - "elemAt [\"foo\"] 3", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", 3, 1)); -} - -TEST_F(ErrorTraceTest, head) -{ - ASSERT_TRACE2( - "head 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.head'")); - - ASSERT_TRACE1("head []", Error, HintFmt("'builtins.head' called on an empty list")); -} - -TEST_F(ErrorTraceTest, tail) -{ - ASSERT_TRACE2( - "tail 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.tail'")); - - ASSERT_TRACE1("tail []", Error, HintFmt("'builtins.tail' called on an empty list")); -} - -TEST_F(ErrorTraceTest, map) -{ - ASSERT_TRACE2( - "map 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.map")); - - ASSERT_TRACE2( - "map 1 [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.map")); -} - -TEST_F(ErrorTraceTest, filter) -{ - ASSERT_TRACE2( - "filter 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter (_: 5) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "5" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the filtering function passed to builtins.filter")); -} - -TEST_F(ErrorTraceTest, elem) -{ - ASSERT_TRACE2( - "elem 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.elem")); -} - -TEST_F(ErrorTraceTest, concatLists) -{ - ASSERT_TRACE2( - "concatLists 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ 1 ]", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ [1] \"foo\" ]", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); -} - -TEST_F(ErrorTraceTest, length) -{ - ASSERT_TRACE2( - "length 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); - - ASSERT_TRACE2( - "length \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); -} - -TEST_F(ErrorTraceTest, foldlPrime) -{ - ASSERT_TRACE2( - "foldl' 1 \"foo\" true", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.foldlStrict")); - - ASSERT_TRACE2( - "foldl' (_: 1) \"foo\" true", - TypeError, - HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the third argument passed to builtins.foldlStrict")); - - ASSERT_TRACE1( - "foldl' (_: 1) \"foo\" [ true ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "foldl' (a: b: a && b) \"foo\" [ true ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("in the left operand of the AND (&&) operator")); -} - -TEST_F(ErrorTraceTest, any) -{ - ASSERT_TRACE2( - "any 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.any")); -} - -TEST_F(ErrorTraceTest, all) -{ - ASSERT_TRACE2( - "all 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.all")); -} - -TEST_F(ErrorTraceTest, genList) -{ - ASSERT_TRACE2( - "genList 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.genList")); - - ASSERT_TRACE2( - "genList 1 2", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.genList")); - - // XXX: deferred - // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO", - // TypeError, - // HintFmt("cannot add %s to an integer", "a string"), - // HintFmt("while evaluating anonymous lambda")); - - ASSERT_TRACE1("genList false (-3)", EvalError, HintFmt("cannot create list of size %d", -3)); -} - -TEST_F(ErrorTraceTest, sort) -{ - ASSERT_TRACE2( - "sort 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.sort")); - - ASSERT_TRACE2( - "sort 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.sort")); - - ASSERT_TRACE1( - "sort (_: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "sort (_: _: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the sorting function passed to builtins.sort")); - - // XXX: Trace too deep, need better asserts - // ASSERT_TRACE1("sort (a: b: a <= b) [ \"foo\" {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s", "a string", "a set")); - - // ASSERT_TRACE1("sort (a: b: a <= b) [ {} {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); -} - -TEST_F(ErrorTraceTest, partition) -{ - ASSERT_TRACE2( - "partition 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the partition function passed to builtins.partition")); -} - -TEST_F(ErrorTraceTest, groupBy) -{ - ASSERT_TRACE2( - "groupBy 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (x: x) [ \"foo\" \"bar\" 1 ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); -} - -TEST_F(ErrorTraceTest, concatMap) -{ - ASSERT_TRACE2( - "concatMap 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) [ \"foo\" ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: \"foo\") [ 1 2 ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); -} - -TEST_F(ErrorTraceTest, add) -{ - ASSERT_TRACE2( - "add \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the addition")); - - ASSERT_TRACE2( - "add 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the addition")); -} - -TEST_F(ErrorTraceTest, sub) -{ - ASSERT_TRACE2( - "sub \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the subtraction")); - - ASSERT_TRACE2( - "sub 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the subtraction")); -} - -TEST_F(ErrorTraceTest, mul) -{ - ASSERT_TRACE2( - "mul \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the multiplication")); - - ASSERT_TRACE2( - "mul 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the multiplication")); -} - -TEST_F(ErrorTraceTest, div) -{ - ASSERT_TRACE2( - "div \"foo\" 1 # TODO: an integer was expected -> a number", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first operand of the division")); - - ASSERT_TRACE2( - "div 1 \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second operand of the division")); - - ASSERT_TRACE1("div \"foo\" 0", EvalError, HintFmt("division by zero")); -} - -TEST_F(ErrorTraceTest, bitAnd) -{ - ASSERT_TRACE2( - "bitAnd 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitAnd")); - - ASSERT_TRACE2( - "bitAnd 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitAnd")); -} - -TEST_F(ErrorTraceTest, bitOr) -{ - ASSERT_TRACE2( - "bitOr 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitOr")); - - ASSERT_TRACE2( - "bitOr 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitOr")); -} - -TEST_F(ErrorTraceTest, bitXor) -{ - ASSERT_TRACE2( - "bitXor 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitXor")); - - ASSERT_TRACE2( - "bitXor 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitXor")); -} - -TEST_F(ErrorTraceTest, lessThan) -{ - ASSERT_TRACE1( - "lessThan 1 \"foo\"", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL))); - - ASSERT_TRACE1( - "lessThan {} {}", - EvalError, - HintFmt( - "cannot compare %s with %s; values of that type are incomparable (values are %s and %s)", - "a set", - "a set", - Uncolored("{ }"), - Uncolored("{ }"))); - - ASSERT_TRACE2( - "lessThan [ 1 2 ] [ \"foo\" ]", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while comparing two list elements")); -} - -TEST_F(ErrorTraceTest, toString) -{ - ASSERT_TRACE2( - "toString { a = 1; }", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ a = " ANSI_CYAN "1" ANSI_NORMAL "; }")), - HintFmt("while evaluating the first argument passed to builtins.toString")); -} - -TEST_F(ErrorTraceTest, substring) -{ - ASSERT_TRACE2( - "substring {} \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the first argument (the start offset) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 3 \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument (the substring length) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 0 3 {}", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument (the string) passed to builtins.substring")); - - ASSERT_TRACE1("substring (-3) 3 \"sometext\"", EvalError, HintFmt("negative start position in 'substring'")); -} - -TEST_F(ErrorTraceTest, stringLength) -{ - ASSERT_TRACE2( - "stringLength {} # TODO: context is missing ???", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the argument passed to builtins.stringLength")); -} - -TEST_F(ErrorTraceTest, hashString) -{ - ASSERT_TRACE2( - "hashString 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.hashString")); - - ASSERT_TRACE1( - "hashString \"foo\" \"content\"", - UsageError, - HintFmt("unknown hash algorithm '%s', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); - - ASSERT_TRACE2( - "hashString \"sha256\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.hashString")); -} - -TEST_F(ErrorTraceTest, match) -{ - ASSERT_TRACE2( - "match 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.match")); - - ASSERT_TRACE2( - "match \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.match")); - - ASSERT_TRACE1("match \"(.*\" \"\"", EvalError, HintFmt("invalid regular expression '%s'", "(.*")); -} - -TEST_F(ErrorTraceTest, split) -{ - ASSERT_TRACE2( - "split 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.split")); - - ASSERT_TRACE2( - "split \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.split")); - - ASSERT_TRACE1("split \"f(o*o\" \"1foo2\"", EvalError, HintFmt("invalid regular expression '%s'", "f(o*o")); -} - -TEST_F(ErrorTraceTest, concatStringsSep) -{ - ASSERT_TRACE2( - "concatStringsSep 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt( - "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); -} - -TEST_F(ErrorTraceTest, parseDrvName) -{ - ASSERT_TRACE2( - "parseDrvName 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.parseDrvName")); -} - -TEST_F(ErrorTraceTest, compareVersions) -{ - ASSERT_TRACE2( - "compareVersions 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.compareVersions")); - - ASSERT_TRACE2( - "compareVersions \"abd\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.compareVersions")); -} - -TEST_F(ErrorTraceTest, splitVersion) -{ - ASSERT_TRACE2( - "splitVersion 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.splitVersion")); -} - -TEST_F(ErrorTraceTest, traceVerbose) {} - -TEST_F(ErrorTraceTest, derivationStrict) -{ - ASSERT_TRACE2( - "derivationStrict \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", "\"\""), - HintFmt("while evaluating the argument passed to builtins.derivationStrict")); - - ASSERT_TRACE2( - "derivationStrict {}", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in the attrset passed as argument to builtins.derivationStrict")); - - ASSERT_TRACE3( - "derivationStrict { name = 1; }", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute passed to builtins.derivationStrict"), - HintFmt("while evaluating the derivation attribute 'name'")); - - ASSERT_DERIVATION_TRACE1( - "derivationStrict { name = \"foo\"; }", EvalError, HintFmt("required attribute 'builder' missing")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __structuredAttrs = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __ignoreNulls = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = 15; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "15"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = \"custom\"; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "custom"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "system", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drvPath\"; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; outputs = \"out\"; __structuredAttrs = true; }", - EvalError, - HintFmt("expected a list but found %s: %s", "a string", "\"out\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = []; }", - EvalError, - HintFmt("derivation cannot have an empty set of outputs"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"drvPath\" ]; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"out\" \"out\" ]; }", - EvalError, - HintFmt("duplicate derivation output '%s'", "out"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __contentAddressed = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__contentAddressed", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = \"foo\"; }", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", "\"foo\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "FOO", "foo")); -} - } /* namespace nix */ diff --git a/tests/functional/lang/eval-fail-add-1.err.exp b/tests/functional/lang/eval-fail-add-1.err.exp new file mode 100644 index 000000000000..4a72eba125e7 --- /dev/null +++ b/tests/functional/lang/eval-fail-add-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'add' builtin + at /pwd/lang/eval-fail-add-1.nix:1:1: + 1| builtins.add "foo" 1 + | ^ + 2| + + … while evaluating the first argument of the addition + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-add-1.nix b/tests/functional/lang/eval-fail-add-1.nix new file mode 100644 index 000000000000..4a59d94e68af --- /dev/null +++ b/tests/functional/lang/eval-fail-add-1.nix @@ -0,0 +1 @@ +builtins.add "foo" 1 diff --git a/tests/functional/lang/eval-fail-add-2.err.exp b/tests/functional/lang/eval-fail-add-2.err.exp new file mode 100644 index 000000000000..429636348d27 --- /dev/null +++ b/tests/functional/lang/eval-fail-add-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'add' builtin + at /pwd/lang/eval-fail-add-2.nix:1:1: + 1| builtins.add 1 "foo" + | ^ + 2| + + … while evaluating the second argument of the addition + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-add-2.nix b/tests/functional/lang/eval-fail-add-2.nix new file mode 100644 index 000000000000..1b785aba2700 --- /dev/null +++ b/tests/functional/lang/eval-fail-add-2.nix @@ -0,0 +1 @@ +builtins.add 1 "foo" diff --git a/tests/functional/lang/eval-fail-all-1.err.exp b/tests/functional/lang/eval-fail-all-1.err.exp new file mode 100644 index 000000000000..95e8735f8ca5 --- /dev/null +++ b/tests/functional/lang/eval-fail-all-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'all' builtin + at /pwd/lang/eval-fail-all-1.nix:1:1: + 1| builtins.all 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.all + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-all-1.nix b/tests/functional/lang/eval-fail-all-1.nix new file mode 100644 index 000000000000..c1ae6ff5e18c --- /dev/null +++ b/tests/functional/lang/eval-fail-all-1.nix @@ -0,0 +1 @@ +builtins.all 1 "foo" diff --git a/tests/functional/lang/eval-fail-all-2.err.exp b/tests/functional/lang/eval-fail-all-2.err.exp new file mode 100644 index 000000000000..584543f5e85e --- /dev/null +++ b/tests/functional/lang/eval-fail-all-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'all' builtin + at /pwd/lang/eval-fail-all-2.nix:1:1: + 1| builtins.all (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.all + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-all-2.nix b/tests/functional/lang/eval-fail-all-2.nix new file mode 100644 index 000000000000..b8ec8c87125c --- /dev/null +++ b/tests/functional/lang/eval-fail-all-2.nix @@ -0,0 +1 @@ +builtins.all (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-all-3.err.exp b/tests/functional/lang/eval-fail-all-3.err.exp new file mode 100644 index 000000000000..692cebe3c214 --- /dev/null +++ b/tests/functional/lang/eval-fail-all-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'all' builtin + at /pwd/lang/eval-fail-all-3.nix:1:1: + 1| builtins.all (_: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the function passed to builtins.all + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-all-3.nix b/tests/functional/lang/eval-fail-all-3.nix new file mode 100644 index 000000000000..68f6ce6567e5 --- /dev/null +++ b/tests/functional/lang/eval-fail-all-3.nix @@ -0,0 +1 @@ +builtins.all (_: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-any-1.err.exp b/tests/functional/lang/eval-fail-any-1.err.exp new file mode 100644 index 000000000000..2d81dae42547 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'any' builtin + at /pwd/lang/eval-fail-any-1.nix:1:1: + 1| builtins.any 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.any + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-any-1.nix b/tests/functional/lang/eval-fail-any-1.nix new file mode 100644 index 000000000000..aa59791b73fe --- /dev/null +++ b/tests/functional/lang/eval-fail-any-1.nix @@ -0,0 +1 @@ +builtins.any 1 "foo" diff --git a/tests/functional/lang/eval-fail-any-2.err.exp b/tests/functional/lang/eval-fail-any-2.err.exp new file mode 100644 index 000000000000..a03a9e7c34e8 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'any' builtin + at /pwd/lang/eval-fail-any-2.nix:1:1: + 1| builtins.any (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.any + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-any-2.nix b/tests/functional/lang/eval-fail-any-2.nix new file mode 100644 index 000000000000..64a5d3acc801 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-2.nix @@ -0,0 +1 @@ +builtins.any (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-any-3.err.exp b/tests/functional/lang/eval-fail-any-3.err.exp new file mode 100644 index 000000000000..b06d27abdf65 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'any' builtin + at /pwd/lang/eval-fail-any-3.nix:1:1: + 1| builtins.any (_: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the function passed to builtins.any + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-any-3.nix b/tests/functional/lang/eval-fail-any-3.nix new file mode 100644 index 000000000000..906d08252aa8 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-3.nix @@ -0,0 +1 @@ +builtins.any (_: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-attrNames-1.err.exp b/tests/functional/lang/eval-fail-attrNames-1.err.exp new file mode 100644 index 000000000000..46f71a49102d --- /dev/null +++ b/tests/functional/lang/eval-fail-attrNames-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'attrNames' builtin + at /pwd/lang/eval-fail-attrNames-1.nix:1:1: + 1| builtins.attrNames [ ] + | ^ + 2| + + … while evaluating the argument passed to builtins.attrNames + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-attrNames-1.nix b/tests/functional/lang/eval-fail-attrNames-1.nix new file mode 100644 index 000000000000..38f68e4d67b1 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrNames-1.nix @@ -0,0 +1 @@ +builtins.attrNames [ ] diff --git a/tests/functional/lang/eval-fail-attrValues-1.err.exp b/tests/functional/lang/eval-fail-attrValues-1.err.exp new file mode 100644 index 000000000000..c7018784f1b3 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrValues-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'attrValues' builtin + at /pwd/lang/eval-fail-attrValues-1.nix:1:1: + 1| builtins.attrValues [ ] + | ^ + 2| + + … while evaluating the argument passed to builtins.attrValues + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-attrValues-1.nix b/tests/functional/lang/eval-fail-attrValues-1.nix new file mode 100644 index 000000000000..c3154854c78f --- /dev/null +++ b/tests/functional/lang/eval-fail-attrValues-1.nix @@ -0,0 +1 @@ +builtins.attrValues [ ] diff --git a/tests/functional/lang/eval-fail-baseNameOf-1.err.exp b/tests/functional/lang/eval-fail-baseNameOf-1.err.exp new file mode 100644 index 000000000000..4116a1a06391 --- /dev/null +++ b/tests/functional/lang/eval-fail-baseNameOf-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'baseNameOf' builtin + at /pwd/lang/eval-fail-baseNameOf-1.nix:1:1: + 1| builtins.baseNameOf [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.baseNameOf + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-baseNameOf-1.nix b/tests/functional/lang/eval-fail-baseNameOf-1.nix new file mode 100644 index 000000000000..3099c87f5714 --- /dev/null +++ b/tests/functional/lang/eval-fail-baseNameOf-1.nix @@ -0,0 +1 @@ +builtins.baseNameOf [ ] diff --git a/tests/functional/lang/eval-fail-bitAnd-1.err.exp b/tests/functional/lang/eval-fail-bitAnd-1.err.exp new file mode 100644 index 000000000000..04ae5c85400b --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitAnd' builtin + at /pwd/lang/eval-fail-bitAnd-1.nix:1:1: + 1| builtins.bitAnd 1.1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.bitAnd + + error: expected an integer but found a float: 1.1 diff --git a/tests/functional/lang/eval-fail-bitAnd-1.nix b/tests/functional/lang/eval-fail-bitAnd-1.nix new file mode 100644 index 000000000000..24c5d60bdd02 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-1.nix @@ -0,0 +1 @@ +builtins.bitAnd 1.1 2 diff --git a/tests/functional/lang/eval-fail-bitAnd-2.err.exp b/tests/functional/lang/eval-fail-bitAnd-2.err.exp new file mode 100644 index 000000000000..f845462f1262 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitAnd' builtin + at /pwd/lang/eval-fail-bitAnd-2.nix:1:1: + 1| builtins.bitAnd 1 2.2 + | ^ + 2| + + … while evaluating the second argument passed to builtins.bitAnd + + error: expected an integer but found a float: 2.2 diff --git a/tests/functional/lang/eval-fail-bitAnd-2.nix b/tests/functional/lang/eval-fail-bitAnd-2.nix new file mode 100644 index 000000000000..12ea1451f582 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-2.nix @@ -0,0 +1 @@ +builtins.bitAnd 1 2.2 diff --git a/tests/functional/lang/eval-fail-bitOr-1.err.exp b/tests/functional/lang/eval-fail-bitOr-1.err.exp new file mode 100644 index 000000000000..38b498da4697 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitOr' builtin + at /pwd/lang/eval-fail-bitOr-1.nix:1:1: + 1| builtins.bitOr 1.1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.bitOr + + error: expected an integer but found a float: 1.1 diff --git a/tests/functional/lang/eval-fail-bitOr-1.nix b/tests/functional/lang/eval-fail-bitOr-1.nix new file mode 100644 index 000000000000..2eab4871ca65 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-1.nix @@ -0,0 +1 @@ +builtins.bitOr 1.1 2 diff --git a/tests/functional/lang/eval-fail-bitOr-2.err.exp b/tests/functional/lang/eval-fail-bitOr-2.err.exp new file mode 100644 index 000000000000..a1ca655add7a --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitOr' builtin + at /pwd/lang/eval-fail-bitOr-2.nix:1:1: + 1| builtins.bitOr 1 2.2 + | ^ + 2| + + … while evaluating the second argument passed to builtins.bitOr + + error: expected an integer but found a float: 2.2 diff --git a/tests/functional/lang/eval-fail-bitOr-2.nix b/tests/functional/lang/eval-fail-bitOr-2.nix new file mode 100644 index 000000000000..f66e3a3635ab --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-2.nix @@ -0,0 +1 @@ +builtins.bitOr 1 2.2 diff --git a/tests/functional/lang/eval-fail-bitXor-1.err.exp b/tests/functional/lang/eval-fail-bitXor-1.err.exp new file mode 100644 index 000000000000..f051ef622069 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitXor' builtin + at /pwd/lang/eval-fail-bitXor-1.nix:1:1: + 1| builtins.bitXor 1.1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.bitXor + + error: expected an integer but found a float: 1.1 diff --git a/tests/functional/lang/eval-fail-bitXor-1.nix b/tests/functional/lang/eval-fail-bitXor-1.nix new file mode 100644 index 000000000000..c1b0f1b4e82d --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-1.nix @@ -0,0 +1 @@ +builtins.bitXor 1.1 2 diff --git a/tests/functional/lang/eval-fail-bitXor-2.err.exp b/tests/functional/lang/eval-fail-bitXor-2.err.exp new file mode 100644 index 000000000000..3b5ab1317ab0 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitXor' builtin + at /pwd/lang/eval-fail-bitXor-2.nix:1:1: + 1| builtins.bitXor 1 2.2 + | ^ + 2| + + … while evaluating the second argument passed to builtins.bitXor + + error: expected an integer but found a float: 2.2 diff --git a/tests/functional/lang/eval-fail-bitXor-2.nix b/tests/functional/lang/eval-fail-bitXor-2.nix new file mode 100644 index 000000000000..b9f7ad9c2890 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-2.nix @@ -0,0 +1 @@ +builtins.bitXor 1 2.2 diff --git a/tests/functional/lang/eval-fail-catAttrs-1.err.exp b/tests/functional/lang/eval-fail-catAttrs-1.err.exp new file mode 100644 index 000000000000..e669dc39a6e8 --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-1.nix:1:1: + 1| builtins.catAttrs [ ] { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.catAttrs + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-catAttrs-1.nix b/tests/functional/lang/eval-fail-catAttrs-1.nix new file mode 100644 index 000000000000..97d06f5b63be --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-1.nix @@ -0,0 +1 @@ +builtins.catAttrs [ ] { } diff --git a/tests/functional/lang/eval-fail-catAttrs-2.err.exp b/tests/functional/lang/eval-fail-catAttrs-2.err.exp new file mode 100644 index 000000000000..2f6c1425d6a6 --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-2.nix:1:1: + 1| builtins.catAttrs "foo" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.catAttrs + + error: expected a list but found a set: { } diff --git a/tests/functional/lang/eval-fail-catAttrs-2.nix b/tests/functional/lang/eval-fail-catAttrs-2.nix new file mode 100644 index 000000000000..caf616b20b84 --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-2.nix @@ -0,0 +1 @@ +builtins.catAttrs "foo" { } diff --git a/tests/functional/lang/eval-fail-catAttrs-3.err.exp b/tests/functional/lang/eval-fail-catAttrs-3.err.exp new file mode 100644 index 000000000000..2643d37f9e96 --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-3.nix:1:1: + 1| builtins.catAttrs "foo" [ 1 ] + | ^ + 2| + + … while evaluating an element in the list passed as second argument to builtins.catAttrs + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-catAttrs-3.nix b/tests/functional/lang/eval-fail-catAttrs-3.nix new file mode 100644 index 000000000000..82595b94ed0f --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-3.nix @@ -0,0 +1 @@ +builtins.catAttrs "foo" [ 1 ] diff --git a/tests/functional/lang/eval-fail-catAttrs-4.err.exp b/tests/functional/lang/eval-fail-catAttrs-4.err.exp new file mode 100644 index 000000000000..ffd5fcec7c0b --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-4.nix:1:1: + 1| builtins.catAttrs "foo" [ + | ^ + 2| { foo = 1; } + + … while evaluating an element in the list passed as second argument to builtins.catAttrs + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-catAttrs-4.nix b/tests/functional/lang/eval-fail-catAttrs-4.nix new file mode 100644 index 000000000000..1f294a693c6a --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-4.nix @@ -0,0 +1,5 @@ +builtins.catAttrs "foo" [ + { foo = 1; } + 1 + { bar = 5; } +] diff --git a/tests/functional/lang/eval-fail-ceil-1.err.exp b/tests/functional/lang/eval-fail-ceil-1.err.exp new file mode 100644 index 000000000000..96d8b1df7a8e --- /dev/null +++ b/tests/functional/lang/eval-fail-ceil-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'ceil' builtin + at /pwd/lang/eval-fail-ceil-1.nix:1:1: + 1| builtins.ceil "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.ceil + + error: expected a float but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-ceil-1.nix b/tests/functional/lang/eval-fail-ceil-1.nix new file mode 100644 index 000000000000..0ea67754c358 --- /dev/null +++ b/tests/functional/lang/eval-fail-ceil-1.nix @@ -0,0 +1 @@ +builtins.ceil "foo" diff --git a/tests/functional/lang/eval-fail-compareVersions-1.err.exp b/tests/functional/lang/eval-fail-compareVersions-1.err.exp new file mode 100644 index 000000000000..8632cf237e1b --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'compareVersions' builtin + at /pwd/lang/eval-fail-compareVersions-1.nix:1:1: + 1| builtins.compareVersions 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.compareVersions + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-compareVersions-1.nix b/tests/functional/lang/eval-fail-compareVersions-1.nix new file mode 100644 index 000000000000..f408381cf4c5 --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-1.nix @@ -0,0 +1 @@ +builtins.compareVersions 1 { } diff --git a/tests/functional/lang/eval-fail-compareVersions-2.err.exp b/tests/functional/lang/eval-fail-compareVersions-2.err.exp new file mode 100644 index 000000000000..bcd246460f2f --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'compareVersions' builtin + at /pwd/lang/eval-fail-compareVersions-2.nix:1:1: + 1| builtins.compareVersions "abd" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.compareVersions + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-compareVersions-2.nix b/tests/functional/lang/eval-fail-compareVersions-2.nix new file mode 100644 index 000000000000..49c9f0a8874e --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-2.nix @@ -0,0 +1 @@ +builtins.compareVersions "abd" { } diff --git a/tests/functional/lang/eval-fail-concatLists-1.err.exp b/tests/functional/lang/eval-fail-concatLists-1.err.exp new file mode 100644 index 000000000000..15193daae1ca --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatLists' builtin + at /pwd/lang/eval-fail-concatLists-1.nix:1:1: + 1| builtins.concatLists 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.concatLists + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatLists-1.nix b/tests/functional/lang/eval-fail-concatLists-1.nix new file mode 100644 index 000000000000..97e171cb6fa4 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-1.nix @@ -0,0 +1 @@ +builtins.concatLists 1 diff --git a/tests/functional/lang/eval-fail-concatLists-2.err.exp b/tests/functional/lang/eval-fail-concatLists-2.err.exp new file mode 100644 index 000000000000..35fbc09bfe35 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatLists' builtin + at /pwd/lang/eval-fail-concatLists-2.nix:1:1: + 1| builtins.concatLists [ 1 ] + | ^ + 2| + + … while evaluating a value of the list passed to builtins.concatLists + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatLists-2.nix b/tests/functional/lang/eval-fail-concatLists-2.nix new file mode 100644 index 000000000000..0ad98a8eb4ac --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-2.nix @@ -0,0 +1 @@ +builtins.concatLists [ 1 ] diff --git a/tests/functional/lang/eval-fail-concatLists-3.err.exp b/tests/functional/lang/eval-fail-concatLists-3.err.exp new file mode 100644 index 000000000000..d762eb55bdaa --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatLists' builtin + at /pwd/lang/eval-fail-concatLists-3.nix:1:1: + 1| builtins.concatLists [ + | ^ + 2| [ 1 ] + + … while evaluating a value of the list passed to builtins.concatLists + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-concatLists-3.nix b/tests/functional/lang/eval-fail-concatLists-3.nix new file mode 100644 index 000000000000..2366f314e1b7 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-3.nix @@ -0,0 +1,4 @@ +builtins.concatLists [ + [ 1 ] + "foo" +] diff --git a/tests/functional/lang/eval-fail-concatMap-1.err.exp b/tests/functional/lang/eval-fail-concatMap-1.err.exp new file mode 100644 index 000000000000..e43c160a38e2 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-1.nix:1:1: + 1| builtins.concatMap 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.concatMap + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatMap-1.nix b/tests/functional/lang/eval-fail-concatMap-1.nix new file mode 100644 index 000000000000..68c9c3560f6a --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-1.nix @@ -0,0 +1 @@ +builtins.concatMap 1 "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-2.err.exp b/tests/functional/lang/eval-fail-concatMap-2.err.exp new file mode 100644 index 000000000000..52c51fd0157b --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-2.nix:1:1: + 1| builtins.concatMap (x: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.concatMap + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-2.nix b/tests/functional/lang/eval-fail-concatMap-2.nix new file mode 100644 index 000000000000..b98860470caa --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-2.nix @@ -0,0 +1 @@ +builtins.concatMap (x: 1) "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-3.err.exp b/tests/functional/lang/eval-fail-concatMap-3.err.exp new file mode 100644 index 000000000000..21d13118f0cd --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-3.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-3.nix:1:1: + 1| builtins.concatMap (x: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the function passed to builtins.concatMap + at /pwd/lang/eval-fail-concatMap-3.nix:1:21: + 1| builtins.concatMap (x: 1) [ "foo" ] + | ^ + 2| + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatMap-3.nix b/tests/functional/lang/eval-fail-concatMap-3.nix new file mode 100644 index 000000000000..f4a519060358 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-3.nix @@ -0,0 +1 @@ +builtins.concatMap (x: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-concatMap-4.err.exp b/tests/functional/lang/eval-fail-concatMap-4.err.exp new file mode 100644 index 000000000000..1b8e865fd3a1 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-4.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-4.nix:1:1: + 1| builtins.concatMap (x: "foo") [ + | ^ + 2| 1 + + … while evaluating the return value of the function passed to builtins.concatMap + at /pwd/lang/eval-fail-concatMap-4.nix:1:21: + 1| builtins.concatMap (x: "foo") [ + | ^ + 2| 1 + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-4.nix b/tests/functional/lang/eval-fail-concatMap-4.nix new file mode 100644 index 000000000000..0f678005bced --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-4.nix @@ -0,0 +1,4 @@ +builtins.concatMap (x: "foo") [ + 1 + 2 +] diff --git a/tests/functional/lang/eval-fail-concatStringsSep-1.err.exp b/tests/functional/lang/eval-fail-concatStringsSep-1.err.exp new file mode 100644 index 000000000000..4b40db46d681 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatStringsSep' builtin + at /pwd/lang/eval-fail-concatStringsSep-1.nix:1:1: + 1| builtins.concatStringsSep 1 { } + | ^ + 2| + + … while evaluating the first argument (the separator string) passed to builtins.concatStringsSep + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatStringsSep-1.nix b/tests/functional/lang/eval-fail-concatStringsSep-1.nix new file mode 100644 index 000000000000..0c10cea34630 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-1.nix @@ -0,0 +1 @@ +builtins.concatStringsSep 1 { } diff --git a/tests/functional/lang/eval-fail-concatStringsSep-2.err.exp b/tests/functional/lang/eval-fail-concatStringsSep-2.err.exp new file mode 100644 index 000000000000..9cc806f4955f --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatStringsSep' builtin + at /pwd/lang/eval-fail-concatStringsSep-2.nix:1:1: + 1| builtins.concatStringsSep "foo" { } + | ^ + 2| + + … while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep + + error: expected a list but found a set: { } diff --git a/tests/functional/lang/eval-fail-concatStringsSep-2.nix b/tests/functional/lang/eval-fail-concatStringsSep-2.nix new file mode 100644 index 000000000000..ae6a5f1984d4 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-2.nix @@ -0,0 +1 @@ +builtins.concatStringsSep "foo" { } diff --git a/tests/functional/lang/eval-fail-concatStringsSep-3.err.exp b/tests/functional/lang/eval-fail-concatStringsSep-3.err.exp new file mode 100644 index 000000000000..5c098d471e72 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatStringsSep' builtin + at /pwd/lang/eval-fail-concatStringsSep-3.nix:1:1: + 1| builtins.concatStringsSep "foo" [ + | ^ + 2| 1 + + … while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep + + error: cannot coerce an integer to a string: 1 diff --git a/tests/functional/lang/eval-fail-concatStringsSep-3.nix b/tests/functional/lang/eval-fail-concatStringsSep-3.nix new file mode 100644 index 000000000000..f51ae1c7ddc5 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-3.nix @@ -0,0 +1,5 @@ +builtins.concatStringsSep "foo" [ + 1 + 2 + { } +] diff --git a/tests/functional/lang/eval-fail-derivationStrict-1.err.exp b/tests/functional/lang/eval-fail-derivationStrict-1.err.exp new file mode 100644 index 000000000000..468c789e8db4 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-1.nix:1:1: + 1| builtins.derivationStrict "" + | ^ + 2| + + … while evaluating the argument passed to builtins.derivationStrict + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-derivationStrict-1.nix b/tests/functional/lang/eval-fail-derivationStrict-1.nix new file mode 100644 index 000000000000..948230469d98 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-1.nix @@ -0,0 +1 @@ +builtins.derivationStrict "" diff --git a/tests/functional/lang/eval-fail-derivationStrict-10.err.exp b/tests/functional/lang/eval-fail-derivationStrict-10.err.exp new file mode 100644 index 000000000000..21eca42fad06 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-10.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-10.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-10.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-10.nix:5:3: + 4| system = 1; + 5| outputs = { }; + | ^ + 6| } + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-10.nix b/tests/functional/lang/eval-fail-derivationStrict-10.nix new file mode 100644 index 000000000000..410f48a46537 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-10.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = { }; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-11.err.exp b/tests/functional/lang/eval-fail-derivationStrict-11.err.exp new file mode 100644 index 000000000000..b0fe664e5202 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-11.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-11.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-11.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-11.nix:5:3: + 4| system = 1; + 5| outputs = "drvPath"; + | ^ + 6| } + + error: invalid derivation output name 'drvPath' diff --git a/tests/functional/lang/eval-fail-derivationStrict-11.nix b/tests/functional/lang/eval-fail-derivationStrict-11.nix new file mode 100644 index 000000000000..60947b6e2ee6 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-11.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "drvPath"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-12.err.exp b/tests/functional/lang/eval-fail-derivationStrict-12.err.exp new file mode 100644 index 000000000000..adfc0064bfb7 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-12.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-12.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-12.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-12.nix:3:3: + 2| name = "foo"; + 3| outputs = "out"; + | ^ + 4| __structuredAttrs = true; + + error: expected a list but found a string: "out" diff --git a/tests/functional/lang/eval-fail-derivationStrict-12.nix b/tests/functional/lang/eval-fail-derivationStrict-12.nix new file mode 100644 index 000000000000..516b8ba39ace --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-12.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + outputs = "out"; + __structuredAttrs = true; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-13.err.exp b/tests/functional/lang/eval-fail-derivationStrict-13.err.exp new file mode 100644 index 000000000000..8a313333d58f --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-13.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-13.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-13.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-13.nix:5:3: + 4| system = 1; + 5| outputs = [ ]; + | ^ + 6| } + + error: derivation cannot have an empty set of outputs diff --git a/tests/functional/lang/eval-fail-derivationStrict-13.nix b/tests/functional/lang/eval-fail-derivationStrict-13.nix new file mode 100644 index 000000000000..9a104d2253fd --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-13.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = [ ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-14.err.exp b/tests/functional/lang/eval-fail-derivationStrict-14.err.exp new file mode 100644 index 000000000000..fb0c5423c4dc --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-14.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-14.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-14.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-14.nix:5:3: + 4| system = 1; + 5| outputs = [ "drvPath" ]; + | ^ + 6| } + + error: invalid derivation output name 'drvPath' diff --git a/tests/functional/lang/eval-fail-derivationStrict-14.nix b/tests/functional/lang/eval-fail-derivationStrict-14.nix new file mode 100644 index 000000000000..4d286d34c725 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-14.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = [ "drvPath" ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-15.err.exp b/tests/functional/lang/eval-fail-derivationStrict-15.err.exp new file mode 100644 index 000000000000..8df047069136 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-15.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-15.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-15.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-15.nix:5:3: + 4| system = 1; + 5| outputs = [ + | ^ + 6| "out" + + error: duplicate derivation output 'out' diff --git a/tests/functional/lang/eval-fail-derivationStrict-15.nix b/tests/functional/lang/eval-fail-derivationStrict-15.nix new file mode 100644 index 000000000000..96e5a9de404b --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-15.nix @@ -0,0 +1,9 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = [ + "out" + "out" + ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-16.err.exp b/tests/functional/lang/eval-fail-derivationStrict-16.err.exp new file mode 100644 index 000000000000..c969aa30d0d3 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-16.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-16.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-16.nix:2:3 + + … while evaluating attribute '__contentAddressed' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-16.nix:6:3: + 5| outputs = "out"; + 6| __contentAddressed = "true"; + | ^ + 7| } + + error: expected a Boolean but found a string: "true" diff --git a/tests/functional/lang/eval-fail-derivationStrict-16.nix b/tests/functional/lang/eval-fail-derivationStrict-16.nix new file mode 100644 index 000000000000..554ad09df1a2 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-16.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + __contentAddressed = "true"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-17.err.exp b/tests/functional/lang/eval-fail-derivationStrict-17.err.exp new file mode 100644 index 000000000000..ed7558c4cbab --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-17.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-17.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-17.nix:2:3 + + … while evaluating attribute '__impure' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-17.nix:6:3: + 5| outputs = "out"; + 6| __impure = "true"; + | ^ + 7| } + + error: expected a Boolean but found a string: "true" diff --git a/tests/functional/lang/eval-fail-derivationStrict-17.nix b/tests/functional/lang/eval-fail-derivationStrict-17.nix new file mode 100644 index 000000000000..4a357ba12d7f --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-17.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + __impure = "true"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-19.err.exp b/tests/functional/lang/eval-fail-derivationStrict-19.err.exp new file mode 100644 index 000000000000..4861b1cd3abd --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-19.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-19.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-19.nix:2:3 + + … while evaluating attribute 'args' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-19.nix:6:3: + 5| outputs = "out"; + 6| args = "foo"; + | ^ + 7| } + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-derivationStrict-19.nix b/tests/functional/lang/eval-fail-derivationStrict-19.nix new file mode 100644 index 000000000000..912be12c5708 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-19.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + args = "foo"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-2.err.exp b/tests/functional/lang/eval-fail-derivationStrict-2.err.exp new file mode 100644 index 000000000000..7ba9fc825595 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-2.nix:1:1: + 1| builtins.derivationStrict { } + | ^ + 2| + + … in the attrset passed as argument to builtins.derivationStrict + + error: attribute 'name' missing diff --git a/tests/functional/lang/eval-fail-derivationStrict-2.nix b/tests/functional/lang/eval-fail-derivationStrict-2.nix new file mode 100644 index 000000000000..70a07b36d2d5 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-2.nix @@ -0,0 +1 @@ +builtins.derivationStrict { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-20.err.exp b/tests/functional/lang/eval-fail-derivationStrict-20.err.exp new file mode 100644 index 000000000000..df1805bef681 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-20.err.exp @@ -0,0 +1,20 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-20.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-20.nix:2:3 + + … while evaluating attribute 'args' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-20.nix:6:3: + 5| outputs = "out"; + 6| args = [ { } ]; + | ^ + 7| } + + … while evaluating an element of the argument list + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-20.nix b/tests/functional/lang/eval-fail-derivationStrict-20.nix new file mode 100644 index 000000000000..17bc3a26c906 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-20.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + args = [ { } ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-21.err.exp b/tests/functional/lang/eval-fail-derivationStrict-21.err.exp new file mode 100644 index 000000000000..0a6384a77957 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-21.err.exp @@ -0,0 +1,20 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-21.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-21.nix:2:3 + + … while evaluating attribute 'args' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-21.nix:6:3: + 5| outputs = "out"; + 6| args = [ + | ^ + 7| "a" + + … while evaluating an element of the argument list + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-21.nix b/tests/functional/lang/eval-fail-derivationStrict-21.nix new file mode 100644 index 000000000000..7846874194c3 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-21.nix @@ -0,0 +1,10 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + args = [ + "a" + { } + ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-22.err.exp b/tests/functional/lang/eval-fail-derivationStrict-22.err.exp new file mode 100644 index 000000000000..91a44ce1e425 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-22.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-22.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-22.nix:2:3 + + … while evaluating attribute 'FOO' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-22.nix:6:3: + 5| outputs = "out"; + 6| FOO = { }; + | ^ + 7| } + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-22.nix b/tests/functional/lang/eval-fail-derivationStrict-22.nix new file mode 100644 index 000000000000..7b3b7c824855 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-22.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + FOO = { }; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-3.err.exp b/tests/functional/lang/eval-fail-derivationStrict-3.err.exp new file mode 100644 index 000000000000..f8c301a97793 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-3.err.exp @@ -0,0 +1,16 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-3.nix:1:1: + 1| builtins.derivationStrict { name = 1; } + | ^ + 2| + + … while evaluating the derivation attribute 'name' + at /pwd/lang/eval-fail-derivationStrict-3.nix:1:29: + 1| builtins.derivationStrict { name = 1; } + | ^ + 2| + + … while evaluating the `name` attribute passed to builtins.derivationStrict + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-derivationStrict-3.nix b/tests/functional/lang/eval-fail-derivationStrict-3.nix new file mode 100644 index 000000000000..a18edebd8701 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-3.nix @@ -0,0 +1 @@ +builtins.derivationStrict { name = 1; } diff --git a/tests/functional/lang/eval-fail-derivationStrict-4.err.exp b/tests/functional/lang/eval-fail-derivationStrict-4.err.exp new file mode 100644 index 000000000000..b598f0f11ccd --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-4.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-4.nix:1:1: + 1| builtins.derivationStrict { name = "foo"; } + | ^ + 2| + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-4.nix:1:29 + + error: required attribute 'builder' missing diff --git a/tests/functional/lang/eval-fail-derivationStrict-4.nix b/tests/functional/lang/eval-fail-derivationStrict-4.nix new file mode 100644 index 000000000000..f13692bc5e10 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-4.nix @@ -0,0 +1 @@ +builtins.derivationStrict { name = "foo"; } diff --git a/tests/functional/lang/eval-fail-derivationStrict-5.err.exp b/tests/functional/lang/eval-fail-derivationStrict-5.err.exp new file mode 100644 index 000000000000..930fda3a95c9 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-5.err.exp @@ -0,0 +1,13 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-5.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-5.nix:2:3 + + … while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict + + error: expected a Boolean but found an integer: 15 diff --git a/tests/functional/lang/eval-fail-derivationStrict-5.nix b/tests/functional/lang/eval-fail-derivationStrict-5.nix new file mode 100644 index 000000000000..e74a8caf45c4 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-5.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + __structuredAttrs = 15; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-6.err.exp b/tests/functional/lang/eval-fail-derivationStrict-6.err.exp new file mode 100644 index 000000000000..323db3b37949 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-6.err.exp @@ -0,0 +1,13 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-6.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-6.nix:2:3 + + … while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict + + error: expected a Boolean but found an integer: 15 diff --git a/tests/functional/lang/eval-fail-derivationStrict-6.nix b/tests/functional/lang/eval-fail-derivationStrict-6.nix new file mode 100644 index 000000000000..feda92b00d13 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-6.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + __ignoreNulls = 15; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-7.err.exp b/tests/functional/lang/eval-fail-derivationStrict-7.err.exp new file mode 100644 index 000000000000..78ddd1573982 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-7.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-7.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-7.nix:2:3 + + … while evaluating attribute 'outputHashMode' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-7.nix:4:3: + 3| builder = 1; + 4| outputHashMode = 15; + | ^ + 5| } + + error: invalid value '15' for 'outputHashMode' attribute diff --git a/tests/functional/lang/eval-fail-derivationStrict-7.nix b/tests/functional/lang/eval-fail-derivationStrict-7.nix new file mode 100644 index 000000000000..1ffdb6fdbe99 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-7.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + outputHashMode = 15; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-8.err.exp b/tests/functional/lang/eval-fail-derivationStrict-8.err.exp new file mode 100644 index 000000000000..ba4e0109c62d --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-8.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-8.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-8.nix:2:3 + + … while evaluating attribute 'outputHashMode' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-8.nix:4:3: + 3| builder = 1; + 4| outputHashMode = "custom"; + | ^ + 5| } + + error: invalid value 'custom' for 'outputHashMode' attribute diff --git a/tests/functional/lang/eval-fail-derivationStrict-8.nix b/tests/functional/lang/eval-fail-derivationStrict-8.nix new file mode 100644 index 000000000000..38243416fdda --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-8.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + outputHashMode = "custom"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-9.err.exp b/tests/functional/lang/eval-fail-derivationStrict-9.err.exp new file mode 100644 index 000000000000..6bfbbc65b867 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-9.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-9.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-9.nix:2:3 + + … while evaluating attribute 'system' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-9.nix:4:3: + 3| builder = 1; + 4| system = { }; + | ^ + 5| } + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-9.nix b/tests/functional/lang/eval-fail-derivationStrict-9.nix new file mode 100644 index 000000000000..a3a567a14ce3 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-9.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = { }; +} diff --git a/tests/functional/lang/eval-fail-div-1.err.exp b/tests/functional/lang/eval-fail-div-1.err.exp new file mode 100644 index 000000000000..286e1f79c3c5 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'div' builtin + at /pwd/lang/eval-fail-div-1.nix:1:1: + 1| builtins.div "foo" 1 + | ^ + 2| + + … while evaluating the first operand of the division + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-div-1.nix b/tests/functional/lang/eval-fail-div-1.nix new file mode 100644 index 000000000000..c6edce369753 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-1.nix @@ -0,0 +1 @@ +builtins.div "foo" 1 diff --git a/tests/functional/lang/eval-fail-div-2.err.exp b/tests/functional/lang/eval-fail-div-2.err.exp new file mode 100644 index 000000000000..20e6e710c096 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'div' builtin + at /pwd/lang/eval-fail-div-2.nix:1:1: + 1| builtins.div 1 "foo" + | ^ + 2| + + … while evaluating the second operand of the division + + error: expected a float but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-div-2.nix b/tests/functional/lang/eval-fail-div-2.nix new file mode 100644 index 000000000000..71d9947719eb --- /dev/null +++ b/tests/functional/lang/eval-fail-div-2.nix @@ -0,0 +1 @@ +builtins.div 1 "foo" diff --git a/tests/functional/lang/eval-fail-div-3.err.exp b/tests/functional/lang/eval-fail-div-3.err.exp new file mode 100644 index 000000000000..6d263c956ae4 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'div' builtin + at /pwd/lang/eval-fail-div-3.nix:1:1: + 1| builtins.div "foo" 0 + | ^ + 2| + + error: division by zero diff --git a/tests/functional/lang/eval-fail-div-3.nix b/tests/functional/lang/eval-fail-div-3.nix new file mode 100644 index 000000000000..11c606baac7c --- /dev/null +++ b/tests/functional/lang/eval-fail-div-3.nix @@ -0,0 +1 @@ +builtins.div "foo" 0 diff --git a/tests/functional/lang/eval-fail-elem-1.err.exp b/tests/functional/lang/eval-fail-elem-1.err.exp new file mode 100644 index 000000000000..cd0a2281414a --- /dev/null +++ b/tests/functional/lang/eval-fail-elem-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'elem' builtin + at /pwd/lang/eval-fail-elem-1.nix:1:1: + 1| builtins.elem 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.elem + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-elem-1.nix b/tests/functional/lang/eval-fail-elem-1.nix new file mode 100644 index 000000000000..eecc965cb438 --- /dev/null +++ b/tests/functional/lang/eval-fail-elem-1.nix @@ -0,0 +1 @@ +builtins.elem 1 "foo" diff --git a/tests/functional/lang/eval-fail-elemAt-1.err.exp b/tests/functional/lang/eval-fail-elemAt-1.err.exp new file mode 100644 index 000000000000..3ea5b0aef0b5 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'elemAt' builtin + at /pwd/lang/eval-fail-elemAt-1.nix:1:1: + 1| builtins.elemAt "foo" (-1) + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.elemAt' + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-elemAt-1.nix b/tests/functional/lang/eval-fail-elemAt-1.nix new file mode 100644 index 000000000000..27de601ebac2 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-1.nix @@ -0,0 +1 @@ +builtins.elemAt "foo" (-1) diff --git a/tests/functional/lang/eval-fail-elemAt-2.err.exp b/tests/functional/lang/eval-fail-elemAt-2.err.exp new file mode 100644 index 000000000000..6d746c7739e5 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'elemAt' builtin + at /pwd/lang/eval-fail-elemAt-2.nix:1:1: + 1| builtins.elemAt [ ] (-1) + | ^ + 2| + + error: 'builtins.elemAt' called with index -1 on a list of size 0 diff --git a/tests/functional/lang/eval-fail-elemAt-2.nix b/tests/functional/lang/eval-fail-elemAt-2.nix new file mode 100644 index 000000000000..cf48f089f2fe --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-2.nix @@ -0,0 +1 @@ +builtins.elemAt [ ] (-1) diff --git a/tests/functional/lang/eval-fail-elemAt-3.err.exp b/tests/functional/lang/eval-fail-elemAt-3.err.exp new file mode 100644 index 000000000000..1f34afb45871 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'elemAt' builtin + at /pwd/lang/eval-fail-elemAt-3.nix:1:1: + 1| builtins.elemAt [ "foo" ] 3 + | ^ + 2| + + error: 'builtins.elemAt' called with index 3 on a list of size 1 diff --git a/tests/functional/lang/eval-fail-elemAt-3.nix b/tests/functional/lang/eval-fail-elemAt-3.nix new file mode 100644 index 000000000000..8493015def6e --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-3.nix @@ -0,0 +1 @@ +builtins.elemAt [ "foo" ] 3 diff --git a/tests/functional/lang/eval-fail-filter-1.err.exp b/tests/functional/lang/eval-fail-filter-1.err.exp new file mode 100644 index 000000000000..d3c55d9f651e --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filter' builtin + at /pwd/lang/eval-fail-filter-1.nix:1:1: + 1| builtins.filter 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.filter + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-filter-1.nix b/tests/functional/lang/eval-fail-filter-1.nix new file mode 100644 index 000000000000..37b379f87d50 --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-1.nix @@ -0,0 +1 @@ +builtins.filter 1 "foo" diff --git a/tests/functional/lang/eval-fail-filter-2.err.exp b/tests/functional/lang/eval-fail-filter-2.err.exp new file mode 100644 index 000000000000..5e5d0ecc5a79 --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filter' builtin + at /pwd/lang/eval-fail-filter-2.nix:1:1: + 1| builtins.filter 1 [ "foo" ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.filter + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-filter-2.nix b/tests/functional/lang/eval-fail-filter-2.nix new file mode 100644 index 000000000000..177adfa82c1d --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-2.nix @@ -0,0 +1 @@ +builtins.filter 1 [ "foo" ] diff --git a/tests/functional/lang/eval-fail-filter-3.err.exp b/tests/functional/lang/eval-fail-filter-3.err.exp new file mode 100644 index 000000000000..ba5a87e27567 --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filter' builtin + at /pwd/lang/eval-fail-filter-3.nix:1:1: + 1| builtins.filter (_: 5) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the filtering function passed to builtins.filter + + error: expected a Boolean but found an integer: 5 diff --git a/tests/functional/lang/eval-fail-filter-3.nix b/tests/functional/lang/eval-fail-filter-3.nix new file mode 100644 index 000000000000..c262f25bf6c5 --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-3.nix @@ -0,0 +1 @@ +builtins.filter (_: 5) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-filterSource-1.err.exp b/tests/functional/lang/eval-fail-filterSource-1.err.exp new file mode 100644 index 000000000000..cf2d153f0deb --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-1.nix:1:1: + 1| builtins.filterSource [ ] [ ] + | ^ + 2| + + … while evaluating the second argument (the path to filter) passed to 'builtins.filterSource' + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-filterSource-1.nix b/tests/functional/lang/eval-fail-filterSource-1.nix new file mode 100644 index 000000000000..5833e0de595e --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-1.nix @@ -0,0 +1 @@ +builtins.filterSource [ ] [ ] diff --git a/tests/functional/lang/eval-fail-filterSource-2.err.exp b/tests/functional/lang/eval-fail-filterSource-2.err.exp new file mode 100644 index 000000000000..e3302f032f9d --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-2.nix:1:1: + 1| builtins.filterSource [ ] "foo" + | ^ + 2| + + … while evaluating the second argument (the path to filter) passed to 'builtins.filterSource' + + error: string 'foo' doesn't represent an absolute path diff --git a/tests/functional/lang/eval-fail-filterSource-2.nix b/tests/functional/lang/eval-fail-filterSource-2.nix new file mode 100644 index 000000000000..5fc0b68cb0d5 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-2.nix @@ -0,0 +1 @@ +builtins.filterSource [ ] "foo" diff --git a/tests/functional/lang/eval-fail-filterSource-3.err.exp b/tests/functional/lang/eval-fail-filterSource-3.err.exp new file mode 100644 index 000000000000..5750f8198c63 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-3.nix:1:1: + 1| builtins.filterSource [ ] ./. + | ^ + 2| + + … while evaluating the first argument passed to builtins.filterSource + + error: expected a function but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-filterSource-3.nix b/tests/functional/lang/eval-fail-filterSource-3.nix new file mode 100644 index 000000000000..6c4124aadb38 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-3.nix @@ -0,0 +1 @@ +builtins.filterSource [ ] ./. diff --git a/tests/functional/lang/eval-fail-filterSource-4.err.exp b/tests/functional/lang/eval-fail-filterSource-4.err.exp new file mode 100644 index 000000000000..4318616ee8fb --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-4.nix:1:1: + 1| builtins.filterSource (_: 1) ./. + | ^ + 2| + + … while adding path '/pwd/lang' + + error: attempt to call something which is not a function but an integer: 1 diff --git a/tests/functional/lang/eval-fail-filterSource-4.nix b/tests/functional/lang/eval-fail-filterSource-4.nix new file mode 100644 index 000000000000..28a729c614b6 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-4.nix @@ -0,0 +1 @@ +builtins.filterSource (_: 1) ./. diff --git a/tests/functional/lang/eval-fail-filterSource-5.err.exp b/tests/functional/lang/eval-fail-filterSource-5.err.exp new file mode 100644 index 000000000000..fe017b2efb8d --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-5.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-5.nix:1:1: + 1| builtins.filterSource (_: _: 1) ./. + | ^ + 2| + + … while adding path '/pwd/lang' + + … while evaluating the return value of the path filter function + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-filterSource-5.nix b/tests/functional/lang/eval-fail-filterSource-5.nix new file mode 100644 index 000000000000..dbb0decee0ee --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-5.nix @@ -0,0 +1 @@ +builtins.filterSource (_: _: 1) ./. diff --git a/tests/functional/lang/eval-fail-floor-1.err.exp b/tests/functional/lang/eval-fail-floor-1.err.exp new file mode 100644 index 000000000000..2e5fb0aae3db --- /dev/null +++ b/tests/functional/lang/eval-fail-floor-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'floor' builtin + at /pwd/lang/eval-fail-floor-1.nix:1:1: + 1| builtins.floor "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.floor + + error: expected a float but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-floor-1.nix b/tests/functional/lang/eval-fail-floor-1.nix new file mode 100644 index 000000000000..1d4e1dcf5b7c --- /dev/null +++ b/tests/functional/lang/eval-fail-floor-1.nix @@ -0,0 +1 @@ +builtins.floor "foo" diff --git a/tests/functional/lang/eval-fail-foldlPrime-1.err.exp b/tests/functional/lang/eval-fail-foldlPrime-1.err.exp new file mode 100644 index 000000000000..45540098ea4d --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-1.nix:1:1: + 1| builtins.foldl' 1 "foo" true + | ^ + 2| + + … while evaluating the first argument passed to builtins.foldlStrict + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-foldlPrime-1.nix b/tests/functional/lang/eval-fail-foldlPrime-1.nix new file mode 100644 index 000000000000..e7790d4191a6 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-1.nix @@ -0,0 +1 @@ +builtins.foldl' 1 "foo" true diff --git a/tests/functional/lang/eval-fail-foldlPrime-2.err.exp b/tests/functional/lang/eval-fail-foldlPrime-2.err.exp new file mode 100644 index 000000000000..7597793eb28b --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-2.nix:1:1: + 1| builtins.foldl' (_: 1) "foo" true + | ^ + 2| + + … while evaluating the third argument passed to builtins.foldlStrict + + error: expected a list but found a Boolean: true diff --git a/tests/functional/lang/eval-fail-foldlPrime-2.nix b/tests/functional/lang/eval-fail-foldlPrime-2.nix new file mode 100644 index 000000000000..9599f33cd22b --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-2.nix @@ -0,0 +1 @@ +builtins.foldl' (_: 1) "foo" true diff --git a/tests/functional/lang/eval-fail-foldlPrime-3.err.exp b/tests/functional/lang/eval-fail-foldlPrime-3.err.exp new file mode 100644 index 000000000000..8ba0a3dfb819 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-3.nix:1:1: + 1| builtins.foldl' (_: 1) "foo" [ true ] + | ^ + 2| + + error: attempt to call something which is not a function but an integer: 1 diff --git a/tests/functional/lang/eval-fail-foldlPrime-3.nix b/tests/functional/lang/eval-fail-foldlPrime-3.nix new file mode 100644 index 000000000000..6e570d9c70b1 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-3.nix @@ -0,0 +1 @@ +builtins.foldl' (_: 1) "foo" [ true ] diff --git a/tests/functional/lang/eval-fail-foldlPrime-4.err.exp b/tests/functional/lang/eval-fail-foldlPrime-4.err.exp new file mode 100644 index 000000000000..9ead5b3929e0 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-4.err.exp @@ -0,0 +1,24 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:1: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:21: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| + + … in the left operand of the AND (&&) operator + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:26: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| + + error: expected a Boolean but found a string: "foo" + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:26: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-foldlPrime-4.nix b/tests/functional/lang/eval-fail-foldlPrime-4.nix new file mode 100644 index 000000000000..c34df35dd147 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-4.nix @@ -0,0 +1 @@ +builtins.foldl' (a: b: a && b) "foo" [ true ] diff --git a/tests/functional/lang/eval-fail-functionArgs-1.err.exp b/tests/functional/lang/eval-fail-functionArgs-1.err.exp new file mode 100644 index 000000000000..4f15c0655d92 --- /dev/null +++ b/tests/functional/lang/eval-fail-functionArgs-1.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'functionArgs' builtin + at /pwd/lang/eval-fail-functionArgs-1.nix:1:1: + 1| builtins.functionArgs { } + | ^ + 2| + + error: 'functionArgs' requires a function diff --git a/tests/functional/lang/eval-fail-functionArgs-1.nix b/tests/functional/lang/eval-fail-functionArgs-1.nix new file mode 100644 index 000000000000..151e9fa976d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-functionArgs-1.nix @@ -0,0 +1 @@ +builtins.functionArgs { } diff --git a/tests/functional/lang/eval-fail-genList-1.err.exp b/tests/functional/lang/eval-fail-genList-1.err.exp new file mode 100644 index 000000000000..deea7e7ead79 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'genList' builtin + at /pwd/lang/eval-fail-genList-1.nix:1:1: + 1| builtins.genList 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.genList + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-genList-1.nix b/tests/functional/lang/eval-fail-genList-1.nix new file mode 100644 index 000000000000..276147c1d369 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-1.nix @@ -0,0 +1 @@ +builtins.genList 1 "foo" diff --git a/tests/functional/lang/eval-fail-genList-2.err.exp b/tests/functional/lang/eval-fail-genList-2.err.exp new file mode 100644 index 000000000000..dbbab7d525d8 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'genList' builtin + at /pwd/lang/eval-fail-genList-2.nix:1:1: + 1| builtins.genList 1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.genList + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-genList-2.nix b/tests/functional/lang/eval-fail-genList-2.nix new file mode 100644 index 000000000000..0f5bb6a3b668 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-2.nix @@ -0,0 +1 @@ +builtins.genList 1 2 diff --git a/tests/functional/lang/eval-fail-genList-3.err.exp b/tests/functional/lang/eval-fail-genList-3.err.exp new file mode 100644 index 000000000000..62878d88eb5c --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-3.err.exp @@ -0,0 +1,20 @@ +error: + … while evaluating list element at index 0 + + … from call site + at /pwd/lang/eval-fail-genList-3.nix:1:19: + 1| builtins.genList (x: x + "foo") 2 + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-genList-3.nix:1:19: + 1| builtins.genList (x: x + "foo") 2 + | ^ + 2| + + error: cannot add a string to an integer + at /pwd/lang/eval-fail-genList-3.nix:1:26: + 1| builtins.genList (x: x + "foo") 2 + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-genList-3.nix b/tests/functional/lang/eval-fail-genList-3.nix new file mode 100644 index 000000000000..82b8b30bc1a2 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-3.nix @@ -0,0 +1 @@ +builtins.genList (x: x + "foo") 2 diff --git a/tests/functional/lang/eval-fail-genList-4.err.exp b/tests/functional/lang/eval-fail-genList-4.err.exp new file mode 100644 index 000000000000..a60ffe6d775b --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-4.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'genList' builtin + at /pwd/lang/eval-fail-genList-4.nix:1:1: + 1| builtins.genList false (-3) + | ^ + 2| + + error: cannot create list of size -3 diff --git a/tests/functional/lang/eval-fail-genList-4.nix b/tests/functional/lang/eval-fail-genList-4.nix new file mode 100644 index 000000000000..b4f4ee6188a1 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-4.nix @@ -0,0 +1 @@ +builtins.genList false (-3) diff --git a/tests/functional/lang/eval-fail-getAttr-1.err.exp b/tests/functional/lang/eval-fail-getAttr-1.err.exp new file mode 100644 index 000000000000..d2ff63a710ad --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getAttr' builtin + at /pwd/lang/eval-fail-getAttr-1.nix:1:1: + 1| builtins.getAttr [ ] [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.getAttr + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-1.nix b/tests/functional/lang/eval-fail-getAttr-1.nix new file mode 100644 index 000000000000..80b92f45dec9 --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-1.nix @@ -0,0 +1 @@ +builtins.getAttr [ ] [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-2.err.exp b/tests/functional/lang/eval-fail-getAttr-2.err.exp new file mode 100644 index 000000000000..e4a8afe799ed --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getAttr' builtin + at /pwd/lang/eval-fail-getAttr-2.nix:1:1: + 1| builtins.getAttr "foo" [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.getAttr + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-2.nix b/tests/functional/lang/eval-fail-getAttr-2.nix new file mode 100644 index 000000000000..6d836c033255 --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-2.nix @@ -0,0 +1 @@ +builtins.getAttr "foo" [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-3.err.exp b/tests/functional/lang/eval-fail-getAttr-3.err.exp new file mode 100644 index 000000000000..e3a3dab0589b --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getAttr' builtin + at /pwd/lang/eval-fail-getAttr-3.nix:1:1: + 1| builtins.getAttr "foo" { } + | ^ + 2| + + … in the attribute set under consideration + + error: attribute 'foo' missing diff --git a/tests/functional/lang/eval-fail-getAttr-3.nix b/tests/functional/lang/eval-fail-getAttr-3.nix new file mode 100644 index 000000000000..248a616bf33d --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-3.nix @@ -0,0 +1 @@ +builtins.getAttr "foo" { } diff --git a/tests/functional/lang/eval-fail-getEnv-1.err.exp b/tests/functional/lang/eval-fail-getEnv-1.err.exp new file mode 100644 index 000000000000..92db4e5e4270 --- /dev/null +++ b/tests/functional/lang/eval-fail-getEnv-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getEnv' builtin + at /pwd/lang/eval-fail-getEnv-1.nix:1:1: + 1| builtins.getEnv [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.getEnv + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-getEnv-1.nix b/tests/functional/lang/eval-fail-getEnv-1.nix new file mode 100644 index 000000000000..5e7cb3ea1b02 --- /dev/null +++ b/tests/functional/lang/eval-fail-getEnv-1.nix @@ -0,0 +1 @@ +builtins.getEnv [ ] diff --git a/tests/functional/lang/eval-fail-groupBy-1.err.exp b/tests/functional/lang/eval-fail-groupBy-1.err.exp new file mode 100644 index 000000000000..02d0415b2bd9 --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'groupBy' builtin + at /pwd/lang/eval-fail-groupBy-1.nix:1:1: + 1| builtins.groupBy 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.groupBy + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-groupBy-1.nix b/tests/functional/lang/eval-fail-groupBy-1.nix new file mode 100644 index 000000000000..8fb41ada1352 --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-1.nix @@ -0,0 +1 @@ +builtins.groupBy 1 "foo" diff --git a/tests/functional/lang/eval-fail-groupBy-2.err.exp b/tests/functional/lang/eval-fail-groupBy-2.err.exp new file mode 100644 index 000000000000..4a9b7b6daad1 --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'groupBy' builtin + at /pwd/lang/eval-fail-groupBy-2.nix:1:1: + 1| builtins.groupBy (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.groupBy + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-groupBy-2.nix b/tests/functional/lang/eval-fail-groupBy-2.nix new file mode 100644 index 000000000000..fe91a7bae88d --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-2.nix @@ -0,0 +1 @@ +builtins.groupBy (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-groupBy-3.err.exp b/tests/functional/lang/eval-fail-groupBy-3.err.exp new file mode 100644 index 000000000000..4adc418338ed --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'groupBy' builtin + at /pwd/lang/eval-fail-groupBy-3.nix:1:1: + 1| builtins.groupBy (x: x) [ + | ^ + 2| "foo" + + … while evaluating the return value of the grouping function passed to builtins.groupBy + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-groupBy-3.nix b/tests/functional/lang/eval-fail-groupBy-3.nix new file mode 100644 index 000000000000..a4165783d99e --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-3.nix @@ -0,0 +1,5 @@ +builtins.groupBy (x: x) [ + "foo" + "bar" + 1 +] diff --git a/tests/functional/lang/eval-fail-hasAttr-1.err.exp b/tests/functional/lang/eval-fail-hasAttr-1.err.exp new file mode 100644 index 000000000000..b44ae1c0c77d --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hasAttr' builtin + at /pwd/lang/eval-fail-hasAttr-1.nix:1:1: + 1| builtins.hasAttr [ ] [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.hasAttr + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-hasAttr-1.nix b/tests/functional/lang/eval-fail-hasAttr-1.nix new file mode 100644 index 000000000000..8bdfbaaa7112 --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-1.nix @@ -0,0 +1 @@ +builtins.hasAttr [ ] [ ] diff --git a/tests/functional/lang/eval-fail-hasAttr-2.err.exp b/tests/functional/lang/eval-fail-hasAttr-2.err.exp new file mode 100644 index 000000000000..cd52370acf4b --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hasAttr' builtin + at /pwd/lang/eval-fail-hasAttr-2.nix:1:1: + 1| builtins.hasAttr "foo" [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.hasAttr + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-hasAttr-2.nix b/tests/functional/lang/eval-fail-hasAttr-2.nix new file mode 100644 index 000000000000..d49909c76389 --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-2.nix @@ -0,0 +1 @@ +builtins.hasAttr "foo" [ ] diff --git a/tests/functional/lang/eval-fail-hashString-1.err.exp b/tests/functional/lang/eval-fail-hashString-1.err.exp new file mode 100644 index 000000000000..ecce1a5d846f --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hashString' builtin + at /pwd/lang/eval-fail-hashString-1.nix:1:1: + 1| builtins.hashString 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.hashString + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-hashString-1.nix b/tests/functional/lang/eval-fail-hashString-1.nix new file mode 100644 index 000000000000..bbca23d43f7f --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-1.nix @@ -0,0 +1 @@ +builtins.hashString 1 { } diff --git a/tests/functional/lang/eval-fail-hashString-2.err.exp b/tests/functional/lang/eval-fail-hashString-2.err.exp new file mode 100644 index 000000000000..edbf5e23ff78 --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-2.err.exp @@ -0,0 +1,9 @@ +error: + … while calling the 'hashString' builtin + at /pwd/lang/eval-fail-hashString-2.nix:1:1: + 1| builtins.hashString "foo" "content" + | ^ + 2| + + error: unknown hash algorithm 'foo', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512' +Try 'nix-instantiate --help' for more information. diff --git a/tests/functional/lang/eval-fail-hashString-2.nix b/tests/functional/lang/eval-fail-hashString-2.nix new file mode 100644 index 000000000000..9439b8b41d55 --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-2.nix @@ -0,0 +1 @@ +builtins.hashString "foo" "content" diff --git a/tests/functional/lang/eval-fail-hashString-3.err.exp b/tests/functional/lang/eval-fail-hashString-3.err.exp new file mode 100644 index 000000000000..2126b0062572 --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hashString' builtin + at /pwd/lang/eval-fail-hashString-3.nix:1:1: + 1| builtins.hashString "sha256" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.hashString + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-hashString-3.nix b/tests/functional/lang/eval-fail-hashString-3.nix new file mode 100644 index 000000000000..50b960c8d38c --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-3.nix @@ -0,0 +1 @@ +builtins.hashString "sha256" { } diff --git a/tests/functional/lang/eval-fail-head-1.err.exp b/tests/functional/lang/eval-fail-head-1.err.exp new file mode 100644 index 000000000000..3fcbbe29b68c --- /dev/null +++ b/tests/functional/lang/eval-fail-head-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'head' builtin + at /pwd/lang/eval-fail-head-1.nix:1:1: + 1| builtins.head 1 + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.head' + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-head-1.nix b/tests/functional/lang/eval-fail-head-1.nix new file mode 100644 index 000000000000..11aa7248706e --- /dev/null +++ b/tests/functional/lang/eval-fail-head-1.nix @@ -0,0 +1 @@ +builtins.head 1 diff --git a/tests/functional/lang/eval-fail-head-2.err.exp b/tests/functional/lang/eval-fail-head-2.err.exp new file mode 100644 index 000000000000..68cecc8fa1d0 --- /dev/null +++ b/tests/functional/lang/eval-fail-head-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'head' builtin + at /pwd/lang/eval-fail-head-2.nix:1:1: + 1| builtins.head [ ] + | ^ + 2| + + error: 'builtins.head' called on an empty list diff --git a/tests/functional/lang/eval-fail-head-2.nix b/tests/functional/lang/eval-fail-head-2.nix new file mode 100644 index 000000000000..c7a77da4e20e --- /dev/null +++ b/tests/functional/lang/eval-fail-head-2.nix @@ -0,0 +1 @@ +builtins.head [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-1.err.exp b/tests/functional/lang/eval-fail-intersectAttrs-1.err.exp new file mode 100644 index 000000000000..098084a3f4f8 --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'intersectAttrs' builtin + at /pwd/lang/eval-fail-intersectAttrs-1.nix:1:1: + 1| builtins.intersectAttrs [ ] [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.intersectAttrs + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-1.nix b/tests/functional/lang/eval-fail-intersectAttrs-1.nix new file mode 100644 index 000000000000..5a8c133163d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-1.nix @@ -0,0 +1 @@ +builtins.intersectAttrs [ ] [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-2.err.exp b/tests/functional/lang/eval-fail-intersectAttrs-2.err.exp new file mode 100644 index 000000000000..663df4f6147e --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'intersectAttrs' builtin + at /pwd/lang/eval-fail-intersectAttrs-2.nix:1:1: + 1| builtins.intersectAttrs { } [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.intersectAttrs + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-2.nix b/tests/functional/lang/eval-fail-intersectAttrs-2.nix new file mode 100644 index 000000000000..413ce5d58b3a --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-2.nix @@ -0,0 +1 @@ +builtins.intersectAttrs { } [ ] diff --git a/tests/functional/lang/eval-fail-length-1.err.exp b/tests/functional/lang/eval-fail-length-1.err.exp new file mode 100644 index 000000000000..7161a2fbf26f --- /dev/null +++ b/tests/functional/lang/eval-fail-length-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'length' builtin + at /pwd/lang/eval-fail-length-1.nix:1:1: + 1| builtins.length 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.length + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-length-1.nix b/tests/functional/lang/eval-fail-length-1.nix new file mode 100644 index 000000000000..972eb72c7697 --- /dev/null +++ b/tests/functional/lang/eval-fail-length-1.nix @@ -0,0 +1 @@ +builtins.length 1 diff --git a/tests/functional/lang/eval-fail-length-2.err.exp b/tests/functional/lang/eval-fail-length-2.err.exp new file mode 100644 index 000000000000..8bd66a9a701a --- /dev/null +++ b/tests/functional/lang/eval-fail-length-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'length' builtin + at /pwd/lang/eval-fail-length-2.nix:1:1: + 1| builtins.length "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.length + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-length-2.nix b/tests/functional/lang/eval-fail-length-2.nix new file mode 100644 index 000000000000..a42e1034ab07 --- /dev/null +++ b/tests/functional/lang/eval-fail-length-2.nix @@ -0,0 +1 @@ +builtins.length "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-1.err.exp b/tests/functional/lang/eval-fail-lessThan-1.err.exp new file mode 100644 index 000000000000..09ec5d8ffdbe --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-1.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-lessThan-1.nix:1:1: + 1| builtins.lessThan 1 "foo" + | ^ + 2| + + error: cannot compare an integer with a string; values are 1 and "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-1.nix b/tests/functional/lang/eval-fail-lessThan-1.nix new file mode 100644 index 000000000000..10cd489674ad --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-1.nix @@ -0,0 +1 @@ +builtins.lessThan 1 "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-2.err.exp b/tests/functional/lang/eval-fail-lessThan-2.err.exp new file mode 100644 index 000000000000..6a8a183fa74e --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-lessThan-2.nix:1:1: + 1| builtins.lessThan { } { } + | ^ + 2| + + error: cannot compare a set with a set; values of that type are incomparable (values are { } and { }) diff --git a/tests/functional/lang/eval-fail-lessThan-2.nix b/tests/functional/lang/eval-fail-lessThan-2.nix new file mode 100644 index 000000000000..a20eca6bba14 --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-2.nix @@ -0,0 +1 @@ +builtins.lessThan { } { } diff --git a/tests/functional/lang/eval-fail-lessThan-3.err.exp b/tests/functional/lang/eval-fail-lessThan-3.err.exp new file mode 100644 index 000000000000..4a66625c00cd --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-lessThan-3.nix:1:1: + 1| builtins.lessThan [ 1 2 ] [ "foo" ] + | ^ + 2| + + … while comparing two list elements + + error: cannot compare an integer with a string; values are 1 and "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-3.nix b/tests/functional/lang/eval-fail-lessThan-3.nix new file mode 100644 index 000000000000..bdea0fef172c --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-3.nix @@ -0,0 +1 @@ +builtins.lessThan [ 1 2 ] [ "foo" ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-1.err.exp b/tests/functional/lang/eval-fail-listToAttrs-1.err.exp new file mode 100644 index 000000000000..ce2ef38c8b52 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-1.nix:1:1: + 1| builtins.listToAttrs 1 + | ^ + 2| + + … while evaluating the argument passed to builtins.listToAttrs + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-listToAttrs-1.nix b/tests/functional/lang/eval-fail-listToAttrs-1.nix new file mode 100644 index 000000000000..260f136e51ca --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-1.nix @@ -0,0 +1 @@ +builtins.listToAttrs 1 diff --git a/tests/functional/lang/eval-fail-listToAttrs-2.err.exp b/tests/functional/lang/eval-fail-listToAttrs-2.err.exp new file mode 100644 index 000000000000..9809d216a253 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-2.nix:1:1: + 1| builtins.listToAttrs [ 1 ] + | ^ + 2| + + … while evaluating an element of the list passed to builtins.listToAttrs + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-listToAttrs-2.nix b/tests/functional/lang/eval-fail-listToAttrs-2.nix new file mode 100644 index 000000000000..2db54c3c7b52 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-2.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ 1 ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-3.err.exp b/tests/functional/lang/eval-fail-listToAttrs-3.err.exp new file mode 100644 index 000000000000..5c4886c1745f --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-3.nix:1:1: + 1| builtins.listToAttrs [ { } ] + | ^ + 2| + + … in a {name=...; value=...;} pair + + error: attribute 'name' missing diff --git a/tests/functional/lang/eval-fail-listToAttrs-3.nix b/tests/functional/lang/eval-fail-listToAttrs-3.nix new file mode 100644 index 000000000000..ce5b11798153 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-3.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ { } ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-4.err.exp b/tests/functional/lang/eval-fail-listToAttrs-4.err.exp new file mode 100644 index 000000000000..cb284ad2cd38 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-4.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-4.nix:1:1: + 1| builtins.listToAttrs [ { name = 1; } ] + | ^ + 2| + + … while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs + at /pwd/lang/eval-fail-listToAttrs-4.nix:1:26: + 1| builtins.listToAttrs [ { name = 1; } ] + | ^ + 2| + + error: expected a string but found an integer: 1 + at /pwd/lang/eval-fail-listToAttrs-4.nix:1:26: + 1| builtins.listToAttrs [ { name = 1; } ] + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-listToAttrs-4.nix b/tests/functional/lang/eval-fail-listToAttrs-4.nix new file mode 100644 index 000000000000..a4bc7fd11c67 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-4.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ { name = 1; } ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-5.err.exp b/tests/functional/lang/eval-fail-listToAttrs-5.err.exp new file mode 100644 index 000000000000..a03834b49a37 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-5.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-5.nix:1:1: + 1| builtins.listToAttrs [ { name = "foo"; } ] + | ^ + 2| + + … in a {name=...; value=...;} pair + + error: attribute 'value' missing diff --git a/tests/functional/lang/eval-fail-listToAttrs-5.nix b/tests/functional/lang/eval-fail-listToAttrs-5.nix new file mode 100644 index 000000000000..7cca5d0c7258 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-5.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ { name = "foo"; } ] diff --git a/tests/functional/lang/eval-fail-map-1.err.exp b/tests/functional/lang/eval-fail-map-1.err.exp new file mode 100644 index 000000000000..65c0d5d3abdf --- /dev/null +++ b/tests/functional/lang/eval-fail-map-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'map' builtin + at /pwd/lang/eval-fail-map-1.nix:1:1: + 1| builtins.map 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.map + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-map-1.nix b/tests/functional/lang/eval-fail-map-1.nix new file mode 100644 index 000000000000..65f5cf8a67b2 --- /dev/null +++ b/tests/functional/lang/eval-fail-map-1.nix @@ -0,0 +1 @@ +builtins.map 1 "foo" diff --git a/tests/functional/lang/eval-fail-map-2.err.exp b/tests/functional/lang/eval-fail-map-2.err.exp new file mode 100644 index 000000000000..cf9a3f636753 --- /dev/null +++ b/tests/functional/lang/eval-fail-map-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'map' builtin + at /pwd/lang/eval-fail-map-2.nix:1:1: + 1| builtins.map 1 [ 1 ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.map + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-map-2.nix b/tests/functional/lang/eval-fail-map-2.nix new file mode 100644 index 000000000000..16100c0a7905 --- /dev/null +++ b/tests/functional/lang/eval-fail-map-2.nix @@ -0,0 +1 @@ +builtins.map 1 [ 1 ] diff --git a/tests/functional/lang/eval-fail-mapAttrs-1.err.exp b/tests/functional/lang/eval-fail-mapAttrs-1.err.exp new file mode 100644 index 000000000000..9a10bb92f841 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'mapAttrs' builtin + at /pwd/lang/eval-fail-mapAttrs-1.nix:1:1: + 1| builtins.mapAttrs [ ] [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.mapAttrs + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-mapAttrs-1.nix b/tests/functional/lang/eval-fail-mapAttrs-1.nix new file mode 100644 index 000000000000..c913e838d81a --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-1.nix @@ -0,0 +1 @@ +builtins.mapAttrs [ ] [ ] diff --git a/tests/functional/lang/eval-fail-mapAttrs-2.err.exp b/tests/functional/lang/eval-fail-mapAttrs-2.err.exp new file mode 100644 index 000000000000..9cc2b5495a58 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-2.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the attribute 'foo' + + error: attempt to call something which is not a function but a string: "" diff --git a/tests/functional/lang/eval-fail-mapAttrs-2.nix b/tests/functional/lang/eval-fail-mapAttrs-2.nix new file mode 100644 index 000000000000..b58ed8c3a660 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-2.nix @@ -0,0 +1 @@ +builtins.mapAttrs "" { foo.bar = 1; } diff --git a/tests/functional/lang/eval-fail-mapAttrs-3.err.exp b/tests/functional/lang/eval-fail-mapAttrs-3.err.exp new file mode 100644 index 000000000000..f4c48ee9a48f --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-3.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the attribute 'foo' + + error: attempt to call something which is not a function but a string: "foo1" diff --git a/tests/functional/lang/eval-fail-mapAttrs-3.nix b/tests/functional/lang/eval-fail-mapAttrs-3.nix new file mode 100644 index 000000000000..6fc4de8251cd --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-3.nix @@ -0,0 +1 @@ +builtins.mapAttrs (x: x + "1") { foo.bar = 1; } diff --git a/tests/functional/lang/eval-fail-mapAttrs-4.err.exp b/tests/functional/lang/eval-fail-mapAttrs-4.err.exp new file mode 100644 index 000000000000..bd18ceeac946 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-4.err.exp @@ -0,0 +1,16 @@ +error: + … while evaluating the attribute 'foo' + + … while calling anonymous lambda + at /pwd/lang/eval-fail-mapAttrs-4.nix:1:23: + 1| builtins.mapAttrs (x: y: x + 1) { foo.bar = 1; } + | ^ + 2| + + … while evaluating a path segment + at /pwd/lang/eval-fail-mapAttrs-4.nix:1:30: + 1| builtins.mapAttrs (x: y: x + 1) { foo.bar = 1; } + | ^ + 2| + + error: cannot coerce an integer to a string: 1 diff --git a/tests/functional/lang/eval-fail-mapAttrs-4.nix b/tests/functional/lang/eval-fail-mapAttrs-4.nix new file mode 100644 index 000000000000..2ad09b602434 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-4.nix @@ -0,0 +1 @@ +builtins.mapAttrs (x: y: x + 1) { foo.bar = 1; } diff --git a/tests/functional/lang/eval-fail-match-1.err.exp b/tests/functional/lang/eval-fail-match-1.err.exp new file mode 100644 index 000000000000..2f757a6834c2 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'match' builtin + at /pwd/lang/eval-fail-match-1.nix:1:1: + 1| builtins.match 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.match + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-match-1.nix b/tests/functional/lang/eval-fail-match-1.nix new file mode 100644 index 000000000000..3eb26c017c53 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-1.nix @@ -0,0 +1 @@ +builtins.match 1 { } diff --git a/tests/functional/lang/eval-fail-match-2.err.exp b/tests/functional/lang/eval-fail-match-2.err.exp new file mode 100644 index 000000000000..e415f5ccd3e1 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'match' builtin + at /pwd/lang/eval-fail-match-2.nix:1:1: + 1| builtins.match "foo" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.match + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-match-2.nix b/tests/functional/lang/eval-fail-match-2.nix new file mode 100644 index 000000000000..35eb16cfbf2d --- /dev/null +++ b/tests/functional/lang/eval-fail-match-2.nix @@ -0,0 +1 @@ +builtins.match "foo" { } diff --git a/tests/functional/lang/eval-fail-match-3.err.exp b/tests/functional/lang/eval-fail-match-3.err.exp new file mode 100644 index 000000000000..d69577bd7163 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'match' builtin + at /pwd/lang/eval-fail-match-3.nix:1:1: + 1| builtins.match "(.*" "" + | ^ + 2| + + error: invalid regular expression '(.*' diff --git a/tests/functional/lang/eval-fail-match-3.nix b/tests/functional/lang/eval-fail-match-3.nix new file mode 100644 index 000000000000..8115807f9220 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-3.nix @@ -0,0 +1 @@ +builtins.match "(.*" "" diff --git a/tests/functional/lang/eval-fail-mul-1.err.exp b/tests/functional/lang/eval-fail-mul-1.err.exp new file mode 100644 index 000000000000..c03284987c9f --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-mul-1.nix:1:1: + 1| builtins.mul "foo" 1 + | ^ + 2| + + … while evaluating the first argument of the multiplication + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-mul-1.nix b/tests/functional/lang/eval-fail-mul-1.nix new file mode 100644 index 000000000000..a16333824671 --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-1.nix @@ -0,0 +1 @@ +builtins.mul "foo" 1 diff --git a/tests/functional/lang/eval-fail-mul-2.err.exp b/tests/functional/lang/eval-fail-mul-2.err.exp new file mode 100644 index 000000000000..16744abe7310 --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-mul-2.nix:1:1: + 1| builtins.mul 1 "foo" + | ^ + 2| + + … while evaluating the second argument of the multiplication + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-mul-2.nix b/tests/functional/lang/eval-fail-mul-2.nix new file mode 100644 index 000000000000..57e2ef6925f0 --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-2.nix @@ -0,0 +1 @@ +builtins.mul 1 "foo" diff --git a/tests/functional/lang/eval-fail-parseDrvName-1.err.exp b/tests/functional/lang/eval-fail-parseDrvName-1.err.exp new file mode 100644 index 000000000000..e5fbc2b6ec84 --- /dev/null +++ b/tests/functional/lang/eval-fail-parseDrvName-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'parseDrvName' builtin + at /pwd/lang/eval-fail-parseDrvName-1.nix:1:1: + 1| builtins.parseDrvName 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.parseDrvName + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-parseDrvName-1.nix b/tests/functional/lang/eval-fail-parseDrvName-1.nix new file mode 100644 index 000000000000..f524d500f5fc --- /dev/null +++ b/tests/functional/lang/eval-fail-parseDrvName-1.nix @@ -0,0 +1 @@ +builtins.parseDrvName 1 diff --git a/tests/functional/lang/eval-fail-partition-1.err.exp b/tests/functional/lang/eval-fail-partition-1.err.exp new file mode 100644 index 000000000000..d989969bd552 --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'partition' builtin + at /pwd/lang/eval-fail-partition-1.nix:1:1: + 1| builtins.partition 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.partition + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-partition-1.nix b/tests/functional/lang/eval-fail-partition-1.nix new file mode 100644 index 000000000000..0452d11101d0 --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-1.nix @@ -0,0 +1 @@ +builtins.partition 1 "foo" diff --git a/tests/functional/lang/eval-fail-partition-2.err.exp b/tests/functional/lang/eval-fail-partition-2.err.exp new file mode 100644 index 000000000000..e3fef6848fcf --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'partition' builtin + at /pwd/lang/eval-fail-partition-2.nix:1:1: + 1| builtins.partition (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.partition + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-partition-2.nix b/tests/functional/lang/eval-fail-partition-2.nix new file mode 100644 index 000000000000..1a7219c5f3d4 --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-2.nix @@ -0,0 +1 @@ +builtins.partition (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-partition-3.err.exp b/tests/functional/lang/eval-fail-partition-3.err.exp new file mode 100644 index 000000000000..a1257c7a6b1c --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'partition' builtin + at /pwd/lang/eval-fail-partition-3.nix:1:1: + 1| builtins.partition (_: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the partition function passed to builtins.partition + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-partition-3.nix b/tests/functional/lang/eval-fail-partition-3.nix new file mode 100644 index 000000000000..07aa480fc18d --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-3.nix @@ -0,0 +1 @@ +builtins.partition (_: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-pathExists-1.err.exp b/tests/functional/lang/eval-fail-pathExists-1.err.exp new file mode 100644 index 000000000000..3349247966fd --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'pathExists' builtin + at /pwd/lang/eval-fail-pathExists-1.nix:1:1: + 1| builtins.pathExists [ ] + | ^ + 2| + + … while realising the context of a path + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-pathExists-1.nix b/tests/functional/lang/eval-fail-pathExists-1.nix new file mode 100644 index 000000000000..83bda9afca95 --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-1.nix @@ -0,0 +1 @@ +builtins.pathExists [ ] diff --git a/tests/functional/lang/eval-fail-pathExists-2.err.exp b/tests/functional/lang/eval-fail-pathExists-2.err.exp new file mode 100644 index 000000000000..184e47238d82 --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'pathExists' builtin + at /pwd/lang/eval-fail-pathExists-2.nix:1:1: + 1| builtins.pathExists "zorglub" + | ^ + 2| + + … while realising the context of a path + + error: string 'zorglub' doesn't represent an absolute path diff --git a/tests/functional/lang/eval-fail-pathExists-2.nix b/tests/functional/lang/eval-fail-pathExists-2.nix new file mode 100644 index 000000000000..da594ca602b5 --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-2.nix @@ -0,0 +1 @@ +builtins.pathExists "zorglub" diff --git a/tests/functional/lang/eval-fail-placeholder-1.err.exp b/tests/functional/lang/eval-fail-placeholder-1.err.exp new file mode 100644 index 000000000000..436095448054 --- /dev/null +++ b/tests/functional/lang/eval-fail-placeholder-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'placeholder' builtin + at /pwd/lang/eval-fail-placeholder-1.nix:1:1: + 1| builtins.placeholder [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.placeholder + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-placeholder-1.nix b/tests/functional/lang/eval-fail-placeholder-1.nix new file mode 100644 index 000000000000..ecc934a5e357 --- /dev/null +++ b/tests/functional/lang/eval-fail-placeholder-1.nix @@ -0,0 +1 @@ +builtins.placeholder [ ] diff --git a/tests/functional/lang/eval-fail-removeAttrs-1.err.exp b/tests/functional/lang/eval-fail-removeAttrs-1.err.exp new file mode 100644 index 000000000000..ddfde967f164 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'removeAttrs' builtin + at /pwd/lang/eval-fail-removeAttrs-1.nix:1:1: + 1| builtins.removeAttrs "" "" + | ^ + 2| + + … while evaluating the first argument passed to builtins.removeAttrs + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-1.nix b/tests/functional/lang/eval-fail-removeAttrs-1.nix new file mode 100644 index 000000000000..83fe63003c9d --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-1.nix @@ -0,0 +1 @@ +builtins.removeAttrs "" "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-2.err.exp b/tests/functional/lang/eval-fail-removeAttrs-2.err.exp new file mode 100644 index 000000000000..ab668aaea0e4 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'removeAttrs' builtin + at /pwd/lang/eval-fail-removeAttrs-2.nix:1:1: + 1| builtins.removeAttrs "" [ 1 ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.removeAttrs + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-2.nix b/tests/functional/lang/eval-fail-removeAttrs-2.nix new file mode 100644 index 000000000000..ecedf26cf975 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-2.nix @@ -0,0 +1 @@ +builtins.removeAttrs "" [ 1 ] diff --git a/tests/functional/lang/eval-fail-removeAttrs-3.err.exp b/tests/functional/lang/eval-fail-removeAttrs-3.err.exp new file mode 100644 index 000000000000..1937cd1be86d --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'removeAttrs' builtin + at /pwd/lang/eval-fail-removeAttrs-3.nix:1:1: + 1| builtins.removeAttrs "" [ "1" ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.removeAttrs + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-3.nix b/tests/functional/lang/eval-fail-removeAttrs-3.nix new file mode 100644 index 000000000000..4bc9e02394b7 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-3.nix @@ -0,0 +1 @@ +builtins.removeAttrs "" [ "1" ] diff --git a/tests/functional/lang/eval-fail-replaceStrings-1.err.exp b/tests/functional/lang/eval-fail-replaceStrings-1.err.exp new file mode 100644 index 000000000000..b0bc7c758c73 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-1.nix:1:1: + 1| builtins.replaceStrings 0 0 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.replaceStrings + + error: expected a list but found an integer: 0 diff --git a/tests/functional/lang/eval-fail-replaceStrings-1.nix b/tests/functional/lang/eval-fail-replaceStrings-1.nix new file mode 100644 index 000000000000..7e9edc28d26f --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-1.nix @@ -0,0 +1 @@ +builtins.replaceStrings 0 0 { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-2.err.exp b/tests/functional/lang/eval-fail-replaceStrings-2.err.exp new file mode 100644 index 000000000000..ee289eded6f1 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-2.nix:1:1: + 1| builtins.replaceStrings [ ] 0 { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.replaceStrings + + error: expected a list but found an integer: 0 diff --git a/tests/functional/lang/eval-fail-replaceStrings-2.nix b/tests/functional/lang/eval-fail-replaceStrings-2.nix new file mode 100644 index 000000000000..57311720207f --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-2.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ ] 0 { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-3.err.exp b/tests/functional/lang/eval-fail-replaceStrings-3.err.exp new file mode 100644 index 000000000000..3e71cfa75c9a --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-3.nix:1:1: + 1| builtins.replaceStrings [ 0 ] [ ] { } + | ^ + 2| + + error: 'from' and 'to' arguments passed to builtins.replaceStrings have different lengths diff --git a/tests/functional/lang/eval-fail-replaceStrings-3.nix b/tests/functional/lang/eval-fail-replaceStrings-3.nix new file mode 100644 index 000000000000..c7cdac4a9665 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-3.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ 0 ] [ ] { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-4.err.exp b/tests/functional/lang/eval-fail-replaceStrings-4.err.exp new file mode 100644 index 000000000000..f4e834090531 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-4.nix:1:1: + 1| builtins.replaceStrings [ 1 ] [ "new" ] { } + | ^ + 2| + + … while evaluating one of the strings to replace passed to builtins.replaceStrings + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-replaceStrings-4.nix b/tests/functional/lang/eval-fail-replaceStrings-4.nix new file mode 100644 index 000000000000..cc700ea54a50 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-4.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ 1 ] [ "new" ] { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-5.err.exp b/tests/functional/lang/eval-fail-replaceStrings-5.err.exp new file mode 100644 index 000000000000..75e2b5e8b345 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-5.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-5.nix:1:1: + 1| builtins.replaceStrings [ "oo" ] [ true ] "foo" + | ^ + 2| + + … while evaluating one of the replacement strings passed to builtins.replaceStrings + + error: expected a string but found a Boolean: true diff --git a/tests/functional/lang/eval-fail-replaceStrings-5.nix b/tests/functional/lang/eval-fail-replaceStrings-5.nix new file mode 100644 index 000000000000..40010d845e40 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-5.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ "oo" ] [ true ] "foo" diff --git a/tests/functional/lang/eval-fail-replaceStrings-6.err.exp b/tests/functional/lang/eval-fail-replaceStrings-6.err.exp new file mode 100644 index 000000000000..5ced1affaf21 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-6.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-6.nix:1:1: + 1| builtins.replaceStrings [ "old" ] [ "new" ] { } + | ^ + 2| + + … while evaluating the third argument passed to builtins.replaceStrings + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-6.nix b/tests/functional/lang/eval-fail-replaceStrings-6.nix new file mode 100644 index 000000000000..2953b4888bef --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-6.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ "old" ] [ "new" ] { } diff --git a/tests/functional/lang/eval-fail-sort-1.err.exp b/tests/functional/lang/eval-fail-sort-1.err.exp new file mode 100644 index 000000000000..2797d344d6aa --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-1.nix:1:1: + 1| builtins.sort 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.sort + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-sort-1.nix b/tests/functional/lang/eval-fail-sort-1.nix new file mode 100644 index 000000000000..12d3a102872d --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-1.nix @@ -0,0 +1 @@ +builtins.sort 1 "foo" diff --git a/tests/functional/lang/eval-fail-sort-2.err.exp b/tests/functional/lang/eval-fail-sort-2.err.exp new file mode 100644 index 000000000000..462ade0e447b --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-2.nix:1:1: + 1| builtins.sort 1 [ "foo" ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.sort + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-sort-2.nix b/tests/functional/lang/eval-fail-sort-2.nix new file mode 100644 index 000000000000..27ffb1d3633c --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-2.nix @@ -0,0 +1 @@ +builtins.sort 1 [ "foo" ] diff --git a/tests/functional/lang/eval-fail-sort-3.err.exp b/tests/functional/lang/eval-fail-sort-3.err.exp new file mode 100644 index 000000000000..0d86a7f4d1a3 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-3.nix:1:1: + 1| builtins.sort (_: 1) [ + | ^ + 2| "foo" + + error: attempt to call something which is not a function but an integer: 1 diff --git a/tests/functional/lang/eval-fail-sort-3.nix b/tests/functional/lang/eval-fail-sort-3.nix new file mode 100644 index 000000000000..1fcd434d07f8 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-3.nix @@ -0,0 +1,4 @@ +builtins.sort (_: 1) [ + "foo" + "bar" +] diff --git a/tests/functional/lang/eval-fail-sort-4.err.exp b/tests/functional/lang/eval-fail-sort-4.err.exp new file mode 100644 index 000000000000..517eefeb5c98 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-4.nix:1:1: + 1| builtins.sort (_: _: 1) [ + | ^ + 2| "foo" + + … while evaluating the return value of the sorting function passed to builtins.sort + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-sort-4.nix b/tests/functional/lang/eval-fail-sort-4.nix new file mode 100644 index 000000000000..89868de9abb7 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-4.nix @@ -0,0 +1,4 @@ +builtins.sort (_: _: 1) [ + "foo" + "bar" +] diff --git a/tests/functional/lang/eval-fail-sort-5.err.exp b/tests/functional/lang/eval-fail-sort-5.err.exp new file mode 100644 index 000000000000..f4c66dbd466c --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-5.err.exp @@ -0,0 +1,26 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-5.nix:1:1: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + … while calling anonymous lambda + at /pwd/lang/eval-fail-sort-5.nix:1:19: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + … in the argument of the not operator + at /pwd/lang/eval-fail-sort-5.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-sort-5.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + error: cannot compare a string with a set; values are "foo" and { } diff --git a/tests/functional/lang/eval-fail-sort-5.nix b/tests/functional/lang/eval-fail-sort-5.nix new file mode 100644 index 000000000000..bd50e0b325da --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-5.nix @@ -0,0 +1,4 @@ +builtins.sort (a: b: a <= b) [ + "foo" + { } +] diff --git a/tests/functional/lang/eval-fail-sort-6.err.exp b/tests/functional/lang/eval-fail-sort-6.err.exp new file mode 100644 index 000000000000..302a27f2361b --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-6.err.exp @@ -0,0 +1,26 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-6.nix:1:1: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + … while calling anonymous lambda + at /pwd/lang/eval-fail-sort-6.nix:1:19: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + … in the argument of the not operator + at /pwd/lang/eval-fail-sort-6.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-sort-6.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + error: cannot compare a set with a set; values of that type are incomparable (values are { } and { }) diff --git a/tests/functional/lang/eval-fail-sort-6.nix b/tests/functional/lang/eval-fail-sort-6.nix new file mode 100644 index 000000000000..3c527596d82c --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-6.nix @@ -0,0 +1,4 @@ +builtins.sort (a: b: a <= b) [ + { } + { } +] diff --git a/tests/functional/lang/eval-fail-split-1.err.exp b/tests/functional/lang/eval-fail-split-1.err.exp new file mode 100644 index 000000000000..c8ac7085741c --- /dev/null +++ b/tests/functional/lang/eval-fail-split-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'split' builtin + at /pwd/lang/eval-fail-split-1.nix:1:1: + 1| builtins.split 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.split + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-split-1.nix b/tests/functional/lang/eval-fail-split-1.nix new file mode 100644 index 000000000000..ad702a1a3a80 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-1.nix @@ -0,0 +1 @@ +builtins.split 1 { } diff --git a/tests/functional/lang/eval-fail-split-2.err.exp b/tests/functional/lang/eval-fail-split-2.err.exp new file mode 100644 index 000000000000..a912771d0639 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'split' builtin + at /pwd/lang/eval-fail-split-2.nix:1:1: + 1| builtins.split "foo" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.split + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-split-2.nix b/tests/functional/lang/eval-fail-split-2.nix new file mode 100644 index 000000000000..43489c5374e2 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-2.nix @@ -0,0 +1 @@ +builtins.split "foo" { } diff --git a/tests/functional/lang/eval-fail-split-3.err.exp b/tests/functional/lang/eval-fail-split-3.err.exp new file mode 100644 index 000000000000..a7cfd394188b --- /dev/null +++ b/tests/functional/lang/eval-fail-split-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'split' builtin + at /pwd/lang/eval-fail-split-3.nix:1:1: + 1| builtins.split "f(o*o" "1foo2" + | ^ + 2| + + error: invalid regular expression 'f(o*o' diff --git a/tests/functional/lang/eval-fail-split-3.nix b/tests/functional/lang/eval-fail-split-3.nix new file mode 100644 index 000000000000..f45294e93050 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-3.nix @@ -0,0 +1 @@ +builtins.split "f(o*o" "1foo2" diff --git a/tests/functional/lang/eval-fail-splitVersion-1.err.exp b/tests/functional/lang/eval-fail-splitVersion-1.err.exp new file mode 100644 index 000000000000..f7eb7614d89d --- /dev/null +++ b/tests/functional/lang/eval-fail-splitVersion-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'splitVersion' builtin + at /pwd/lang/eval-fail-splitVersion-1.nix:1:1: + 1| builtins.splitVersion 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.splitVersion + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-splitVersion-1.nix b/tests/functional/lang/eval-fail-splitVersion-1.nix new file mode 100644 index 000000000000..43941d1b6640 --- /dev/null +++ b/tests/functional/lang/eval-fail-splitVersion-1.nix @@ -0,0 +1 @@ +builtins.splitVersion 1 diff --git a/tests/functional/lang/eval-fail-storePath-1.err.exp b/tests/functional/lang/eval-fail-storePath-1.err.exp new file mode 100644 index 000000000000..711ed13ed8ce --- /dev/null +++ b/tests/functional/lang/eval-fail-storePath-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'storePath' builtin + at /pwd/lang/eval-fail-storePath-1.nix:1:1: + 1| builtins.storePath true + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.storePath' + + error: cannot coerce a Boolean to a string: true diff --git a/tests/functional/lang/eval-fail-storePath-1.nix b/tests/functional/lang/eval-fail-storePath-1.nix new file mode 100644 index 000000000000..d4ce341022d0 --- /dev/null +++ b/tests/functional/lang/eval-fail-storePath-1.nix @@ -0,0 +1 @@ +builtins.storePath true diff --git a/tests/functional/lang/eval-fail-stringLength-1.err.exp b/tests/functional/lang/eval-fail-stringLength-1.err.exp new file mode 100644 index 000000000000..37d63fd493c2 --- /dev/null +++ b/tests/functional/lang/eval-fail-stringLength-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'stringLength' builtin + at /pwd/lang/eval-fail-stringLength-1.nix:1:1: + 1| builtins.stringLength { } + | ^ + 2| + + … while evaluating the argument passed to builtins.stringLength + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-stringLength-1.nix b/tests/functional/lang/eval-fail-stringLength-1.nix new file mode 100644 index 000000000000..0a5ba6d8a842 --- /dev/null +++ b/tests/functional/lang/eval-fail-stringLength-1.nix @@ -0,0 +1 @@ +builtins.stringLength { } diff --git a/tests/functional/lang/eval-fail-sub-1.err.exp b/tests/functional/lang/eval-fail-sub-1.err.exp new file mode 100644 index 000000000000..769bf252bbb8 --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sub' builtin + at /pwd/lang/eval-fail-sub-1.nix:1:1: + 1| builtins.sub "foo" 1 + | ^ + 2| + + … while evaluating the first argument of the subtraction + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-sub-1.nix b/tests/functional/lang/eval-fail-sub-1.nix new file mode 100644 index 000000000000..cbff4ca1ed72 --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-1.nix @@ -0,0 +1 @@ +builtins.sub "foo" 1 diff --git a/tests/functional/lang/eval-fail-sub-2.err.exp b/tests/functional/lang/eval-fail-sub-2.err.exp new file mode 100644 index 000000000000..ee5ed4618a53 --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sub' builtin + at /pwd/lang/eval-fail-sub-2.nix:1:1: + 1| builtins.sub 1 "foo" + | ^ + 2| + + … while evaluating the second argument of the subtraction + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-sub-2.nix b/tests/functional/lang/eval-fail-sub-2.nix new file mode 100644 index 000000000000..4a3cfc6e0254 --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-2.nix @@ -0,0 +1 @@ +builtins.sub 1 "foo" diff --git a/tests/functional/lang/eval-fail-substring-1.err.exp b/tests/functional/lang/eval-fail-substring-1.err.exp new file mode 100644 index 000000000000..84e61c17c075 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-1.nix:1:1: + 1| builtins.substring { } "foo" true + | ^ + 2| + + … while evaluating the first argument (the start offset) passed to builtins.substring + + error: expected an integer but found a set: { } diff --git a/tests/functional/lang/eval-fail-substring-1.nix b/tests/functional/lang/eval-fail-substring-1.nix new file mode 100644 index 000000000000..c12228550bf8 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-1.nix @@ -0,0 +1 @@ +builtins.substring { } "foo" true diff --git a/tests/functional/lang/eval-fail-substring-2.err.exp b/tests/functional/lang/eval-fail-substring-2.err.exp new file mode 100644 index 000000000000..641fde5a83dc --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-2.nix:1:1: + 1| builtins.substring 3 "foo" true + | ^ + 2| + + … while evaluating the second argument (the substring length) passed to builtins.substring + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-substring-2.nix b/tests/functional/lang/eval-fail-substring-2.nix new file mode 100644 index 000000000000..358b0d083506 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-2.nix @@ -0,0 +1 @@ +builtins.substring 3 "foo" true diff --git a/tests/functional/lang/eval-fail-substring-3.err.exp b/tests/functional/lang/eval-fail-substring-3.err.exp new file mode 100644 index 000000000000..67b12cab2709 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-3.nix:1:1: + 1| builtins.substring 0 3 { } + | ^ + 2| + + … while evaluating the third argument (the string) passed to builtins.substring + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-substring-3.nix b/tests/functional/lang/eval-fail-substring-3.nix new file mode 100644 index 000000000000..f52921e48103 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-3.nix @@ -0,0 +1 @@ +builtins.substring 0 3 { } diff --git a/tests/functional/lang/eval-fail-substring-4.err.exp b/tests/functional/lang/eval-fail-substring-4.err.exp new file mode 100644 index 000000000000..cdaa11b20227 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-4.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-4.nix:1:1: + 1| builtins.substring (-3) 3 "sometext" + | ^ + 2| + + error: negative start position in 'substring' diff --git a/tests/functional/lang/eval-fail-substring-4.nix b/tests/functional/lang/eval-fail-substring-4.nix new file mode 100644 index 000000000000..82813b2c5838 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-4.nix @@ -0,0 +1 @@ +builtins.substring (-3) 3 "sometext" diff --git a/tests/functional/lang/eval-fail-tail-1.err.exp b/tests/functional/lang/eval-fail-tail-1.err.exp new file mode 100644 index 000000000000..2473767896ff --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'tail' builtin + at /pwd/lang/eval-fail-tail-1.nix:1:1: + 1| builtins.tail 1 + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.tail' + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-tail-1.nix b/tests/functional/lang/eval-fail-tail-1.nix new file mode 100644 index 000000000000..229ab7c8d7a1 --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-1.nix @@ -0,0 +1 @@ +builtins.tail 1 diff --git a/tests/functional/lang/eval-fail-tail-2.err.exp b/tests/functional/lang/eval-fail-tail-2.err.exp new file mode 100644 index 000000000000..63abbf2cba3d --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'tail' builtin + at /pwd/lang/eval-fail-tail-2.nix:1:1: + 1| builtins.tail [ ] + | ^ + 2| + + error: 'builtins.tail' called on an empty list diff --git a/tests/functional/lang/eval-fail-tail-2.nix b/tests/functional/lang/eval-fail-tail-2.nix new file mode 100644 index 000000000000..3bd318d1c33f --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-2.nix @@ -0,0 +1 @@ +builtins.tail [ ] diff --git a/tests/functional/lang/eval-fail-toPath-1.err.exp b/tests/functional/lang/eval-fail-toPath-1.err.exp new file mode 100644 index 000000000000..6e977c6a2bab --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'toPath' builtin + at /pwd/lang/eval-fail-toPath-1.nix:1:1: + 1| builtins.toPath [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-toPath-1.nix b/tests/functional/lang/eval-fail-toPath-1.nix new file mode 100644 index 000000000000..cd96c880b803 --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-1.nix @@ -0,0 +1 @@ +builtins.toPath [ ] diff --git a/tests/functional/lang/eval-fail-toPath-2.err.exp b/tests/functional/lang/eval-fail-toPath-2.err.exp new file mode 100644 index 000000000000..f7d421b76d8d --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'toPath' builtin + at /pwd/lang/eval-fail-toPath-2.nix:1:1: + 1| builtins.toPath "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + error: string 'foo' doesn't represent an absolute path diff --git a/tests/functional/lang/eval-fail-toPath-2.nix b/tests/functional/lang/eval-fail-toPath-2.nix new file mode 100644 index 000000000000..4bcb7fc14778 --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-2.nix @@ -0,0 +1 @@ +builtins.toPath "foo" diff --git a/tests/functional/lang/eval-fail-toString-1.err.exp b/tests/functional/lang/eval-fail-toString-1.err.exp new file mode 100644 index 000000000000..d73b9ad84c50 --- /dev/null +++ b/tests/functional/lang/eval-fail-toString-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'toString' builtin + at /pwd/lang/eval-fail-toString-1.nix:1:1: + 1| builtins.toString { a = 1; } + | ^ + 2| + + … while evaluating the first argument passed to builtins.toString + + error: cannot coerce a set to a string: { a = 1; } diff --git a/tests/functional/lang/eval-fail-toString-1.nix b/tests/functional/lang/eval-fail-toString-1.nix new file mode 100644 index 000000000000..e4b2d8d7955b --- /dev/null +++ b/tests/functional/lang/eval-fail-toString-1.nix @@ -0,0 +1 @@ +builtins.toString { a = 1; } diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp new file mode 100644 index 000000000000..7aced51b97d0 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'zipAttrsWith' builtin + at /pwd/lang/eval-fail-zipAttrsWith-1.nix:1:1: + 1| builtins.zipAttrsWith [ ] [ 1 ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.zipAttrsWith + + error: expected a function but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-1.nix b/tests/functional/lang/eval-fail-zipAttrsWith-1.nix new file mode 100644 index 000000000000..1f44609cd2ae --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-1.nix @@ -0,0 +1 @@ +builtins.zipAttrsWith [ ] [ 1 ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp new file mode 100644 index 000000000000..0a27e2d0195e --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'zipAttrsWith' builtin + at /pwd/lang/eval-fail-zipAttrsWith-2.nix:1:1: + 1| builtins.zipAttrsWith (_: 1) [ 1 ] + | ^ + 2| + + … while evaluating a value of the list passed as second argument to builtins.zipAttrsWith + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-2.nix b/tests/functional/lang/eval-fail-zipAttrsWith-2.nix new file mode 100644 index 000000000000..331da91a535b --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-2.nix @@ -0,0 +1 @@ +builtins.zipAttrsWith (_: 1) [ 1 ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp new file mode 100644 index 000000000000..780643b9df28 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the attribute 'foo' + + error: attempt to call something which is not a function but an integer: 1 + at /pwd/lang/eval-fail-zipAttrsWith-3.nix:1:24: + 1| builtins.zipAttrsWith (_: 1) [ { foo = 1; } ] + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-3.nix b/tests/functional/lang/eval-fail-zipAttrsWith-3.nix new file mode 100644 index 000000000000..6ac25c011010 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-3.nix @@ -0,0 +1 @@ +builtins.zipAttrsWith (_: 1) [ { foo = 1; } ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp new file mode 100644 index 000000000000..034d84254198 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the attribute 'foo' + + … from call site + at /pwd/lang/eval-fail-zipAttrsWith-4.nix:1:24: + 1| builtins.zipAttrsWith (a: b: a + b) [ + | ^ + 2| { foo = 1; } + + … while calling anonymous lambda + at /pwd/lang/eval-fail-zipAttrsWith-4.nix:1:27: + 1| builtins.zipAttrsWith (a: b: a + b) [ + | ^ + 2| { foo = 1; } + + … while evaluating a path segment + at /pwd/lang/eval-fail-zipAttrsWith-4.nix:1:34: + 1| builtins.zipAttrsWith (a: b: a + b) [ + | ^ + 2| { foo = 1; } + + error: cannot coerce a list to a string: [ 1 2 ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-4.nix b/tests/functional/lang/eval-fail-zipAttrsWith-4.nix new file mode 100644 index 000000000000..dbe63d0b564d --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-4.nix @@ -0,0 +1,4 @@ +builtins.zipAttrsWith (a: b: a + b) [ + { foo = 1; } + { foo = 2; } +] From 940825bce6d2496e58e6f892125ad02038267ccd Mon Sep 17 00:00:00 2001 From: Artemis Tosini Date: Thu, 30 Apr 2026 12:34:46 -0400 Subject: [PATCH 330/555] Filter systemd socket activation sockets by name When available, use the `LISTEN_FDNAMES` environment variable provided by systemd to listen only to the correct sockets in `serveUnixSocket`. Filtering makes it possible to differentiate different activation sockets connecting to the same daemon process. --- src/libcmd/include/nix/cmd/unix-socket-server.hh | 7 +++++++ src/libcmd/unix/unix-socket-server.cc | 8 ++++++++ src/nix/unix/daemon.cc | 1 + src/nix/unix/store-roots-daemon.cc | 1 + 4 files changed, 17 insertions(+) diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh index 7a0d9fa79317..544b988e2d95 100644 --- a/src/libcmd/include/nix/cmd/unix-socket-server.hh +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -55,6 +55,13 @@ struct ServeUnixSocketOptions mode_t socketMode = 0666; #ifndef _WIN32 + /** + * Name of the socket for socket activation, as included in `LISTEN_FDNAMES` + * Ordinarily the name of the socket unit, e.g. `nix-daemon.socket` + * If this field is empty, no name filtering will be performed. + */ + std::string activationName = ""; + /** * Additional file descriptor to poll. Useful for doing a self-pipe trick * https://cr.yp.to/docs/selfpipe.html. diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index 5d1fba462207..1794148b6ff9 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -5,6 +5,7 @@ #include "nix/util/file-system.hh" #include "nix/util/logging.hh" #include "nix/util/signals.hh" +#include "nix/util/strings.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/util.hh" @@ -65,9 +66,16 @@ PeerInfo getPeerInfo(Descriptor remote) if (listenFds) { if (getEnv("LISTEN_PID") != std::to_string(getpid())) throw Error("unexpected systemd environment variables"); + + auto fdNames = tokenizeString>(getEnv("LISTEN_FDNAMES").value_or(""), ":"); auto count = string2Int(*listenFds); assert(count); for (unsigned int i = 0; i < count; ++i) { + // Not all implementations of LISTEN_FDS will implement names, + // listen anyway if we do not have enough names + if (i < fdNames.size() && options.activationName != "" && fdNames[i] != options.activationName) + continue; + AutoCloseFD fdSocket(SD_LISTEN_FDS_START + i); closeOnExec(fdSocket.get()); listeningSockets.push_back(std::move(fdSocket)); diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 05e47f79c36b..c50c9f594228 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -285,6 +285,7 @@ static void daemonLoop( { .socketPath = std::move(socketPath), .socketMode = 0666, + .activationName = "nix-daemon.socket", .auxiliaryFd = sigChldPipe.pipe.readSide.get(), .onAuxiliaryFdPollin = []() { diff --git a/src/nix/unix/store-roots-daemon.cc b/src/nix/unix/store-roots-daemon.cc index 85b0a67a9156..b803c8cea6cf 100644 --- a/src/nix/unix/store-roots-daemon.cc +++ b/src/nix/unix/store-roots-daemon.cc @@ -44,6 +44,7 @@ struct CmdRootsDaemon : StoreConfigCommand { .socketPath = gcSocketPath, .socketMode = 0666, + .activationName = "nix-roots-daemon.socket", }, [&](AutoCloseFD remote, std::function closeListeners) { std::thread([&, remote = std::move(remote)]() mutable { From a79b0f4dc1521d27ea5c9829ff422914843775ee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2026 20:06:48 +0200 Subject: [PATCH 331/555] LocalStore::addToStore(): Handle negative path info cache entry If we have a negative path info cache entry for the path being added (e.g. due to a prior call to maybeQueryPathInfo()), then we need to use isValidPathUncached(), otherwise addToStore() will fail. --- src/libstore/local-store.cc | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 53d7456f94f6..0bc7b6e1b6c8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1013,7 +1013,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ - pathInfoCache->lock()->erase(path); + invalidatePathInfoCacheFor(path); } const PublicKeys & LocalStore::getPublicKeys() @@ -1049,17 +1049,18 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF auto realPath = toRealPath(info.path); /* Lock the output path. But don't lock if we're being called - from a build hook (whose parent process already acquired a - lock on this path). */ + from a build hook (whose parent process already acquired a + lock on this path). */ if (!locksHeld.count(printStorePath(info.path))) outputLock.lockPaths({realPath}); - if (repair || !isValidPath(info.path)) { + /* The path may have been created by another process in the meantime, so check again. */ + if (repair || !isValidPathUncached(info.path)) { deletePath(realPath); /* While restoring the path from the NAR, compute the hash - of the NAR. */ + of the NAR. */ HashSink hashSink(HashAlgorithm::SHA256); TeeSource wrapperSource{source, hashSink}; @@ -1129,7 +1130,9 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF } registerValidPath(info); - } + } else + // We may have a negative cache entry for this path, so get rid of it. + invalidatePathInfoCacheFor(info.path); outputLock.setDeletion(true); } @@ -1246,7 +1249,8 @@ StorePath LocalStore::addToStoreFromDump( PathLocks outputLock({realPath}); - if (repair || !isValidPath(dstPath)) { + /* The path may have been created by another process in the meantime, so check again. */ + if (repair || !isValidPathUncached(dstPath)) { deletePath(realPath); @@ -1293,7 +1297,9 @@ StorePath LocalStore::addToStoreFromDump( auto info = ValidPathInfo::makeFromCA(*this, name, std::move(desc), narHash.hash); info.narSize = narHash.numBytesDigested; registerValidPath(info); - } + } else + // We may have a negative cache entry for this path, so get rid of it. + invalidatePathInfoCacheFor(dstPath); outputLock.setDeletion(true); } From 026e930912d459290fac3eabbf11b6eb31ce681d Mon Sep 17 00:00:00 2001 From: ryota2357 Date: Sat, 18 Oct 2025 11:55:40 +0900 Subject: [PATCH 332/555] nix-profile{,-daemon}.fish: set NIX_PROFILES to use $NIX_LINK --- scripts/nix-profile-daemon.fish.in | 2 +- scripts/nix-profile.fish.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 1a20dffd2459..93cb3c45a55b 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -53,7 +53,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index abf716cec6fc..201a56438950 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -58,7 +58,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" From 0072b3271f6f23429805a6f7ea878a9f1b682cb6 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Fri, 1 May 2026 14:16:00 -0400 Subject: [PATCH 333/555] Ignore keep-{outputs,derivations} for delete `keep-outputs` and `keep-derivations` were not meant to affect delete operations. This change makes them only read when using the `WholeStore` variant type. Signed-off-by: Lisanna Dettwyler --- doc/manual/rl-next/delete-keep.md | 8 +++ src/libstore/gc.cc | 65 ++++++++++--------- .../include/nix/store/local-settings.hh | 6 ++ tests/functional/delete-no-keep.sh | 32 +++++++++ tests/functional/meson.build | 1 + 5 files changed, 82 insertions(+), 30 deletions(-) create mode 100644 doc/manual/rl-next/delete-keep.md create mode 100755 tests/functional/delete-no-keep.sh diff --git a/doc/manual/rl-next/delete-keep.md b/doc/manual/rl-next/delete-keep.md new file mode 100644 index 000000000000..0332e0e3933e --- /dev/null +++ b/doc/manual/rl-next/delete-keep.md @@ -0,0 +1,8 @@ +--- +synopsis: "Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands" +prs: [15776] +--- + +Setting `keep-derivations = true` and trying to delete a derivation with realised outputs would previously fail. +Same with `keep-outputs = true` and trying to delete an output that still has derivers. +These options no longer affect the deletion commands, and are now documented as such. diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 701fc66e69d0..3b9e8dbf5cd2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -357,8 +357,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) const auto & gcSettings = config->getLocalSettings().getGCSettings(); bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; - bool keepOutputs = gcSettings.keepOutputs; - bool keepDerivations = gcSettings.keepDerivations; boost::unordered_flat_set> roots, dead, alive; @@ -382,16 +380,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) std::condition_variable wakeup; - /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `keep-outputs' or `keep-derivations' are true - (the garbage collector will recurse into deleting the outputs - or derivers, respectively, even if they aren't in the - pathsToDelete). So disable them. */ - if (std::holds_alternative(options.pathsToDelete) && options.ignoreLiveness) { - keepOutputs = false; - keepDerivations = false; - } - if (shouldDelete) deletePath(reservedPath); @@ -625,12 +613,23 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) alive.insert(start); try { StorePathSet closure; + bool includeOutputs = false; + bool includeDerivers = false; + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + includeOutputs = gcSettings.keepOutputs; + includeDerivers = gcSettings.keepDerivations; + }, + [](const StorePathSet &) {}, + }, + options.pathsToDelete); computeFSClosure( *path, closure, /* flipDirection */ false, - keepOutputs, - keepDerivations); + includeOutputs, + includeDerivers); for (auto & p : closure) alive.insert(p); } catch (InvalidPath &) { @@ -675,22 +674,28 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) for (auto & p : i->second) enqueue(p); - /* If keep-derivations is set and this is a derivation, then we only want to delete this derivation if - * we can also delete all its outputs, so visit the derivation outputs. */ - if (keepDerivations && path->isDerivation()) { - for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) - if (maybeOutPath && isValidPath(*maybeOutPath) - && queryPathInfo(*maybeOutPath)->deriver == *path) - enqueue(*maybeOutPath); - } - - /* If keep-outputs is set, we only want to delete this path if we - * can also delete its derivers, so visit the derivers. */ - if (keepOutputs) { - auto derivers = queryValidDerivers(*path); - for (auto & i : derivers) - enqueue(i); - } + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + /* If keep-derivations is set and this is a derivation, then we only want to delete this + * derivation if we can also delete all its outputs, so visit the derivation outputs. */ + if (gcSettings.keepDerivations && path->isDerivation()) + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) + if (maybeOutPath && isValidPath(*maybeOutPath) + && queryPathInfo(*maybeOutPath)->deriver == path) + enqueue(*maybeOutPath); + + /* If keep-outputs is set, we only want to delete this path if we + * can also delete its derivers, so visit the derivers. */ + if (gcSettings.keepOutputs) { + auto derivers = queryValidDerivers(*path); + for (auto & i : derivers) + enqueue(i); + } + }, + [](const StorePathSet &) {}, + }, + options.pathsToDelete); } } for (auto & path : topoSortPaths(visited)) { diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index 4fe28818d2b8..7381b5b8e766 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -61,6 +61,9 @@ struct GCSettings : public virtual Config collector still deletes store paths that are used only at build time (e.g., the C compiler, or source tarballs downloaded from the network). To prevent it from doing so, set this option to `true`. + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-outputs"}, }; @@ -80,6 +83,9 @@ struct GCSettings : public virtual Config store path was built), so by default this option is on. Turn it off to save a bit of disk space (or a lot if `keep-outputs` is also turned on). + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-derivations"}, }; diff --git a/tests/functional/delete-no-keep.sh b/tests/functional/delete-no-keep.sh new file mode 100755 index 000000000000..8ac64fd44493 --- /dev/null +++ b/tests/functional/delete-no-keep.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +deleteNoKeep() { + keep="$1" + + clearStore + drvPath=$(nix-instantiate simple.nix) + outPath=$(nix build -f simple.nix --no-link --print-out-paths) + + { + echo "keep-outputs = false" + echo "keep-derivations = false" + echo "keep-$keep = true" + } >> "$test_nix_conf" + + if [[ "$keep" = "outputs" ]]; then + nix store delete "$outPath" + [[ ! -e "$outPath" ]] || fail "$outPath should have been deleted" + else + nix store delete "$drvPath" + [[ ! -e "$drvPath" ]] || fail "$drvPath should have been deleted" + fi +} + +if isDaemonNewer "2.35pre"; then + deleteNoKeep outputs + deleteNoKeep derivations +fi diff --git a/tests/functional/meson.build b/tests/functional/meson.build index e2668c716ed8..ecb9ef87f663 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -179,6 +179,7 @@ suites = [ 'help.sh', 'symlinks.sh', 'external-builders.sh', + 'delete-no-keep.sh', ], 'workdir' : meson.current_source_dir(), }, From 6bf83e259a83d0324584977db81d0b66d9e649af Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 22 Apr 2026 15:38:15 -0400 Subject: [PATCH 334/555] Add `--also-referrers` to `nix store delete` This makes collecting garbage within a closure more ergonomic. If you build `hello` from scratch, it'll also build `glibc` and `glibc-static`. `glibc-static` depends on `glibc`, but is not part of `hello`'s closure. This normally prevents deletion of `glibc`, but using this flag allows `glibc-static` to also be deleted as long as it is dead. Signed-off-by: Lisanna Dettwyler --- doc/manual/rl-next/closure-gc.md | 7 ++-- src/libstore/daemon.cc | 12 ++++-- src/libstore/gc.cc | 37 +++++++++++------- src/libstore/include/nix/store/gc-store.hh | 12 +++++- .../include/nix/store/worker-protocol.hh | 8 +++- src/libstore/remote-store.cc | 12 ++++-- src/libstore/worker-protocol.cc | 22 +++++++++-- src/nix/nix-store/nix-store.cc | 4 +- src/nix/store-delete.cc | 12 +++++- tests/functional/dependencies2.nix | 39 +++++++++++++++++++ tests/functional/gc-closure.sh | 34 ++++++++++++---- 11 files changed, 159 insertions(+), 40 deletions(-) create mode 100644 tests/functional/dependencies2.nix diff --git a/doc/manual/rl-next/closure-gc.md b/doc/manual/rl-next/closure-gc.md index fc24d5e08b9b..945a78dc9f8c 100644 --- a/doc/manual/rl-next/closure-gc.md +++ b/doc/manual/rl-next/closure-gc.md @@ -1,9 +1,8 @@ --- synopsis: "Added `--skip-alive` option to `nix store delete` for collecting garbage within a closure" issues: 7239 -prs: 15236 +prs: [15236, 15727] --- -`nix store delete --recursive --skip-alive` can be used to collect garbage -within a closure, in which case it will only collect the dead paths that are -part of the closure of its arguments. +`nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. +The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 52e1c121c3e7..8a02ea6ae149 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -746,14 +746,17 @@ static void performOp( case WorkerProto::Op::CollectGarbage: { GCOptions options; options.action = WorkerProto::Serialise::read(*store, rconn); - if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); } else { auto paths = WorkerProto::Serialise::read(*store, rconn); if (options.action != GCAction::gcDeleteSpecific && paths.empty()) options.pathsToDelete = GCOptions::WholeStore{}; else - options.pathsToDelete = paths; + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = paths, + .deleteReferrers = false, + }; } conn.from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields @@ -761,8 +764,9 @@ static void performOp( readInt(conn.from); readInt(conn.from); - if (options.action == GCAction::gcDeleteDead && std::holds_alternative(options.pathsToDelete) - && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + if (options.action == GCAction::gcDeleteDead + && std::holds_alternative(options.pathsToDelete) + && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { throw Error( "Garbage collecting specific paths requested but it is not supported by the negotiated protocol"); } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3b9e8dbf5cd2..645c4c6f97b1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -361,8 +361,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) boost::unordered_flat_set> roots, dead, alive; /* Return early if nothing to delete */ - if (std::holds_alternative(options.pathsToDelete) - && std::get(options.pathsToDelete).empty()) + if (std::visit( + overloaded{ + [](const GCOptions::SpecificPaths & pathsToDelete) { return pathsToDelete.paths.empty(); }, + [](const GCOptions::WholeStore & _) { return false; }}, + options.pathsToDelete)) return; struct Shared @@ -621,7 +624,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) includeOutputs = gcSettings.keepOutputs; includeDerivers = gcSettings.keepDerivations; }, - [](const StorePathSet &) {}, + [](const GCOptions::SpecificPaths &) {}, }, options.pathsToDelete); computeFSClosure( @@ -642,14 +645,22 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) return markAlive(); } - if (std::holds_alternative(options.pathsToDelete) - && !std::get(options.pathsToDelete).contains(*path)) { - debug( - "cannot delete '%s' because '%s' is not in the specified paths to delete", - printStorePath(start), - printStorePath(*path)); + if (std::visit( + overloaded{ + [&](const GCOptions::SpecificPaths & pathsToDelete) { + if (!pathsToDelete.deleteReferrers && !pathsToDelete.paths.contains(*path)) { + debug( + "cannot delete '%s' because '%s' is not in the specified paths to delete", + printStorePath(start), + printStorePath(*path)); + return true; + } + return false; + }, + [](const GCOptions::WholeStore & _) { return false; }, + }, + options.pathsToDelete)) return; - } { auto hashPart = path->hashPart(); @@ -693,7 +704,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) enqueue(i); } }, - [](const StorePathSet &) {}, + [](const GCOptions::SpecificPaths &) {}, }, options.pathsToDelete); } @@ -719,7 +730,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Either delete all garbage paths, or just the specified paths. */ std::visit( overloaded{ - [&](const StorePathSet & paths) { + [&](const GCOptions::SpecificPaths & pathsToDelete) { switch (options.action) { case GCOptions::gcDeleteDead: printInfo("deleting garbage within specified paths..."); @@ -732,7 +743,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printInfo("determining live/dead paths..."); } - for (auto & i : paths) { + for (auto & i : pathsToDelete.paths) { maybeDeleteReferrersClosure(i); if (options.action == GCOptions::gcDeleteSpecific && !dead.contains(i)) diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 015478e79fd3..0a0335016b74 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -42,6 +42,16 @@ struct GCOptions struct WholeStore {}; + struct SpecificPaths + { + StorePathSet paths; + + /** + * Allow dead referrers of candidate paths to also be deleted. + */ + bool deleteReferrers = false; + }; + GCAction action{gcDeleteDead}; /** @@ -55,7 +65,7 @@ struct GCOptions /** * The paths from which to delete. */ - using GCPaths = std::variant; + using GCPaths = std::variant; GCPaths pathsToDelete; /** diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index ea1cbb502d7a..64cab1494a76 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -126,9 +126,9 @@ struct WorkerProto static constexpr std::string_view featureRealisationWithPath = "realisation-with-path-not-hash"; /** - * Feature for garbage collecting a specific set of paths. + * Feature for garbage collecting a specific set of paths and deleting referrers. */ - static constexpr std::string_view featureDeleteDeadSpecific = "delete-dead-specific"; + static constexpr std::string_view featureDeleteDeadSpecificReferrers = "delete-dead-specific-referrers"; /** * A unidirectional read connection, to be used by the read half of the @@ -345,6 +345,10 @@ template<> DECLARE_WORKER_SERIALISER(std::optional); template<> DECLARE_WORKER_SERIALISER(WorkerProto::ClientHandshakeInfo); + +template<> +DECLARE_WORKER_SERIALISER(GCOptions::SpecificPaths); + template<> DECLARE_WORKER_SERIALISER(GCOptions::GCPaths); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a2fb3a10d251..2c1ea0c63886 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -686,18 +686,23 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { auto conn(getConnection()); - if (conn->protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + bool supportsDeleteSpecificReferrers = + conn->protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers); + + if (supportsDeleteSpecificReferrers) { conn->to << WorkerProto::Op::CollectGarbage; WorkerProto::write(*this, *conn, options.action); WorkerProto::write(*this, *conn, options.pathsToDelete); } else { auto paths = std::visit( overloaded{ - [&](const StorePathSet & paths) { + [&](const GCOptions::SpecificPaths & paths) { if (options.action != GCOptions::gcDeleteSpecific) throw Error( "Your daemon version is too old to support garbage collecting a specific set of paths"); - return paths; + if (paths.deleteReferrers) + throw Error("Your daemon version is too old to support deleting referrers."); + return paths.paths; }, [](const GCOptions::WholeStore & _) { return StorePathSet{}; }, }, @@ -706,6 +711,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) WorkerProto::write(*this, *conn, options.action); WorkerProto::write(*this, *conn, paths); } + conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 31373a128f96..37aa4e31bec2 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -28,7 +28,7 @@ const WorkerProto::Version WorkerProto::latest = { std::string{ WorkerProto::featureRealisationWithPath, }, - std::string{WorkerProto::featureDeleteDeadSpecific}, + std::string{WorkerProto::featureDeleteDeadSpecificReferrers}, }, }; @@ -533,13 +533,29 @@ void WorkerProto::Serialise::write(const StoreDirConfig & store, Wr WorkerProto::write(store, conn, static_cast(info)); } +GCOptions::SpecificPaths +WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + GCOptions::SpecificPaths paths; + paths.paths = WorkerProto::Serialise::read(store, conn); + conn.from >> paths.deleteReferrers; + return paths; +} + +void WorkerProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const GCOptions::SpecificPaths & paths) +{ + WorkerProto::write(store, conn, paths.paths); + conn.to << paths.deleteReferrers; +} + GCOptions::GCPaths WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { uint8_t wholeStore; conn.from >> wholeStore; switch (wholeStore) { case 0: - return WorkerProto::Serialise::read(store, conn); + return WorkerProto::Serialise::read(store, conn); case 1: return GCOptions::WholeStore{}; default: @@ -552,7 +568,7 @@ void WorkerProto::Serialise::write( { std::visit( overloaded{ - [&](const StorePathSet paths) { + [&](const GCOptions::SpecificPaths paths) { conn.to << uint8_t{0}; WorkerProto::write(store, conn, paths); }, diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 15a4e878f5ac..63952ba63a17 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -727,7 +727,9 @@ static void opDelete(Strings opFlags, Strings opArgs) StorePathSet paths; for (auto & i : opArgs) paths.insert(store->followLinksToStorePath(i)); - options.pathsToDelete = std::move(paths); + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = std::move(paths), + }; auto & gcStore = require(*store); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index a1a387898491..c2f649ff568e 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -9,6 +9,7 @@ namespace nix { struct CmdStoreDelete : StorePathsCommand { GCOptions options{.action = GCOptions::gcDeleteSpecific}; + bool deleteReferrers = false; CmdStoreDelete() { @@ -25,6 +26,12 @@ struct CmdStoreDelete : StorePathsCommand "Do not emit errors when attempting to delete something that is still alive, useful with --recursive.", .handler = {&options.action, GCOptions::gcDeleteDead}, }); + + addFlag({ + .longName = "also-referrers", + .description = "Also allow deletion of any referrers of the specified paths.", + .handler = {&deleteReferrers, true}, + }); } std::string description() override @@ -46,7 +53,10 @@ struct CmdStoreDelete : StorePathsCommand StorePathSet paths; for (auto & path : storePaths) paths.insert(path); - options.pathsToDelete = std::move(paths); + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = std::move(paths), + .deleteReferrers = deleteReferrers, + }; GCResults results; Finally printer([&] { printFreed(false, results); }); diff --git a/tests/functional/dependencies2.nix b/tests/functional/dependencies2.nix new file mode 100644 index 000000000000..6300f0ac41fe --- /dev/null +++ b/tests/functional/dependencies2.nix @@ -0,0 +1,39 @@ +with import ./config.nix; + +let + + input0 = mkDerivation { + name = "dependencies-input-0"; + buildCommand = "mkdir $out; echo foo > $out/bar"; + }; + + input1 = mkDerivation { + name = "dependencies-input-1"; + buildCommand = "mkdir $out; echo FOO > $out/foo"; + }; + + input2 = mkDerivation { + name = "dependencies-input-2"; + buildCommand = '' + mkdir $out + echo BAR > $out/bar + echo ${input0} > $out/input0 + echo "$out" > $out2 + ''; + outputs = [ + "out" + "out2" + ]; + }; + +in +mkDerivation { + name = "dependencies-top"; + builder = ./dependencies.builder0.sh + "/FOOBAR/../."; + input1 = input1 + "/."; + input2 = "${input2}/."; + input1_drv = input1; + input2_drv = input2; + input0_drv = input0; + meta.description = "Random test package"; +} diff --git a/tests/functional/gc-closure.sh b/tests/functional/gc-closure.sh index 03e2a0fc25d6..4b811e779717 100755 --- a/tests/functional/gc-closure.sh +++ b/tests/functional/gc-closure.sh @@ -5,25 +5,43 @@ source common.sh TODO_NixOS nix_gc_closure() { + ensureNoDeleteReferrer="${1}" + extraArg="${2:-""}" clearStore - nix build -f dependencies.nix input0_drv --out-link "$TEST_ROOT/gc-root" + nix build -f dependencies2.nix input0_drv --out-link "$TEST_ROOT/gc-root" input0=$(realpath "$TEST_ROOT/gc-root") - input1=$(nix build -f dependencies.nix input1_drv --no-link --print-out-paths) - input2=$(nix build -f dependencies.nix input2_drv --no-link --print-out-paths) - top=$(nix build -f dependencies.nix --no-link --print-out-paths) - somthing_else=$(nix store add-path ./dependencies.nix) + input1=$(nix build -f dependencies2.nix input1_drv --no-link --print-out-paths) + input2=$(nix build -f dependencies2.nix input2_drv --no-link --print-out-paths) + input2_out=$(printf "%s" "$input2" | head -n1) + input2_out2=$(printf "%s" "$input2" | tail -n1) + top=$(nix build -f dependencies2.nix --no-link --print-out-paths) + somthing_else=$(nix store add-path ./dependencies2.nix) if isDaemonNewer "2.35pre"; then + if [[ "$extraArg" != "--also-referrers" ]] && ! "$ensureNoDeleteReferrer"; then + nix store delete "$input2_out2" + fi # Check that nix store delete --recursive --skip-alive is best-effort (doesn't fail when some paths in the closure are alive) - nix store delete --recursive --skip-alive "$top" + # shellcheck disable=SC2086 # we want $extraArg to expand to nothing if unset + nix store delete --recursive --skip-alive $extraArg "$top" [[ ! -e "$top" ]] || fail "top should have been deleted" [[ -e "$input0" ]] || fail "input0 is a gc root, shouldn't have been deleted" - [[ ! -e "$input2" ]] || fail "input2 is not a gc root and is part of top's closure, it should have been deleted" [[ -e "$input1" ]] || fail "input1 is not in the closure of top, it shouldn't have been deleted" [[ -e "$somthing_else" ]] || fail "somthing_else is not in the closure of top, it shouldn't have been deleted" + if [[ "$extraArg" = "--also-referrers" ]]; then + [[ ! -e "$input2_out" ]] || fail "input2_out is part of top's closure and we can delete dead referrers, it should have been deleted" + elif "$ensureNoDeleteReferrer"; then + [[ -e "$input2_out" ]] || fail "input2_out is part of top's closure but we can't delete dead referrers, it shouldn't have been deleted" + else + [[ ! -e "$input2_out" ]] || fail "input2_out is not a gc root, is part of top's closure, and has no referrers, it should have been deleted" + fi + elif [[ "$extraArg" = "--also-referrers" ]]; then + expectStderr 1 nix store delete --recursive --also-referrers "$top" | grepQuiet "Your daemon version is too old to support deleting referrers" else expectStderr 1 nix store delete --recursive --skip-alive "$top" | grepQuiet "Your daemon version is too old to support garbage collecting a specific set of paths" fi } -nix_gc_closure +nix_gc_closure false +nix_gc_closure true +nix_gc_closure false --also-referrers From e6d05b4bab021a695efc0f26b706cf6df5ce182d Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 22 Apr 2026 15:50:10 -0400 Subject: [PATCH 335/555] Add some examples for GCing a closure Signed-off-by: Lisanna Dettwyler --- src/nix/store-delete.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md index 026dccd0f19a..9b6fd9701d3e 100644 --- a/src/nix/store-delete.md +++ b/src/nix/store-delete.md @@ -8,6 +8,18 @@ R""( # nix store delete /nix/store/fdhrijyv3670djsgprx596nn89iwlj2s-hello-2.10 ``` +* Garbage collect a closure: + + ```console + # nix store delete --recursive --skip-alive nixpkgs#hello + ``` + +* Garbage collect a closure including dead referrers of the closure: + + ```console + # nix store delete --recursive --skip-alive --also-referrers nixpkgs#hello + ``` + # Description This command deletes the store paths specified by [*installables*](./nix.md#installables), From 66d9c62cde51b9c2e75f64c57a81b880467ef0fb Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 03:16:09 +0300 Subject: [PATCH 336/555] Fix observe_string_cb to not assume a NUL terminated string With std::string_view::data() we really can't assume that the string ends up being NUL terminated. We already pass the length when needed, but we just don't use it in the test harness. All existing API consumers really should not be relying on this being the case in all situations, but this wasn't properly documented anywhere (though it is rather self-evident from the size parameter passed to the callback). --- src/libstore-tests/nix_api_store.cc | 4 ++-- src/libutil-c/nix_api_util.h | 1 + src/libutil-test-support/string_callback.cc | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 60626869a919..0162684daf4b 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -77,8 +77,8 @@ TEST_F(nix_api_store_test, ReturnsValidStorePath) { StorePath * result = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); ASSERT_NE(result, nullptr); - ASSERT_STREQ("name", result->path.name().data()); - ASSERT_STREQ(PATH_SUFFIX.substr(1).c_str(), result->path.to_string().data()); + ASSERT_EQ("name", result->path.name()); + ASSERT_EQ(PATH_SUFFIX.substr(1), result->path.to_string()); nix_store_path_free(result); } diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index 66ea17522213..b48d9166d4be 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -163,6 +163,7 @@ typedef struct nix_c_context nix_c_context; * @brief Called to get the value of a string owned by Nix. * * The `start` data is borrowed and the function must not assume that the buffer persists after it returns. + * @warning Don't assume that the string is NUL-terminated. * * @param[in] start the string to copy. * @param[in] n the string length. diff --git a/src/libutil-test-support/string_callback.cc b/src/libutil-test-support/string_callback.cc index b64389e4adbd..70ff7ca7f4ef 100644 --- a/src/libutil-test-support/string_callback.cc +++ b/src/libutil-test-support/string_callback.cc @@ -5,7 +5,7 @@ namespace nix::testing { void observe_string_cb(const char * start, unsigned int n, void * user_data) { auto user_data_casted = reinterpret_cast(user_data); - *user_data_casted = std::string(start); + *user_data_casted = std::string(start, n); } } // namespace nix::testing From 5f90b0c5d5944638ead691ccc9437614ecce51f1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 03:53:20 +0300 Subject: [PATCH 337/555] Don't assume NUL terminated std::string_view in SQLiteStmt::Use::operator() std::string_view doesn't have to be NUL terminated - we were just getting lucky because everything that we passed in happened to be NUL terminated. This bug has existed since 7a9687ba30d579bc51e0aaf3193e0ab8d86400d2. --- src/libstore/sqlite.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index c65fc240c66f..4f39b61c5da7 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -177,7 +177,7 @@ SQLiteStmt::Use::~Use() SQLiteStmt::Use & SQLiteStmt::Use::operator()(std::string_view value, bool notNull) { if (notNull) { - if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmt, curArg++, value.data(), value.size(), SQLITE_TRANSIENT) != SQLITE_OK) SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); From 77ecdaf6d184e553492574d10c7648f3b9332779 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Sun, 26 Apr 2026 18:47:53 +0200 Subject: [PATCH 338/555] Add unit test to reproduce #15713. --- src/libstore-tests/meson.build | 1 + src/libstore-tests/outputs-query.cc | 114 ++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 src/libstore-tests/outputs-query.cc diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 78a1cc4a12dd..ab03bb2d2a0e 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -77,6 +77,7 @@ sources = files( 'nar-info-disk-cache.cc', 'nar-info.cc', 'nix_api_store.cc', + 'outputs-query.cc', 'outputs-spec.cc', 'path-info.cc', 'path.cc', diff --git a/src/libstore-tests/outputs-query.cc b/src/libstore-tests/outputs-query.cc new file mode 100644 index 000000000000..4be25e2ad3b8 --- /dev/null +++ b/src/libstore-tests/outputs-query.cc @@ -0,0 +1,114 @@ +// Regression tests for the functions in outputs-query.cc +// +// See https://github.com/NixOS/nix/issues/15713 + +#include + +#include "nix/store/outputs-query.hh" +#include "nix/store/derivations.hh" +#include "nix/store/dummy-store-impl.hh" +#include "nix/store/realisation.hh" +#include "nix/store/tests/libstore.hh" + +namespace nix { + +class OutputsQueryTest : public ::testing::Test +{ +public: + static void SetUpTestSuite() + { + initLibStore(false); + } + +protected: + EnableExperimentalFeature caFeature{"ca-derivations"}; + + ref store = [] { + auto cfg = make_ref(StoreReference::Params{}); + cfg->readOnly = false; + return cfg->openDummyStore(); + }(); + + static DerivationOutput caFloatingOutput() + { + return DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}; + } + + /** + * Build a simple floating CA derivation with a given name and no input + * derivations. + */ + Derivation makeLeafDrv(std::string name) + { + Derivation drv; + drv.name = std::move(name); + drv.platform = "x86_64-linux"; + drv.builder = "/bin/sh"; + drv.outputs = {{"out", caFloatingOutput()}}; + return drv; + } +}; + +/** + * Regression test for https://github.com/NixOS/nix/issues/15713 + * + * In a Fibonacci-style chain of floating CA derivations, the resolution + * algorithm used to call queryRealisation O(Fib(N)) times. + * This test verifies that memoization reduces this to O(N). + */ +TEST_F(OutputsQueryTest, fibonacciChainQueryCount) +{ + constexpr static size_t N = 10; + std::vector drvPaths; + + // d0, d1: leaf derivations + for (int i = 0; i < 2; ++i) { + drvPaths.push_back(store->writeDerivation(makeLeafDrv("d" + std::to_string(i)))); + } + + // d_i depends on d_{i-1} and d_{i-2} + for (size_t i = 2; i <= N; ++i) { + Derivation drv = makeLeafDrv("d" + std::to_string(i)); + drv.inputDrvs.map[drvPaths[i - 1]].value.insert("out"); + drv.inputDrvs.map[drvPaths[i - 2]].value.insert("out"); + drvPaths.push_back(store->writeDerivation(drv)); + } + + // Tracker for queryRealisation calls. + std::map callCounts; + std::map outPaths; + + QueryRealisationFun queryRealisation = [&](const DrvOutput & id) -> std::shared_ptr { + assert(id.outputName == "out"); + callCounts[id.drvPath]++; + + // Memoize mock output paths. + auto it = outPaths.find(id.drvPath); + if (it == outPaths.end()) { + auto hash = hashString(HashAlgorithm::SHA1, "mock-output-" + std::to_string(outPaths.size())); + it = outPaths.emplace(id.drvPath, StorePath(hash, "out")).first; + } + + return std::make_shared(UnkeyedRealisation{.outPath = it->second}); + }; + + auto result = deepQueryPartialDerivationOutput(*store, drvPaths[N], "out", nullptr, queryRealisation); + + ASSERT_TRUE(result); + + int totalCalls = 0; + for (auto & [path, count] : callCounts) { + totalCalls += count; + if (count > 1) + ADD_FAILURE() << "Derivation at " << store->printStorePath(path) << " was queried " << count + << " times (expected 1)"; + } + + // With full memoization (ResolveCache + RealisationCache), each derivation should be queried exactly once. + EXPECT_EQ(totalCalls, N + 1) << "queryRealisation called " << totalCalls << " times; expected exactly " << (N + 1); +} + +} // namespace nix From 6974f9e1257c9c2371ed3307fc4f5f748d2e5883 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Sat, 2 May 2026 13:42:19 -0400 Subject: [PATCH 339/555] Fix FreeBSD non-unity build Adds missing `` include. Signed-off-by: Lisanna Dettwyler --- src/libutil-tests/unix/unix-domain-socket.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil-tests/unix/unix-domain-socket.cc b/src/libutil-tests/unix/unix-domain-socket.cc index 439d3188ac2d..b5050e564585 100644 --- a/src/libutil-tests/unix/unix-domain-socket.cc +++ b/src/libutil-tests/unix/unix-domain-socket.cc @@ -6,6 +6,7 @@ #include #include +#include namespace nix { From 2c26a23a9a055df33850de3106488fa7e54d0c9d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 2 May 2026 13:58:44 -0400 Subject: [PATCH 340/555] Remove Perl bindings `nix-serve` is being deprecated, and so Hydra is the only active user of the perl bindings. As such, they have been moved to the hydra repo. Now that that is done, we can remove them from this repo. Delete `src/perl/` and all references to `nix-perl-bindings` across the build system, packaging, CI, and documentation. The `fetchers-substitute` NixOS test still uses `nix-serve`, which depends on Perl bindings, but now uses the older Nix from nixpkgs rather than this project's bindings. This is a stop-gap until we stop using `nix-serve` in that test (because it is deprecated) completely. --- ci/gha/tests/default.nix | 2 - doc/manual/source/development/debugging.md | 1 - flake.nix | 10 - meson.build | 5 - meson.options | 7 - .../clang-tidy/build_required_targets.py | 2 - .../common/clang-tidy/clean_compdb.py | 3 - packaging/components.nix | 2 - packaging/dev-shell.nix | 15 +- packaging/everything.nix | 20 +- packaging/hydra.nix | 12 - src/perl/.version | 1 - src/perl/.yath.rc.in | 2 - src/perl/MANIFEST | 7 - src/perl/lib/Nix/Config.pm.in | 24 - src/perl/lib/Nix/CopyClosure.pm | 61 --- src/perl/lib/Nix/Manifest.pm | 325 ------------- src/perl/lib/Nix/SSH.pm | 110 ----- src/perl/lib/Nix/Store.pm | 45 -- src/perl/lib/Nix/Store.xs | 430 ------------------ src/perl/lib/Nix/Utils.pm | 47 -- src/perl/lib/Nix/meson.build | 61 --- src/perl/meson.build | 196 -------- src/perl/meson.options | 30 -- src/perl/package.nix | 82 ---- src/perl/t/init.t | 13 - src/perl/t/meson.build | 15 - tests/nixos/fetchers-substitute.nix | 18 +- 28 files changed, 4 insertions(+), 1542 deletions(-) delete mode 120000 src/perl/.version delete mode 100644 src/perl/.yath.rc.in delete mode 100644 src/perl/MANIFEST delete mode 100644 src/perl/lib/Nix/Config.pm.in delete mode 100644 src/perl/lib/Nix/CopyClosure.pm delete mode 100644 src/perl/lib/Nix/Manifest.pm delete mode 100644 src/perl/lib/Nix/SSH.pm delete mode 100644 src/perl/lib/Nix/Store.pm delete mode 100644 src/perl/lib/Nix/Store.xs delete mode 100644 src/perl/lib/Nix/Utils.pm delete mode 100644 src/perl/lib/Nix/meson.build delete mode 100644 src/perl/meson.build delete mode 100644 src/perl/meson.options delete mode 100644 src/perl/package.nix delete mode 100644 src/perl/t/init.t delete mode 100644 src/perl/t/meson.build diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index 5e1b23c7a36b..8d2383a92fb9 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -57,8 +57,6 @@ rec { nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; }; mesonComponentOverrides = lib.composeManyExtensions componentOverrides; - # Unclear how to make Perl bindings work with a dynamically linked ASAN. - nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings; } ); diff --git a/doc/manual/source/development/debugging.md b/doc/manual/source/development/debugging.md index 6578632d991a..35e4c71ec388 100644 --- a/doc/manual/source/development/debugging.md +++ b/doc/manual/source/development/debugging.md @@ -26,7 +26,6 @@ or GCC. This is useful when debugging memory corruption issues. ```console [nix-shell]$ export mesonBuildType=debugoptimized [nix-shell]$ appendToVar mesonFlags "-Dlibexpr:gc=disabled" # Disable Boehm -[nix-shell]$ appendToVar mesonFlags "-Dbindings=false" # Disable nix-perl [nix-shell]$ appendToVar mesonFlags "-Db_sanitize=address,undefined" ``` diff --git a/flake.nix b/flake.nix index c9be5e1dd58c..8a2fe16c2770 100644 --- a/flake.nix +++ b/flake.nix @@ -328,12 +328,6 @@ // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; } - // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { - # Some perl dependencies are broken on i686-linux. - # Since the support is only best-effort there, disable the perl - # bindings - perlBindings = self.hydraJobs.perlBindings.${system}; - } # Add "passthru" tests // flatMapAttrs @@ -422,10 +416,6 @@ supportsCross = false; }; - "nix-perl-bindings" = { - supportsCross = false; - }; - "nix-clang-tidy-plugin" = { supportsCross = false; }; diff --git a/meson.build b/meson.build index bef2e6221a53..b5d9434a1a84 100644 --- a/meson.build +++ b/meson.build @@ -47,11 +47,6 @@ subproject('libmain-c') asan_enabled = 'address' in get_option('b_sanitize') -# Language Bindings -if get_option('bindings') and not meson.is_cross_build() and not asan_enabled - subproject('perl') -endif - # Testing if get_option('unit-tests') subproject('libutil-test-support') diff --git a/meson.options b/meson.options index a306a84252ea..7b847beba831 100644 --- a/meson.options +++ b/meson.options @@ -14,13 +14,6 @@ option( description : 'Build unit tests', ) -option( - 'bindings', - type : 'boolean', - value : true, - description : 'Build language bindings (e.g. Perl)', -) - option( 'benchmarks', type : 'boolean', diff --git a/nix-meson-build-support/common/clang-tidy/build_required_targets.py b/nix-meson-build-support/common/clang-tidy/build_required_targets.py index d55acd74a21e..24e4f290c607 100755 --- a/nix-meson-build-support/common/clang-tidy/build_required_targets.py +++ b/nix-meson-build-support/common/clang-tidy/build_required_targets.py @@ -53,8 +53,6 @@ def main(): + [t for t in custom_commands if t.endswith(".gen.inc")] # Flex/Bison generated parsers + [t for t in custom_commands if t.endswith("-tab.cc")] - # Perl XS generated bindings - + [t for t in custom_commands if t.endswith(".cc") and "perl" in t.lower()] ) ninja_build(args.build_root, targets) diff --git a/nix-meson-build-support/common/clang-tidy/clean_compdb.py b/nix-meson-build-support/common/clang-tidy/clean_compdb.py index 659667209ac3..8087b0bf9b6e 100755 --- a/nix-meson-build-support/common/clang-tidy/clean_compdb.py +++ b/nix-meson-build-support/common/clang-tidy/clean_compdb.py @@ -48,9 +48,6 @@ def cmdfilter(item: dict) -> bool: # Filter out Flex/Bison generated parsers (generated code) if file.endswith("-tab.cc"): return False - # Filter out Perl XS generated bindings (generated code) - if "/perl/" in file and file.endswith(".cc"): - return False return True return [chomp(x) for x in compdb if cmdfilter(x)] diff --git a/packaging/components.nix b/packaging/components.nix index 112791c27d38..fae41caf5084 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -512,8 +512,6 @@ in */ nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { }; - nix-perl-bindings = callPackage ../src/perl/package.nix { }; - # The clang-tidy plugin is a build-time tool loaded into clang-tidy itself, # so it must be built with a clang stdenv for ABI compatibility with the # clang-tidy binary from the same llvmPackages set, regardless of the diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index a29d65cb2a8d..783818e0ebf0 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -127,7 +127,6 @@ nixComponents.callPackage ( rest = builtins.substring 2 (builtins.stringLength flag) flag; in "-D${prefix}:${rest}"; - havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; availableComponents = lib.filterAttrs ( @@ -170,12 +169,7 @@ nixComponents.callPackage ( # perhaps other things that are primarily for overriding and not the shell. config = { # Default getComponents - getComponents = - c: - builtins.removeAttrs c ( - lib.optionals (!havePerl) [ "nix-perl-bindings" ] - ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ] - ); + getComponents = c: builtins.removeAttrs c (lib.optionals (!buildCanExecuteHost) [ "nix-manual" ]); }; /** @@ -211,7 +205,6 @@ nixComponents.callPackage ( "nix-fetchers-tests" "nix-flake-tests" "nix-functional-tests" - "nix-perl-bindings" ] (_: null)) c ); }; @@ -287,9 +280,6 @@ nixComponents.callPackage ( ++ map (transformFlag "libutil") (ignoreCrossFile nixComponents.nix-util.mesonFlags) ++ map (transformFlag "libstore") (ignoreCrossFile nixComponents.nix-store.mesonFlags) ++ map (transformFlag "libfetchers") (ignoreCrossFile nixComponents.nix-fetchers.mesonFlags) - ++ lib.optionals havePerl ( - map (transformFlag "perl") (ignoreCrossFile nixComponents.nix-perl-bindings.mesonFlags) - ) ++ map (transformFlag "libexpr") (ignoreCrossFile nixComponents.nix-expr.mesonFlags) ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags) ++ map (transformFlag "nix") (ignoreCrossFile nixComponents.nix-cli.mesonFlags); @@ -345,8 +335,7 @@ nixComponents.callPackage ( lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark ++ dedupByString (v: "${v}") ( lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) - ) - ++ lib.optional havePerl pkgs.perl; + ); propagatedBuildInputs = dedupByString (v: "${v}") ( lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) diff --git a/packaging/everything.nix b/packaging/everything.nix index 751d861c9e80..df7d57a85860 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -41,8 +41,6 @@ nix-internal-api-docs, nix-external-api-docs, - nix-perl-bindings, - testers, patchedSrc ? null, @@ -65,16 +63,7 @@ let nix-main-c nix-cmd ; - } - // - lib.optionalAttrs - (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - { - # Currently fails in static build - inherit - nix-perl-bindings - ; - }; + }; devdoc = buildEnv { name = "nix-${nix-cli.version}-devdoc"; @@ -144,13 +133,6 @@ stdenv.mkDerivation (finalAttrs: { lib.optionals (stdenv.hostPlatform.isLinux && stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ nix-util-tests.tests.run-without-new-syscalls - ] - ++ - lib.optionals (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - [ - # Perl currently fails in static build - # TODO: Split out tests into a separate derivation? - nix-perl-bindings ]; nativeBuildInputs = [ diff --git a/packaging/hydra.nix b/packaging/hydra.nix index e3f8f1c1f1cb..e30b62eb578d 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -33,7 +33,6 @@ let forAllPackages = forAllPackages' { }; forAllPackages' = { - enableBindings ? false, enableDocs ? false, # already have separate attrs for these }: lib.genAttrs ( @@ -66,9 +65,6 @@ let "nix-json-schema-checks" "nix-clang-tidy-plugin" ] - ++ lib.optionals enableBindings [ - "nix-perl-bindings" - ] ++ lib.optionals enableDocs [ "nix-manual" "nix-manual-manpages-only" @@ -85,7 +81,6 @@ rec { let arbitrarySystem = "x86_64-linux"; listedPkgs = forAllPackages' { - enableBindings = true; enableDocs = true; } (_: null); actualPkgs = lib.concatMapAttrs ( @@ -176,8 +171,6 @@ rec { # Build without unity to catch include issues. withUnityBuild = false; nix-expr = super.nix-expr.override { enableGC = false; }; - # Unclear how to make Perl bindings work with a dynamically linked ASAN. - nix-perl-bindings = null; } ) ); @@ -201,8 +194,6 @@ rec { pkgs.nixComponents2.overrideScope ( self: super: { withTSan = true; - # Dies at startup. - nix-perl-bindings = null; # TSan has issues with fork and threads. nix-functional-tests = super.nix-functional-tests.overrideAttrs { doCheck = false; }; } @@ -256,9 +247,6 @@ rec { ) (forAllSystems (system: components.${system}.${pkgName})) ); - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-perl-bindings); - # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. diff --git a/src/perl/.version b/src/perl/.version deleted file mode 120000 index b7badcd0cc85..000000000000 --- a/src/perl/.version +++ /dev/null @@ -1 +0,0 @@ -../../.version \ No newline at end of file diff --git a/src/perl/.yath.rc.in b/src/perl/.yath.rc.in deleted file mode 100644 index e6f5f93ecdd1..000000000000 --- a/src/perl/.yath.rc.in +++ /dev/null @@ -1,2 +0,0 @@ -[test] --I=rel(@lib_dir@) diff --git a/src/perl/MANIFEST b/src/perl/MANIFEST deleted file mode 100644 index 08897647c978..000000000000 --- a/src/perl/MANIFEST +++ /dev/null @@ -1,7 +0,0 @@ -Changes -Makefile.PL -MANIFEST -Nix.xs -README -t/Nix.t -lib/Nix.pm diff --git a/src/perl/lib/Nix/Config.pm.in b/src/perl/lib/Nix/Config.pm.in deleted file mode 100644 index ad51cff3b28b..000000000000 --- a/src/perl/lib/Nix/Config.pm.in +++ /dev/null @@ -1,24 +0,0 @@ -package Nix::Config; - -use MIME::Base64; -use Nix::Store; - -$version = "@PACKAGE_VERSION@"; - -$storeDir = Nix::Store::getStoreDir; - -%config = (); - -sub readConfig { - my $config = "$confDir/nix.conf"; - return unless -f $config; - - open CONFIG, "<$config" or die "cannot open '$config'"; - while () { - /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next; - $config{$1} = $2; - } - close CONFIG; -} - -return 1; diff --git a/src/perl/lib/Nix/CopyClosure.pm b/src/perl/lib/Nix/CopyClosure.pm deleted file mode 100644 index 902ee1a1bc9f..000000000000 --- a/src/perl/lib/Nix/CopyClosure.pm +++ /dev/null @@ -1,61 +0,0 @@ -package Nix::CopyClosure; - -use utf8; -use strict; -use Nix::Config; -use Nix::Store; -use Nix::SSH; -use List::Util qw(sum); -use IPC::Open2; - - -sub copyToOpen { - my ($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $useSubstitutes) = @_; - - $useSubstitutes = 0 if $dryRun || !defined $useSubstitutes; - - # Get the closure of this path. - my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, - map { followLinksToStorePath $_ } @{$storePaths}))); - - # Send the "query valid paths" command with the "lock" option - # enabled. This prevents a race where the remote host - # garbage-collect paths that are already there. Optionally, ask - # the remote host to substitute missing paths. - syswrite($to, pack("L{url} eq $info->{url}; - } - - push @{$narFileList}, $info if !$found; -} - - -sub addPatch { - my ($patches, $storePath, $patch) = @_; - - $$patches{$storePath} = [] - unless defined $$patches{$storePath}; - - my $patchList = $$patches{$storePath}; - - my $found = 0; - foreach my $patch2 (@{$patchList}) { - $found = 1 if - $patch2->{url} eq $patch->{url} && - $patch2->{basePath} eq $patch->{basePath}; - } - - push @{$patchList}, $patch if !$found; - - return !$found; -} - - -sub readManifest_ { - my ($manifest, $addNAR, $addPatch) = @_; - - # Decompress the manifest if necessary. - if ($manifest =~ /\.bz2$/) { - open MANIFEST, "$Nix::Config::bzip2 -d < $manifest |" - or die "cannot decompress '$manifest': $!"; - } else { - open MANIFEST, "<$manifest" - or die "cannot open '$manifest': $!"; - } - - my $inside = 0; - my $type; - - my $manifestVersion = 2; - - my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType); - my ($narHash, $narSize, $references, $deriver, $copyFrom, $system, $compressionType); - - while () { - chomp; - s/\#.*$//g; - next if (/^$/); - - if (!$inside) { - - if (/^\s*(\w*)\s*\{$/) { - $type = $1; - $type = "narfile" if $type eq ""; - $inside = 1; - undef $storePath; - undef $url; - undef $hash; - undef $size; - undef $narHash; - undef $narSize; - undef $basePath; - undef $baseHash; - undef $patchType; - undef $system; - $references = ""; - $deriver = ""; - $compressionType = "bzip2"; - } - - } else { - - if (/^\}$/) { - $inside = 0; - - if ($type eq "narfile") { - &$addNAR($storePath, - { url => $url, hash => $hash, size => $size - , narHash => $narHash, narSize => $narSize - , references => $references - , deriver => $deriver - , system => $system - , compressionType => $compressionType - }); - } - - elsif ($type eq "patch") { - &$addPatch($storePath, - { url => $url, hash => $hash, size => $size - , basePath => $basePath, baseHash => $baseHash - , narHash => $narHash, narSize => $narSize - , patchType => $patchType - }); - } - - } - - elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; } - elsif (/^\s*CopyFrom:\s*(\/\S+)\s*$/) { $copyFrom = $1; } - elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; } - elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; } - elsif (/^\s*Compression:\s*(\S+)\s*$/) { $compressionType = $1; } - elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; } - elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; } - elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; } - elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; } - elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; } - elsif (/^\s*NarSize:\s*(\d+)\s*$/) { $narSize = $1; } - elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; } - elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; } - elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; } - elsif (/^\s*System:\s*(\S+)\s*$/) { $system = $1; } - - # Compatibility; - elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; } - elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = "md5:$1"; } - - } - } - - close MANIFEST; - - return $manifestVersion; -} - - -sub readManifest { - my ($manifest, $narFiles, $patches) = @_; - readManifest_($manifest, - sub { addNAR($narFiles, @_); }, - sub { addPatch($patches, @_); } ); -} - - -sub writeManifest { - my ($manifest, $narFiles, $patches, $noCompress) = @_; - - open MANIFEST, ">$manifest.tmp"; # !!! check exclusive - - print MANIFEST "version {\n"; - print MANIFEST " ManifestVersion: 3\n"; - print MANIFEST "}\n"; - - foreach my $storePath (sort (keys %{$narFiles})) { - my $narFileList = $$narFiles{$storePath}; - foreach my $narFile (@{$narFileList}) { - print MANIFEST "{\n"; - print MANIFEST " StorePath: $storePath\n"; - print MANIFEST " NarURL: $narFile->{url}\n"; - print MANIFEST " Compression: $narFile->{compressionType}\n"; - print MANIFEST " Hash: $narFile->{hash}\n" if defined $narFile->{hash}; - print MANIFEST " Size: $narFile->{size}\n" if defined $narFile->{size}; - print MANIFEST " NarHash: $narFile->{narHash}\n"; - print MANIFEST " NarSize: $narFile->{narSize}\n" if $narFile->{narSize}; - print MANIFEST " References: $narFile->{references}\n" - if defined $narFile->{references} && $narFile->{references} ne ""; - print MANIFEST " Deriver: $narFile->{deriver}\n" - if defined $narFile->{deriver} && $narFile->{deriver} ne ""; - print MANIFEST " System: $narFile->{system}\n" if defined $narFile->{system}; - print MANIFEST "}\n"; - } - } - - foreach my $storePath (sort (keys %{$patches})) { - my $patchList = $$patches{$storePath}; - foreach my $patch (@{$patchList}) { - print MANIFEST "patch {\n"; - print MANIFEST " StorePath: $storePath\n"; - print MANIFEST " NarURL: $patch->{url}\n"; - print MANIFEST " Hash: $patch->{hash}\n"; - print MANIFEST " Size: $patch->{size}\n"; - print MANIFEST " NarHash: $patch->{narHash}\n"; - print MANIFEST " NarSize: $patch->{narSize}\n" if $patch->{narSize}; - print MANIFEST " BasePath: $patch->{basePath}\n"; - print MANIFEST " BaseHash: $patch->{baseHash}\n"; - print MANIFEST " Type: $patch->{patchType}\n"; - print MANIFEST "}\n"; - } - } - - - close MANIFEST; - - rename("$manifest.tmp", $manifest) - or die "cannot rename $manifest.tmp: $!"; - - - # Create a bzipped manifest. - unless (defined $noCompress) { - system("$Nix::Config::bzip2 < $manifest > $manifest.bz2.tmp") == 0 - or die "cannot compress manifest"; - - rename("$manifest.bz2.tmp", "$manifest.bz2") - or die "cannot rename $manifest.bz2.tmp: $!"; - } -} - - -# Return a fingerprint of a store path to be used in binary cache -# signatures. It contains the store path, the base-32 SHA-256 hash of -# the contents of the path, and the references. -sub fingerprintPath { - my ($storePath, $narHash, $narSize, $references) = @_; - die if substr($storePath, 0, length($Nix::Config::storeDir)) ne $Nix::Config::storeDir; - die if substr($narHash, 0, 7) ne "sha256:"; - # Convert hash from base-16 to base-32, if necessary. - $narHash = "sha256:" . convertHash("sha256", substr($narHash, 7), 1) - if length($narHash) == 71; - die if length($narHash) != 59; - foreach my $ref (@{$references}) { - die if substr($ref, 0, length($Nix::Config::storeDir)) ne $Nix::Config::storeDir; - } - return "1;" . $storePath . ";" . $narHash . ";" . $narSize . ";" . join(",", @{$references}); -} - - -# Parse a NAR info file. -sub parseNARInfo { - my ($storePath, $content, $requireValidSig, $location) = @_; - - my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver, $system, $sig); - my $compression = "bzip2"; - my @refs; - - foreach my $line (split "\n", $content) { - return undef unless $line =~ /^(.*): (.*)$/; - if ($1 eq "StorePath") { $storePath2 = $2; } - elsif ($1 eq "URL") { $url = $2; } - elsif ($1 eq "Compression") { $compression = $2; } - elsif ($1 eq "FileHash") { $fileHash = $2; } - elsif ($1 eq "FileSize") { $fileSize = int($2); } - elsif ($1 eq "NarHash") { $narHash = $2; } - elsif ($1 eq "NarSize") { $narSize = int($2); } - elsif ($1 eq "References") { @refs = split / /, $2; } - elsif ($1 eq "Deriver") { $deriver = $2; } - elsif ($1 eq "System") { $system = $2; } - elsif ($1 eq "Sig") { $sig = $2; } - } - - return undef if $storePath ne $storePath2 || !defined $url || !defined $narHash; - - my $res = - { url => $url - , compression => $compression - , fileHash => $fileHash - , fileSize => $fileSize - , narHash => $narHash - , narSize => $narSize - , refs => [ @refs ] - , deriver => $deriver - , system => $system - }; - - if ($requireValidSig) { - # FIXME: might be useful to support multiple signatures per .narinfo. - - if (!defined $sig) { - warn "NAR info file '$location' lacks a signature; ignoring\n"; - return undef; - } - my ($keyName, $sig64) = split ":", $sig; - return undef unless defined $keyName && defined $sig64; - - my $publicKey = $Nix::Config::binaryCachePublicKeys{$keyName}; - if (!defined $publicKey) { - warn "NAR info file '$location' is signed by unknown key '$keyName'; ignoring\n"; - return undef; - } - - my $fingerprint; - eval { - $fingerprint = fingerprintPath( - $storePath, $narHash, $narSize, - [ map { "$Nix::Config::storeDir/$_" } @refs ]); - }; - if ($@) { - warn "cannot compute fingerprint of '$location'; ignoring\n"; - return undef; - } - - if (!checkSignature($publicKey, decode_base64($sig64), $fingerprint)) { - warn "NAR info file '$location' has an incorrect signature; ignoring\n"; - return undef; - } - - $res->{signedBy} = $keyName; - } - - return $res; -} - - -return 1; diff --git a/src/perl/lib/Nix/SSH.pm b/src/perl/lib/Nix/SSH.pm deleted file mode 100644 index 490ba0ea991e..000000000000 --- a/src/perl/lib/Nix/SSH.pm +++ /dev/null @@ -1,110 +0,0 @@ -package Nix::SSH; - -use utf8; -use strict; -use File::Temp qw(tempdir); -use IPC::Open2; - -our @ISA = qw(Exporter); -our @EXPORT = qw( - @globalSshOpts - readN readInt readString readStrings - writeInt writeString writeStrings - connectToRemoteNix -); - - -our @globalSshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or ""); - - -sub readN { - my ($bytes, $from) = @_; - my $res = ""; - while ($bytes > 0) { - my $s; - my $n = sysread($from, $s, $bytes); - die "I/O error reading from remote side\n" if !defined $n; - die "got EOF while expecting $bytes bytes from remote side\n" if !$n; - $bytes -= $n; - $res .= $s; - } - return $res; -} - - -sub readInt { - my ($from) = @_; - return unpack("L= 0x300; - - return ($from, $to, $pid); -} - - -1; diff --git a/src/perl/lib/Nix/Store.pm b/src/perl/lib/Nix/Store.pm deleted file mode 100644 index f2ae7e88f81d..000000000000 --- a/src/perl/lib/Nix/Store.pm +++ /dev/null @@ -1,45 +0,0 @@ -package Nix::Store; - -use strict; -use warnings; - -require Exporter; - -our @ISA = qw(Exporter); - -our %EXPORT_TAGS = ( 'all' => [ qw( ) ] ); - -our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); - -our @EXPORT = qw( - StoreWrapper - StoreWrapper::new - StoreWrapper::isValidPath StoreWrapper::queryReferences StoreWrapper::queryPathInfo StoreWrapper::queryDeriver StoreWrapper::queryPathHash - StoreWrapper::queryPathFromHashPart - StoreWrapper::topoSortPaths StoreWrapper::computeFSClosure followLinksToStorePath StoreWrapper::exportPaths StoreWrapper::importPaths - StoreWrapper::addToStore StoreWrapper::makeFixedOutputPath - StoreWrapper::derivationFromPath - StoreWrapper::addTempRoot - StoreWrapper::queryRawRealisation - - hashPath hashFile hashString convertHash - signString checkSignature - getStoreDir - setVerbosity -); - -our $VERSION = '0.15'; - -sub backtick { - open(RES, "-|", @_) or die; - local $/; - my $res = || ""; - close RES or die; - return $res; -} - -require XSLoader; -XSLoader::load('Nix::Store', $VERSION); - -1; -__END__ diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs deleted file mode 100644 index 8b28b0e5397c..000000000000 --- a/src/perl/lib/Nix/Store.xs +++ /dev/null @@ -1,430 +0,0 @@ -#include "EXTERN.h" -#include "perl.h" -#include "XSUB.h" - -/* Prevent a clash between some Perl and libstdc++ macros. */ -#undef do_open -#undef do_close - -#include "nix/store/derivations.hh" -#include "nix/store/realisation.hh" -#include "nix/store/globals.hh" -#include "nix/store/store-open.hh" -#include "nix/util/posix-source-accessor.hh" -#include "nix/store/export-import.hh" - -#include -#include - -using namespace nix; - -static bool libStoreInitialized = false; - -struct StoreWrapper { - ref store; -}; - -MODULE = Nix::Store PACKAGE = Nix::Store -PROTOTYPES: ENABLE - -TYPEMAP: < _store; - try { - if (!libStoreInitialized) { - initLibStore(); - libStoreInitialized = true; - } - if (items == 1) { - _store = openStore(); - RETVAL = new StoreWrapper { - .store = ref{_store} - }; - } else { - RETVAL = new StoreWrapper { - .store = openStore(s) - }; - } - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -void init() - CODE: - if (!libStoreInitialized) { - initLibStore(); - libStoreInitialized = true; - } - - -void setVerbosity(int level) - CODE: - verbosity = (Verbosity) level; - - -int -StoreWrapper::isValidPath(char * path) - CODE: - try { - RETVAL = THIS->store->isValidPath(THIS->store->parseStorePath(path)); - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -SV * -StoreWrapper::queryReferences(char * path) - PPCODE: - try { - for (auto & i : THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->references) - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryPathHash(char * path) - PPCODE: - try { - auto s = THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryDeriver(char * path) - PPCODE: - try { - auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); - if (!info->deriver) XSRETURN_UNDEF; - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryPathInfo(char * path, int base32) - PPCODE: - try { - auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); - if (!info->deriver) - XPUSHs(&PL_sv_undef); - else - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - mXPUSHi(info->registrationTime); - mXPUSHi(info->narSize); - AV * refs = newAV(); - for (auto & i : info->references) - av_push(refs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); - XPUSHs(sv_2mortal(newRV((SV *) refs))); - AV * sigs = newAV(); - for (auto & i : info->sigs) - av_push(sigs, newSVpv(i.to_string().c_str(), 0)); - XPUSHs(sv_2mortal(newRV((SV *) sigs))); - } catch (Error & e) { - croak("%s", e.what()); - } - -SV * -StoreWrapper::queryRawRealisation(char * drvPath, char * outputName) - PPCODE: - try { - auto realisation = THIS->store->queryRealisation(DrvOutput{ - .drvPath = THIS->store->parseStorePath(drvPath), - .outputName = outputName, - }); - if (realisation) - XPUSHs(sv_2mortal(newSVpv(static_cast(*realisation).dump().c_str(), 0))); - else - XPUSHs(sv_2mortal(newSVpv("", 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryPathFromHashPart(char * hashPart) - PPCODE: - try { - auto path = THIS->store->queryPathFromHashPart(hashPart); - XPUSHs(sv_2mortal(newSVpv(path ? THIS->store->printStorePath(*path).c_str() : "", 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::computeFSClosure(int flipDirection, int includeOutputs, ...) - PPCODE: - try { - StorePathSet paths; - for (int n = 3; n < items; ++n) - THIS->store->computeFSClosure(THIS->store->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs); - for (auto & i : paths) - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::topoSortPaths(...) - PPCODE: - try { - StorePathSet paths; - for (int n = 1; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); - auto sorted = THIS->store->topoSortPaths(paths); - for (auto & i : sorted) - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::followLinksToStorePath(char * path) - CODE: - try { - RETVAL = newSVpv(THIS->store->printStorePath(THIS->store->followLinksToStorePath(path)).c_str(), 0); - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -void -StoreWrapper::exportPaths(int fd, ...) - PPCODE: - try { - StorePathSet paths; - for (int n = 2; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); - FdSink sink(fd); - exportPaths(*THIS->store, paths, sink); - } catch (Error & e) { - croak("%s", e.what()); - } - - -void -StoreWrapper::importPaths(int fd, int dontCheckSigs) - PPCODE: - try { - FdSource source(fd); - importPaths(*THIS->store, source, dontCheckSigs ? NoCheckSigs : CheckSigs); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -hashPath(char * algo, int base32, char * path) - PPCODE: - try { - Hash h = hashPath( - makeFSSourceAccessor(absPath(path)), - FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first; - auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * hashFile(char * algo, int base32, char * path) - PPCODE: - try { - Hash h = hashFile(parseHashAlgo(algo), path); - auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * hashString(char * algo, int base32, char * s) - PPCODE: - try { - Hash h = hashString(parseHashAlgo(algo), s); - auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * convertHash(char * algo, char * s, int toBase32) - PPCODE: - try { - auto h = Hash::parseAny(s, parseHashAlgo(algo)); - auto s = h.to_string(toBase32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * signString(char * secretKey_, char * msg) - PPCODE: - try { - auto sig = SecretKey(secretKey_).signDetached(msg).to_string(); - XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size()))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -int checkSignature(SV * publicKey_, SV * sig_, char * msg) - CODE: - try { - STRLEN publicKeyLen; - unsigned char * publicKey = (unsigned char *) SvPV(publicKey_, publicKeyLen); - if (publicKeyLen != crypto_sign_PUBLICKEYBYTES) - throw Error("public key is not valid"); - - STRLEN sigLen; - unsigned char * sig = (unsigned char *) SvPV(sig_, sigLen); - if (sigLen != crypto_sign_BYTES) - throw Error("signature is not valid"); - - RETVAL = crypto_sign_verify_detached(sig, (unsigned char *) msg, strlen(msg), publicKey) == 0; - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -SV * -StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) - PPCODE: - try { - auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat; - auto path = THIS->store->addToStore( - std::string(baseNameOf(srcPath)), - makeFSSourceAccessor(absPath(srcPath)), - method, parseHashAlgo(algo)); - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) - PPCODE: - try { - auto h = Hash::parseAny(hash, parseHashAlgo(algo)); - auto method = recursive ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; - auto path = THIS->store->makeFixedOutputPath(name, FixedOutputInfo { - .method = method, - .hash = h, - .references = {}, - }); - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::derivationFromPath(char * drvPath) - PREINIT: - HV *hash; - CODE: - try { - Derivation drv = THIS->store->derivationFromPath(THIS->store->parseStorePath(drvPath)); - hash = newHV(); - - HV * outputs = newHV(); - for (auto & i : drv.outputsAndOptPaths(*THIS->store)) { - hv_store( - outputs, i.first.c_str(), i.first.size(), - !i.second.second - ? newSV(0) /* null value */ - : newSVpv(THIS->store->printStorePath(*i.second.second).c_str(), 0), - 0); - } - hv_stores(hash, "outputs", newRV((SV *) outputs)); - - AV * inputDrvs = newAV(); - for (auto & i : drv.inputDrvs.map) - av_push(inputDrvs, newSVpv(THIS->store->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second - hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs)); - - AV * inputSrcs = newAV(); - for (auto & i : drv.inputSrcs) - av_push(inputSrcs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); - hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs)); - - hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0)); - hv_stores(hash, "builder", newSVpv(drv.builder.c_str(), 0)); - - AV * args = newAV(); - for (auto & i : drv.args) - av_push(args, newSVpv(i.c_str(), 0)); - hv_stores(hash, "args", newRV((SV *) args)); - - HV * env = newHV(); - for (auto & i : drv.env) - hv_store(env, i.first.c_str(), i.first.size(), newSVpv(i.second.c_str(), 0), 0); - hv_stores(hash, "env", newRV((SV *) env)); - - RETVAL = newRV_noinc((SV *)hash); - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -void -StoreWrapper::addTempRoot(char * storePath) - PPCODE: - try { - THIS->store->addTempRoot(THIS->store->parseStorePath(storePath)); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * getStoreDir() - PPCODE: - XPUSHs(sv_2mortal(newSVpv(resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir.c_str(), 0))); diff --git a/src/perl/lib/Nix/Utils.pm b/src/perl/lib/Nix/Utils.pm deleted file mode 100644 index 44955a70698c..000000000000 --- a/src/perl/lib/Nix/Utils.pm +++ /dev/null @@ -1,47 +0,0 @@ -package Nix::Utils; - -use utf8; -use File::Temp qw(tempdir); - -our @ISA = qw(Exporter); -our @EXPORT = qw(checkURL uniq writeFile readFile mkTempDir); - -$urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*]+ )"; - -sub checkURL { - my ($url) = @_; - die "invalid URL '$url'\n" unless $url =~ /^ $urlRE $ /x; -} - -sub uniq { - my %seen; - my @res; - foreach my $name (@_) { - next if $seen{$name}; - $seen{$name} = 1; - push @res, $name; - } - return @res; -} - -sub writeFile { - my ($fn, $s) = @_; - open TMP, ">$fn" or die "cannot create file '$fn': $!"; - print TMP "$s" or die; - close TMP or die; -} - -sub readFile { - local $/ = undef; - my ($fn) = @_; - open TMP, "<$fn" or die "cannot open file '$fn': $!"; - my $s = ; - close TMP or die; - return $s; -} - -sub mkTempDir { - my ($name) = @_; - return tempdir("$name.XXXXXX", CLEANUP => 1, DIR => $ENV{"TMPDIR"} // $ENV{"XDG_RUNTIME_DIR"} // "/tmp") - || die "cannot create a temporary directory"; -} diff --git a/src/perl/lib/Nix/meson.build b/src/perl/lib/Nix/meson.build deleted file mode 100644 index dd5560e21cc5..000000000000 --- a/src/perl/lib/Nix/meson.build +++ /dev/null @@ -1,61 +0,0 @@ -# Nix-Perl Scripts -#============================================================================ - - - -# Sources -#------------------------------------------------- - -nix_perl_store_xs = files('Store.xs') - -nix_perl_scripts = files( - 'CopyClosure.pm', - 'Manifest.pm', - 'SSH.pm', - 'Store.pm', - 'Utils.pm', -) - -nix_perl_scripts_copy_tgts = [] -foreach f : nix_perl_scripts - nix_perl_scripts_copy_tgts += fs.copyfile(f) -endforeach - - -# Targets -#--------------------------------------------------- - -nix_perl_scripts += configure_file( - output : 'Config.pm', - input : 'Config.pm.in', - configuration : nix_perl_conf, -) - -nix_perl_store_cc = custom_target( - 'Store.cc', - output : 'Store.cc', - input : nix_perl_store_xs, - command : [ xsubpp, '@INPUT@', '-output', '@OUTPUT@' ], -) - -# Build Nix::Store Library -#------------------------------------------------- -nix_perl_store_lib = library( - 'Store', - sources : nix_perl_store_cc, - name_prefix : '', - prelink : true, # For C++ static initializers - install : true, - install_mode : 'rwxr-xr-x', - install_dir : join_paths(nix_perl_install_dir, 'auto', 'Nix', 'Store'), - dependencies : nix_perl_store_dep_list, -) - - -# Install Scripts -#--------------------------------------------------- -install_data( - nix_perl_scripts, - install_mode : 'rw-r--r--', - install_dir : join_paths(nix_perl_install_dir, 'Nix'), -) diff --git a/src/perl/meson.build b/src/perl/meson.build deleted file mode 100644 index 59f2a66b8386..000000000000 --- a/src/perl/meson.build +++ /dev/null @@ -1,196 +0,0 @@ -# Nix-Perl Meson build -#============================================================================ - - -# init project -#============================================================================ -project( - 'nix-perl', - 'cpp', - version : files('.version'), - meson_version : '>= 1.1', - license : 'LGPL-2.1-or-later', -) - -# setup env -#------------------------------------------------- -fs = import('fs') -cpp = meson.get_compiler('cpp') -nix_perl_conf = configuration_data() -nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) - - -# set error arguments -#------------------------------------------------- -error_args = [ - '-Wdeprecated-copy', - '-Wdeprecated-declarations', - '-Werror=suggest-override', - '-Werror=unused-result', - '-Wignored-qualifiers', - '-Wno-duplicate-decl-specifier', - '-Wno-literal-suffix', - '-Wno-missing-field-initializers', - '-Wno-non-virtual-dtor', - '-Wno-pedantic', - '-Wno-pointer-bool-conversion', - '-Wno-reserved-user-defined-literal', - '-Wno-unknown-warning-option', - '-Wno-unused-parameter', - '-Wno-unused-variable', - '-Wno-variadic-macros', -] - -add_project_arguments( - cpp.get_supported_arguments(error_args), - language : 'cpp', -) - - -# set install directories -#------------------------------------------------- -prefix = get_option('prefix') -libdir = join_paths(prefix, get_option('libdir')) - -# Dependencies -#============================================================================ - -# Required Programs -#------------------------------------------------- -find_program('xz') -xsubpp = find_program('xsubpp') -perl = find_program('perl') -find_program('curl') -yath = find_program('yath', required : false) - -# Required Libraries -#------------------------------------------------- -bzip2_dep = dependency('bzip2', required : false) -if not bzip2_dep.found() - bzip2_dep = cpp.find_library('bz2') - if not bzip2_dep.found() - error('No "bzip2" pkg-config or "bz2" library found') - endif -endif -curl_dep = dependency('libcurl') -libsodium_dep = dependency('libsodium') - -nix_store_dep = dependency('nix-store') - - -# Finding Perl Headers is a pain. as they do not have -# pkgconfig available, are not in a standard location, -# and are installed into a version folder. Use the -# Perl binary to give hints about perl include dir. -# -# Note that until we have a better solution for this, cross-compiling -# the perl bindings does not appear to be possible. -#------------------------------------------------- -perl_archname = run_command( - perl, - '-e', - 'use Config; print $Config{archname};', - check : true, -).stdout() -perl_version = run_command( - perl, - '-e', - 'use Config; print $Config{version};', - check : true, -).stdout() -perl_archlibexp = run_command( - perl, - '-e', - 'use Config; print $Config{archlibexp};', - check : true, -).stdout() -perl_site_libdir = run_command( - perl, - '-e', - 'use Config; print $Config{installsitearch};', - check : true, -).stdout() -nix_perl_install_dir = join_paths( - libdir, - 'perl5', - 'site_perl', - perl_version, - perl_archname, -) - - -# print perl hints for logs -#------------------------------------------------- -message('Perl archname: @0@'.format(perl_archname)) -message('Perl version: @0@'.format(perl_version)) -message('Perl archlibexp: @0@'.format(perl_archlibexp)) -message('Perl install site: @0@'.format(perl_site_libdir)) -message('Assumed Nix-Perl install dir: @0@'.format(nix_perl_install_dir)) - -# Now find perl modules -#------------------------------------------------- -perl_check_dbi = run_command( - perl, - '-e', - 'use DBI; use DBD::SQLite;', - '-I@0@'.format(get_option('dbi_path')), - '-I@0@'.format(get_option('dbd_sqlite_path')), - check : true, -) - -if perl_check_dbi.returncode() == 2 - error('The Perl modules DBI and/or DBD::SQLite are missing.') -else - message('Found Perl Modules: DBI, DBD::SQLite.') -endif - - - -# declare perl dependency -#------------------------------------------------- -perl_dep = declare_dependency( - dependencies : cpp.find_library( - 'perl', - has_headers : [ - join_paths(perl_archlibexp, 'CORE', 'perl.h'), - join_paths(perl_archlibexp, 'CORE', 'EXTERN.h'), - ], - dirs : [ - join_paths(perl_archlibexp, 'CORE'), - ], - ), - include_directories : join_paths(perl_archlibexp, 'CORE'), -) - -# declare dependencies -#------------------------------------------------- -nix_perl_store_dep_list = [ - perl_dep, - bzip2_dep, - curl_dep, - libsodium_dep, - nix_store_dep, -] - -# # build -# #------------------------------------------------- -lib_dir = join_paths('lib', 'Nix') -subdir(lib_dir) - -if get_option('tests').enabled() - yath_rc_conf = configuration_data() - yath_rc_conf.set('lib_dir', lib_dir) - configure_file( - output : '.yath.rc', - input : '.yath.rc.in', - configuration : yath_rc_conf, - ) - subdir('t') - test( - 'nix-perl-test', - yath, - args : [ 'test' ], - workdir : meson.current_build_dir(), - depends : [ nix_perl_store_lib ] + nix_perl_tests_copy_tgts + nix_perl_scripts_copy_tgts, - ) -endif diff --git a/src/perl/meson.options b/src/perl/meson.options deleted file mode 100644 index 03ddf57f1481..000000000000 --- a/src/perl/meson.options +++ /dev/null @@ -1,30 +0,0 @@ -# Nix-Perl build options -#============================================================================ - - -# compiler args -#============================================================================ - -option( - 'tests', - type : 'feature', - value : 'disabled', - description : 'run nix-perl tests', -) - - -# Location of Perl Modules -#============================================================================ -option( - 'dbi_path', - type : 'string', - value : '/usr', - description : 'path to perl::dbi', -) - -option( - 'dbd_sqlite_path', - type : 'string', - value : '/usr', - description : 'path to perl::dbd-SQLite', -) diff --git a/src/perl/package.nix b/src/perl/package.nix deleted file mode 100644 index e25b2996c83c..000000000000 --- a/src/perl/package.nix +++ /dev/null @@ -1,82 +0,0 @@ -{ - lib, - stdenv, - mkMesonDerivation, - pkg-config, - perl, - perlPackages, - nix-store, - version, - curl, - bzip2, - libsodium, -}: - -let - inherit (lib) fileset; -in - -perl.pkgs.toPerlModule ( - mkMesonDerivation (finalAttrs: { - pname = "nix-perl"; - inherit version; - - workDir = ./.; - fileset = fileset.unions ( - [ - ./.version - ../../.version - ./MANIFEST - ./lib - ./meson.build - ./meson.options - ] - ++ lib.optionals finalAttrs.finalPackage.doCheck [ - ./.yath.rc.in - ./t - ] - ); - - nativeBuildInputs = [ - pkg-config - perl - curl - ]; - - buildInputs = [ - nix-store - bzip2 - libsodium - perlPackages.DBI - perlPackages.DBDSQLite - ]; - - # `perlPackages.Test2Harness` is marked broken for Darwin - doCheck = !stdenv.isDarwin; - - nativeCheckInputs = [ - perlPackages.Test2Harness - ]; - - preConfigure = - # "Inline" .version so its not a symlink, and includes the suffix - '' - chmod u+w .version - echo ${finalAttrs.version} > .version - ''; - - mesonFlags = [ - (lib.mesonEnable "tests" finalAttrs.finalPackage.doCheck) - ]; - - mesonCheckFlags = [ - "--print-errorlogs" - ]; - - strictDeps = false; - - meta = { - platforms = lib.platforms.unix; - }; - }) -) diff --git a/src/perl/t/init.t b/src/perl/t/init.t deleted file mode 100644 index 80197e013766..000000000000 --- a/src/perl/t/init.t +++ /dev/null @@ -1,13 +0,0 @@ -use strict; -use warnings; -use Test2::V0; - -use Nix::Store; - -my $s = new Nix::Store("dummy://"); - -my $res = $s->isValidPath("/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); - -ok(!$res, "should not have path"); - -done_testing; diff --git a/src/perl/t/meson.build b/src/perl/t/meson.build deleted file mode 100644 index f95bee2ffc37..000000000000 --- a/src/perl/t/meson.build +++ /dev/null @@ -1,15 +0,0 @@ -# Nix-Perl Tests -#============================================================================ - - -# src -#--------------------------------------------------- - -nix_perl_tests = files( - 'init.t', -) - -nix_perl_tests_copy_tgts = [] -foreach f : nix_perl_tests - nix_perl_tests_copy_tgts += fs.copyfile(f) -endforeach diff --git a/tests/nixos/fetchers-substitute.nix b/tests/nixos/fetchers-substitute.nix index 7abadd43af64..b61a3929af80 100644 --- a/tests/nixos/fetchers-substitute.nix +++ b/tests/nixos/fetchers-substitute.nix @@ -1,26 +1,9 @@ -{ nixComponents, ... }: { name = "fetchers-substitute"; nodes.substituter = { pkgs, ... }: { - # nix-serve is broken while cross-compiling in nixpkgs 25.11. It's been - # fixed since, but while we're pinning 25.11 we use this workaround. - nixpkgs.overlays = [ - (final: prev: { - nix-serve = - final.lib.warnIf (final.lib.versions.majorMinor final.lib.version != "25.11") - "remove the hack in fetchers-substitute.nix when updating nixpkgs from 25.11" - ( - prev.nix-serve.override { - nix = prev.nix // { - libs.nix-perl-bindings = nixComponents.nix-perl-bindings; - }; - } - ); - }) - ]; virtualisation.writableStore = true; nix.settings.extra-experimental-features = [ @@ -30,6 +13,7 @@ networking.firewall.allowedTCPPorts = [ 5000 ]; + # TODO stop using this, because it has to depend on an older version of Nix that still has the perl bindings. services.nix-serve = { enable = true; secretKeyFile = From 5172f041353f0b26454264fbc5e163518f76a956 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Fri, 1 May 2026 20:35:35 +0200 Subject: [PATCH 341/555] Deduplicate terminal realisation queries in CA derivation resolution This change introduces RealisationCache to ensure each unique derivation output is only queried once from the store during graph traversal. Internal helper functions are moved into an anonymous namespace. --- src/libstore/include/nix/store/realisation.hh | 21 +++ src/libstore/outputs-query.cc | 131 ++++++++++++++---- 2 files changed, 125 insertions(+), 27 deletions(-) diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index 33159b6dc848..ef89d290aa62 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -4,6 +4,7 @@ #include #include "nix/util/hash.hh" +#include "nix/util/std-hash.hh" #include "nix/store/path.hh" #include "nix/store/derived-path.hh" #include @@ -178,6 +179,26 @@ public: } // namespace nix +template<> +struct std::hash +{ + std::size_t operator()(const nix::DrvOutput & id) const noexcept + { + std::size_t h = 0; + nix::hash_combine(h, id.drvPath, id.outputName); + return h; + } +}; + +namespace nix { + +inline std::size_t hash_value(const DrvOutput & id) +{ + return std::hash{}(id); +} + +} // namespace nix + JSON_IMPL(nix::DrvOutput) JSON_IMPL(nix::UnkeyedRealisation) JSON_IMPL(nix::Realisation) diff --git a/src/libstore/outputs-query.cc b/src/libstore/outputs-query.cc index 531bcc85521e..b7bd97d0ed99 100644 --- a/src/libstore/outputs-query.cc +++ b/src/libstore/outputs-query.cc @@ -3,8 +3,26 @@ #include "nix/store/realisation.hh" #include "nix/util/util.hh" +#include + namespace nix { +namespace { + +/** + * Cache mapping a resolved derivation output to its realisation output path. + */ +using RealisationCache = boost::unordered_flat_map>; + +/* Forward declaration so resolveSingleDerivedPath can call it. */ +static std::optional deepQueryPartialDerivationOutputImpl( + Store & store, + const StorePath & drvPath, + const std::string & outputName, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache); + /** * Resolve a `SingleDerivedPath` to a concrete store path. * @@ -15,16 +33,21 @@ namespace nix { * @param queryRealisation must already be initialized (not empty) */ static std::optional resolveSingleDerivedPath( - Store & store, const SingleDerivedPath & path, Store * evalStore_, QueryRealisationFun & queryRealisation) + Store & store, + const SingleDerivedPath & path, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache) { return std::visit( overloaded{ [](const SingleDerivedPath::Opaque & opaque) -> std::optional { return opaque.path; }, [&](const SingleDerivedPath::Built & built) -> std::optional { - auto innerPath = resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation); + auto innerPath = resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation, resCache); if (!innerPath) return std::nullopt; - return deepQueryPartialDerivationOutput(store, *innerPath, built.output, evalStore_, queryRealisation); + return deepQueryPartialDerivationOutputImpl( + store, *innerPath, built.output, evalStore_, queryRealisation, resCache); }, }, path.raw()); @@ -35,8 +58,12 @@ static std::optional resolveSingleDerivedPath( * * @param queryRealisation must already be initialized (not empty) */ -static std::pair -resolveDerivation(Store & store, const StorePath & drvPath, Store * evalStore_, QueryRealisationFun & queryRealisation) +static std::pair resolveDerivation( + Store & store, + const StorePath & drvPath, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache) { auto & evalStore = evalStore_ ? *evalStore_ : store; @@ -50,11 +77,12 @@ resolveDerivation(Store & store, const StorePath & drvPath, Store * evalStore_, store, [&](ref depDrvPath, const std::string & depOutputName) -> std::optional { - auto concreteDrvPath = resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation); + auto concreteDrvPath = + resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation, resCache); if (!concreteDrvPath) return std::nullopt; - return deepQueryPartialDerivationOutput( - store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation); + return deepQueryPartialDerivationOutputImpl( + store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation, resCache); }); if (resolvedDrv) drv = Derivation{*resolvedDrv}; @@ -69,21 +97,79 @@ void queryPartialDerivationOutputMapCA( const StorePath & drvPath, const BasicDerivation & drv, std::map> & outputs, - QueryRealisationFun queryRealisation) + QueryRealisationFun queryRealisation, + RealisationCache & resCache) { if (!queryRealisation) queryRealisation = [&store](const DrvOutput & o) { return store.queryRealisation(o); }; for (auto & [outputName, _] : drv.outputs) { - auto realisation = queryRealisation(DrvOutput{drvPath, outputName}); - if (realisation) { - outputs.insert_or_assign(outputName, realisation->outPath); + DrvOutput id{drvPath, outputName}; + auto it = resCache.find(id); + if (it != resCache.end()) { + outputs.insert_or_assign(outputName, it->second); + continue; + } + + auto realisation = queryRealisation(id); + std::optional outPath = realisation ? std::optional{realisation->outPath} : std::nullopt; + resCache.emplace(id, outPath); + + if (outPath) { + outputs.insert_or_assign(outputName, *outPath); } else { outputs.insert({outputName, std::nullopt}); } } } +/** + * Internal implementation of deepQueryPartialDerivationOutput that accepts a + * shared RealisationCache, allowing memoization of realisation queries across recursive calls. + */ +static std::optional deepQueryPartialDerivationOutputImpl( + Store & store, + const StorePath & drvPath, + const std::string & outputName, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + auto staticResult = evalStore.queryStaticPartialDerivationOutput(drvPath, outputName); + if (staticResult || !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + return staticResult; + + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + + if (drv.outputs.count(outputName) == 0) + throw Error("derivation '%s' does not have an output named '%s'", store.printStorePath(drvPath), outputName); + + DrvOutput id{resolvedDrvPath, outputName}; + auto it = resCache.find(id); + if (it != resCache.end()) + return it->second; + + auto realisation = queryRealisation(id); + std::optional outPath = realisation ? std::optional{realisation->outPath} : std::nullopt; + resCache.emplace(id, outPath); + return outPath; +} + +} // namespace + +void queryPartialDerivationOutputMapCA( + Store & store, + const StorePath & drvPath, + const BasicDerivation & drv, + std::map> & outputs, + QueryRealisationFun queryRealisation) +{ + RealisationCache resCache; + queryPartialDerivationOutputMapCA(store, drvPath, drv, outputs, queryRealisation, resCache); +} + std::map> deepQueryPartialDerivationOutputMap( Store & store, const StorePath & drvPath, Store * evalStore_, QueryRealisationFun queryRealisation) { @@ -97,8 +183,9 @@ std::map> deepQueryPartialDerivationOutput if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return outputs; - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation); - queryPartialDerivationOutputMapCA(store, resolvedDrvPath, drv, outputs, queryRealisation); + RealisationCache resCache; + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + queryPartialDerivationOutputMapCA(store, resolvedDrvPath, drv, outputs, queryRealisation, resCache); return outputs; } @@ -123,22 +210,12 @@ std::optional deepQueryPartialDerivationOutput( Store * evalStore_, QueryRealisationFun queryRealisation) { - auto & evalStore = evalStore_ ? *evalStore_ : store; - if (!queryRealisation) queryRealisation = [&store](const DrvOutput & o) { return store.queryRealisation(o); }; - auto staticResult = evalStore.queryStaticPartialDerivationOutput(drvPath, outputName); - if (staticResult || !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - return staticResult; - - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation); - - if (drv.outputs.count(outputName) == 0) - throw Error("derivation '%s' does not have an output named '%s'", store.printStorePath(drvPath), outputName); - - auto realisation = queryRealisation(DrvOutput{resolvedDrvPath, outputName}); - return realisation ? std::optional{realisation->outPath} : std::nullopt; + RealisationCache resCache; + return deepQueryPartialDerivationOutputImpl( + store, drvPath, outputName, evalStore_, queryRealisation, resCache); } } // namespace nix From e094bbd1cfba091a8cf973d540122cb1a41ca1a5 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Fri, 1 May 2026 20:35:38 +0200 Subject: [PATCH 342/555] Fix exponential complexity in CA derivation output resolution By memoizing the structural resolution of derivations in ResolveCache, we restore linear O(N) performance for graph traversal. This prevents the O(Fib(N)) blowup previously seen in graphs with many overlapping dependencies. --- src/libstore/outputs-query.cc | 45 +++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/libstore/outputs-query.cc b/src/libstore/outputs-query.cc index b7bd97d0ed99..6a5cce222339 100644 --- a/src/libstore/outputs-query.cc +++ b/src/libstore/outputs-query.cc @@ -9,6 +9,13 @@ namespace nix { namespace { +/** + * Cache mapping an unresolved drv path to its resolved (Derivation, StorePath) + * pair. Shared across a single top-level resolution call to prevent exponential + * re-traversal of the closure when many derivations share the same dependencies. + */ +using ResolveCache = boost::unordered_flat_map>; + /** * Cache mapping a resolved derivation output to its realisation output path. */ @@ -21,6 +28,7 @@ static std::optional deepQueryPartialDerivationOutputImpl( const std::string & outputName, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache); /** @@ -37,24 +45,31 @@ static std::optional resolveSingleDerivedPath( const SingleDerivedPath & path, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache) { return std::visit( overloaded{ [](const SingleDerivedPath::Opaque & opaque) -> std::optional { return opaque.path; }, [&](const SingleDerivedPath::Built & built) -> std::optional { - auto innerPath = resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation, resCache); + auto innerPath = + resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation, cache, resCache); if (!innerPath) return std::nullopt; return deepQueryPartialDerivationOutputImpl( - store, *innerPath, built.output, evalStore_, queryRealisation, resCache); + store, *innerPath, built.output, evalStore_, queryRealisation, cache, resCache); }, }, path.raw()); } /** - * Resolve a derivation and compute its store path. + * Resolve a derivation and compute its store path, with memoization. + * + * Results are stored in `cache` (keyed on the unresolved `drvPath`) so that + * each derivation in the closure is resolved at most once per top-level call, + * preventing the exponential re-traversal that would otherwise occur for + * content-addressed derivation closures. * * @param queryRealisation must already be initialized (not empty) */ @@ -63,8 +78,13 @@ static std::pair resolveDerivation( const StorePath & drvPath, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache) { + auto it = cache.find(drvPath); + if (it != cache.end()) + return it->second; + auto & evalStore = evalStore_ ? *evalStore_ : store; Derivation drv = evalStore.readInvalidDerivation(drvPath); @@ -78,18 +98,20 @@ static std::pair resolveDerivation( [&](ref depDrvPath, const std::string & depOutputName) -> std::optional { auto concreteDrvPath = - resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation, resCache); + resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation, cache, resCache); if (!concreteDrvPath) return std::nullopt; return deepQueryPartialDerivationOutputImpl( - store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation, resCache); + store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation, cache, resCache); }); if (resolvedDrv) drv = Derivation{*resolvedDrv}; } auto resolvedDrvPath = computeStorePath(store, drv); - return {std::move(drv), std::move(resolvedDrvPath)}; + auto result = std::make_pair(drv, resolvedDrvPath); + cache.emplace(drvPath, result); + return result; } void queryPartialDerivationOutputMapCA( @@ -125,7 +147,7 @@ void queryPartialDerivationOutputMapCA( /** * Internal implementation of deepQueryPartialDerivationOutput that accepts a - * shared RealisationCache, allowing memoization of realisation queries across recursive calls. + * shared ResolveCache and RealisationCache, allowing memoization across recursive calls. */ static std::optional deepQueryPartialDerivationOutputImpl( Store & store, @@ -133,6 +155,7 @@ static std::optional deepQueryPartialDerivationOutputImpl( const std::string & outputName, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache) { auto & evalStore = evalStore_ ? *evalStore_ : store; @@ -141,7 +164,7 @@ static std::optional deepQueryPartialDerivationOutputImpl( if (staticResult || !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return staticResult; - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, cache, resCache); if (drv.outputs.count(outputName) == 0) throw Error("derivation '%s' does not have an output named '%s'", store.printStorePath(drvPath), outputName); @@ -183,8 +206,9 @@ std::map> deepQueryPartialDerivationOutput if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return outputs; + ResolveCache cache; RealisationCache resCache; - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, cache, resCache); queryPartialDerivationOutputMapCA(store, resolvedDrvPath, drv, outputs, queryRealisation, resCache); return outputs; @@ -213,9 +237,10 @@ std::optional deepQueryPartialDerivationOutput( if (!queryRealisation) queryRealisation = [&store](const DrvOutput & o) { return store.queryRealisation(o); }; + ResolveCache cache; RealisationCache resCache; return deepQueryPartialDerivationOutputImpl( - store, drvPath, outputName, evalStore_, queryRealisation, resCache); + store, drvPath, outputName, evalStore_, queryRealisation, cache, resCache); } } // namespace nix From 94e24ddea9d2e8fb9099a95e8f8c0ea4f9d9bff7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 23:50:00 +0300 Subject: [PATCH 343/555] libutil: Fix path traversal in unpackTarfile Fixes GHSA-gr92-w2r5-qw5p. The primary fix is .relative_path() calls, everything else is making sure we use native path handling in libarchive. --- src/libutil/tarfile.cc | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index eea03766375c..b61710c7e0f9 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -5,6 +5,7 @@ #include "nix/util/serialise.hh" #include "nix/util/tarfile.hh" #include "nix/util/file-system.hh" +#include "nix/util/os-string.hh" namespace nix { @@ -123,6 +124,12 @@ TarArchive::~TarArchive() archive_read_free(this->archive); } +#ifndef _WIN32 +# define NIX_LIBARCHIVE_NATIVE_PATH_FUNC(func) func +#else +# define NIX_LIBARCHIVE_NATIVE_PATH_FUNC(func) func##_w +#endif + static void extract_archive(TarArchive & archive, const std::filesystem::path & destDir) { int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS | ARCHIVE_EXTRACT_SECURE_NODOTDOT; @@ -132,24 +139,34 @@ static void extract_archive(TarArchive & archive, const std::filesystem::path & int r = archive_read_next_header(archive.archive, &entry); if (r == ARCHIVE_EOF) break; - auto name = archive_entry_pathname(entry); - if (!name) - throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); - if (r == ARCHIVE_WARN) - warn("getting archive member '%1%': %2%", name, archive_error_string(archive.archive)); - else - archive.check(r); - archive_entry_copy_pathname(entry, (destDir / name).string().c_str()); + const auto relPath = [&]() -> std::filesystem::path { + /* Some archives might lack a pathname https://github.com/libarchive/libarchive/issues/2089. */ + auto * name = NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_pathname)(entry); + if (!name) + throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); + if (r == ARCHIVE_WARN) + warn( + "getting archive member '%1%': %2%", + os_string_to_string(OsStringView(name)), + archive_error_string(archive.archive)); + else + archive.check(r); + + return std::filesystem::path(name).relative_path(); + }(); + + NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_copy_pathname)(entry, (destDir / relPath).c_str()); // sources can and do contain dirs with no rx bits if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500) archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500); // Patch hardlink path - const char * original_hardlink = archive_entry_hardlink(entry); - if (original_hardlink) { - archive_entry_copy_hardlink(entry, (destDir / original_hardlink).string().c_str()); + const auto * originalHardlink = NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_hardlink)(entry); + if (originalHardlink) { + auto hardlinkPath = std::filesystem::path(originalHardlink).relative_path(); + NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_copy_hardlink)(entry, (destDir / hardlinkPath).c_str()); } archive.check(archive_read_extract(archive.archive, entry, flags)); @@ -158,6 +175,8 @@ static void extract_archive(TarArchive & archive, const std::filesystem::path & archive.close(); } +#undef NIX_LIBARCHIVE_NATIVE_PATH_FUNC + void unpackTarfile(const std::filesystem::path & tarFile, const std::filesystem::path & destDir) { auto archive = TarArchive(tarFile); From e29ca2f114e21ffccaf0de523fe798b6930ee706 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 24 Apr 2025 11:51:10 -0400 Subject: [PATCH 344/555] Simplify the Meson now that upstream bug is fixed Upstream bug is https://github.com/mesonbuild/meson/issues/13584. This has been fixed in https://github.com/mesonbuild/meson/commit/fc9fd42899e1e2160a69ec245931c3aa79b0d267, which is available since Meson 1.8. We now have it with nixpkgs 25.11 and distros should already be caught up. Also does a minor spring clean of our meson accordingly. I noticed that distros have a tendency to patch out our unconditional nix-function-tests subproject from the top-level project, so this also adds an option to disable it. --- doc/manual/meson.build | 2 +- meson.build | 19 +++++++++---------- meson.options | 7 +++++++ src/clang-tidy-plugin/meson.build | 2 +- src/external-api-docs/meson.build | 2 +- src/internal-api-docs/meson.build | 2 +- src/json-schema-checks/meson.build | 2 +- src/libcmd/meson.build | 2 +- src/libexpr-c/meson.build | 2 +- src/libexpr-test-support/meson.build | 2 +- src/libexpr-tests/meson.build | 2 +- src/libexpr/meson.build | 2 +- src/libfetchers-c/meson.build | 2 +- src/libfetchers-tests/meson.build | 2 +- src/libfetchers/meson.build | 2 +- src/libflake-c/meson.build | 2 +- src/libflake-tests/meson.build | 2 +- src/libflake/meson.build | 2 +- src/libmain-c/meson.build | 2 +- src/libmain/meson.build | 2 +- src/libstore-c/meson.build | 2 +- src/libstore-test-support/meson.build | 2 +- src/libstore-tests/meson.build | 2 +- src/libstore/meson.build | 13 +++---------- src/libutil-c/meson.build | 2 +- src/libutil-test-support/meson.build | 2 +- src/libutil-tests/meson.build | 2 +- src/libutil/meson.build | 2 +- src/nix/meson.build | 7 +------ src/nswrapper/meson.build | 3 +-- 30 files changed, 46 insertions(+), 53 deletions(-) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 6fd841e80cbb..5f1fe2ed6732 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -1,7 +1,7 @@ project( 'nix-manual', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/meson.build b/meson.build index b5d9434a1a84..f2a062e396d8 100644 --- a/meson.build +++ b/meson.build @@ -1,15 +1,12 @@ -# This is just a stub project to include all the others as subprojects -# for development shell purposes +# This is just a top-level project to include all the others as subprojects +# for development shell purposes (when building via Nix) or for distro packaging purposes. project( - 'nix-dev-shell', + 'Nix', 'cpp', version : files('.version'), subproject_dir : 'src', - default_options : [ - 'localstatedir=/nix/var', - ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', ) # Internal Libraries @@ -45,8 +42,6 @@ subproject('libexpr-c') subproject('libflake-c') subproject('libmain-c') -asan_enabled = 'address' in get_option('b_sanitize') - # Testing if get_option('unit-tests') subproject('libutil-test-support') @@ -58,7 +53,11 @@ if get_option('unit-tests') subproject('libexpr-tests') subproject('libflake-tests') endif -subproject('nix-functional-tests') + +if get_option('functional-tests') + subproject('nix-functional-tests') +endif + if get_option('json-schema-checks') subproject('json-schema-checks') endif diff --git a/meson.options b/meson.options index 7b847beba831..2e0d873fae08 100644 --- a/meson.options +++ b/meson.options @@ -14,6 +14,13 @@ option( description : 'Build unit tests', ) +option( + 'functional-tests', + type : 'boolean', + value : true, + description : 'Build functional (E2E) tests', +) + option( 'benchmarks', type : 'boolean', diff --git a/src/clang-tidy-plugin/meson.build b/src/clang-tidy-plugin/meson.build index 60cfd1514912..7896bd085479 100644 --- a/src/clang-tidy-plugin/meson.build +++ b/src/clang-tidy-plugin/meson.build @@ -6,7 +6,7 @@ project( 'cpp_std=c++23', 'warning_level=2', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build index 1903b36e589b..d96da3863e6f 100644 --- a/src/external-api-docs/meson.build +++ b/src/external-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-external-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build index 844cb262ee38..3976e4c3f81a 100644 --- a/src/internal-api-docs/meson.build +++ b/src/internal-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-internal-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 8a0bde04b8ed..66dc9b758a44 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -6,7 +6,7 @@ project( 'nix-json-schema-checks', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index d970a8e4b066..9638b491f2c4 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index c47704ce4112..c4ca08a34fd3 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index df28661b7e78..4a87bb4545fd 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index 0b0a01c20654..92e550a0527b 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 510b6d696e3b..d44a0965d96b 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libfetchers-c/meson.build b/src/libfetchers-c/meson.build index db415d9173e7..58f0c26dbffb 100644 --- a/src/libfetchers-c/meson.build +++ b/src/libfetchers-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index ba9774e956b9..b9664eaa1611 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index d34dd4f434d1..ed52e565279d 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index fddb39bdf96b..01fc3e0f4f40 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 3512be10bce9..00b592195c2c 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 58916ecd9ab2..c06bf6ba5450 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 36332fdb70a1..8b144c8aa9cf 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 2ac59924e592..0084643bdaad 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index c81235bf16d4..542b9a1a94f9 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 4d904cb1d06a..ca451db4abce 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index ab03bb2d2a0e..a7353f1ec7dd 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 753c786876ab..0bd42969d8aa 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -9,7 +9,7 @@ project( 'errorlogs=true', # Please print logs for tests that fail 'localstatedir=/nix/var', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -220,15 +220,14 @@ path_opts = [ 'libdir', 'includedir', 'libexecdir', + 'localstatedir', # Homecooked Nix directories. 'store-dir', - 'localstatedir', 'log-dir', ] # For your grepping pleasure, this loop sets the following variables that aren't mentioned # literally above: # store_dir -# localstatedir # log_dir # profile_dir foreach optname : path_opts @@ -399,13 +398,7 @@ libraries_private = [] extra_pkg_config_variables = { 'storedir' : get_option('store-dir'), + 'localstatedir' : get_option('localstatedir'), } -# Working around https://github.com/mesonbuild/meson/issues/13584 -if host_machine.system() != 'darwin' - extra_pkg_config_variables += { - 'localstatedir' : get_option('localstatedir'), - } -endif - subdir('nix-meson-build-support/export') diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index 1806dbb6f9a0..c454853c46b2 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index 64231107eb6b..7299cd65c328 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 6a86504ded42..82322b9cc31b 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libutil/meson.build b/src/libutil/meson.build index d132ce67c748..4c801ffa6762 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/nix/meson.build b/src/nix/meson.build index 3327b846c9c4..93200164957d 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -7,7 +7,6 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'localstatedir=/nix/var', ], meson_version : '>= 1.4', license : 'LGPL-2.1-or-later', @@ -262,11 +261,7 @@ custom_target( # TODO(Ericson3214): Doesn't yet work #meson.override_find_program(linkname, t) -localstatedir = nix_store.get_variable( - 'localstatedir', - default_value : get_option('localstatedir'), -) -assert(localstatedir == get_option('localstatedir')) +localstatedir = nix_store.get_variable('localstatedir') store_dir = nix_store.get_variable('storedir') subdir('scripts') subdir('misc') diff --git a/src/nswrapper/meson.build b/src/nswrapper/meson.build index 77b96d677e72..1d1ccc6b21c3 100644 --- a/src/nswrapper/meson.build +++ b/src/nswrapper/meson.build @@ -7,9 +7,8 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'localstatedir=/nix/var', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) From 2585efd3aa42814b225680f72b33b8eca677ad3a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 3 May 2026 18:39:44 +0300 Subject: [PATCH 345/555] tests/functional: Migrate more repl tests into characterisation framework This is a much more maintainable and easy way of writing tests. I've dropped some filtering (like the number of variables loaded), since we want to test this too and the whole point of characterisation testing is to avoid the manual churn, so it's not an issue. Also added a way to add comments to the repl test without it affecting the output (special `# COM:` prefix in the input, so that tests can be documented). --- tests/functional/repl.sh | 220 +----------------- .../repl/add-lots-of-variables.expected | 32 +++ .../functional/repl/add-lots-of-variables.in | 2 + .../repl/add-overwritten-symbol-env.expected | 13 ++ .../repl/add-overwritten-symbol-env.in | 4 + .../repl/add-variable-with-spaces.expected | 12 + .../repl/add-variable-with-spaces.in | 3 + tests/functional/repl/attribute-set.nix | 6 + .../repl/doc-comment-curried-args.expected | 3 +- .../repl/doc-comment-formals.expected | 3 +- tests/functional/repl/doc-compact.expected | 3 +- tests/functional/repl/doc-constant.expected | 3 +- tests/functional/repl/doc-floatedIn.expected | 3 +- tests/functional/repl/doc-functor.expected | 3 +- .../repl/doc-lambda-flavors.expected | 3 +- .../functional/repl/doc-measurement.expected | 3 +- tests/functional/repl/doc-multiply.expected | 3 +- .../functional/repl/doc-unambiguous.expected | 3 +- .../functional/repl/dollar-escaping.expected | 5 + tests/functional/repl/dollar-escaping.in | 2 + tests/functional/repl/file-a.nix | 1 + tests/functional/repl/file-b.nix | 1 + .../repl/inherit-and-assignment.expected | 9 + .../functional/repl/inherit-and-assignment.in | 4 + .../repl/inherit-current-scope.expected | 9 + .../functional/repl/inherit-current-scope.in | 4 + .../repl/inherit-missing-shows-pos.expected | 13 ++ .../repl/inherit-missing-shows-pos.in | 4 + .../repl/inherit-multiple-attrs.expected | 9 + .../functional/repl/inherit-multiple-attrs.in | 4 + .../repl/inherit-with-semicolon.expected | 9 + .../functional/repl/inherit-with-semicolon.in | 4 + tests/functional/repl/inherit.expected | 9 + tests/functional/repl/inherit.in | 4 + .../repl/list-loaded-nothing-loaded.expected | 5 + .../repl/list-loaded-nothing-loaded.in | 1 + .../repl/multiple-bindings-same-line.expected | 7 + .../repl/multiple-bindings-same-line.in | 3 + .../functional/repl/nested-attr-path.expected | 7 + tests/functional/repl/nested-attr-path.in | 3 + .../repl/pretty-print-idempotent.expected | 3 +- tests/functional/repl/printing.expected | 59 +++++ tests/functional/repl/printing.in | 11 + .../reload-with-non-existent-file.expected | 24 ++ .../repl/reload-with-non-existent-file.in | 5 + 45 files changed, 315 insertions(+), 226 deletions(-) create mode 100644 tests/functional/repl/add-lots-of-variables.expected create mode 100644 tests/functional/repl/add-lots-of-variables.in create mode 100644 tests/functional/repl/add-overwritten-symbol-env.expected create mode 100644 tests/functional/repl/add-overwritten-symbol-env.in create mode 100644 tests/functional/repl/add-variable-with-spaces.expected create mode 100644 tests/functional/repl/add-variable-with-spaces.in create mode 100644 tests/functional/repl/attribute-set.nix create mode 100644 tests/functional/repl/dollar-escaping.expected create mode 100644 tests/functional/repl/dollar-escaping.in create mode 100644 tests/functional/repl/file-a.nix create mode 100644 tests/functional/repl/file-b.nix create mode 100644 tests/functional/repl/inherit-and-assignment.expected create mode 100644 tests/functional/repl/inherit-and-assignment.in create mode 100644 tests/functional/repl/inherit-current-scope.expected create mode 100644 tests/functional/repl/inherit-current-scope.in create mode 100644 tests/functional/repl/inherit-missing-shows-pos.expected create mode 100644 tests/functional/repl/inherit-missing-shows-pos.in create mode 100644 tests/functional/repl/inherit-multiple-attrs.expected create mode 100644 tests/functional/repl/inherit-multiple-attrs.in create mode 100644 tests/functional/repl/inherit-with-semicolon.expected create mode 100644 tests/functional/repl/inherit-with-semicolon.in create mode 100644 tests/functional/repl/inherit.expected create mode 100644 tests/functional/repl/inherit.in create mode 100644 tests/functional/repl/list-loaded-nothing-loaded.expected create mode 100644 tests/functional/repl/list-loaded-nothing-loaded.in create mode 100644 tests/functional/repl/multiple-bindings-same-line.expected create mode 100644 tests/functional/repl/multiple-bindings-same-line.in create mode 100644 tests/functional/repl/nested-attr-path.expected create mode 100644 tests/functional/repl/nested-attr-path.in create mode 100644 tests/functional/repl/printing.expected create mode 100644 tests/functional/repl/printing.in create mode 100644 tests/functional/repl/reload-with-non-existent-file.expected create mode 100644 tests/functional/repl/reload-with-non-existent-file.in diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 88d6e91cd90f..9e752337f10d 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -120,83 +120,6 @@ testReplResponseNoRegex () { testReplResponseGeneral --fixed-strings "$@" } -# :a uses the newest version of a symbol -# -# shellcheck disable=SC2016 -testReplResponse ' -:a { a = "1"; } -:a { a = "2"; } -"result: ${a}" -' "result: 2" - -# check dollar escaping https://github.com/NixOS/nix/issues/4909 -# note the escaped \, -# \\ -# because the second argument is a regex -# -# shellcheck disable=SC2016 -testReplResponseNoRegex ' -"$" + "{hi}" -' '"\${hi}"' - -# Test inherit statement support (issue #15053) -testReplResponseNoRegex ' -a = { b = 1; c = 2; } -inherit (a) b -b -' '1' - -# inherit multiple attributes -testReplResponseNoRegex ' -a = { x = 10; y = 20; } -inherit (a) x y -x + y -' '30' - -# inherit from current scope -testReplResponseNoRegex ' -foo = 42 -inherit foo -foo -' '42' - -# inherit with semicolon (also works) -testReplResponseNoRegex ' -a = { z = 99; } -inherit (a) z; -z -' '99' - -# multiple bindings on one line -testReplResponseNoRegex ' -a = 1; b = 2; -a + b -' '3' - -# nested attribute path -testReplResponseNoRegex ' -a.b.c = 1; -a.b -' '{ c = 1; }' - -# mixed bindings: inherit and assignment -testReplResponseNoRegex ' -x = { p = 10; } -inherit (x) p; q = 20; -p + q -' '30' - -# inherit error shows position (without spurious semicolon from retry) -testReplResponse ' -a = { x = 1; } -inherit (a) y -y -' "error: attribute 'y' missing -.*at .string.:1:13: -.*inherit (a) y -.* \\^ -.*Did you mean x" - testReplResponse ' drvPath ' '".*-simple.drv"' \ @@ -222,32 +145,6 @@ foo + baz ' "3" \ ./flake ./flake\#bar --experimental-features 'flakes' -testReplResponse $' -:a { a = 1; b = 2; longerName = 3; "with spaces" = 4; } -' 'Added 4 variables. -a, b, longerName, "with spaces" -' - -cat < attribute-set.nix -{ - a = 1; - b = 2; - longerName = 3; - "with spaces" = 4; -} -EOF -testReplResponse ' -:l ./attribute-set.nix -' 'Added 4 variables. -a, b, longerName, "with spaces" -' - -testReplResponseNoRegex $' -:a builtins.foldl\' (x: y: x // y) {} (map (x: { ${builtins.toString x} = x; }) (builtins.genList (x: x) 23)) -' 'Added 23 variables. -"0", "1", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "2", "20", "21", "22", "3", "4", "5", "6" -... and 3 more; view with :ll' - # Test the `:reload` mechansim with flakes: # - Eval `./flake#changingThing` # - Modify the flake @@ -313,22 +210,8 @@ EOF grep -q "afterChange" repl_output || fail ":reload didn't pick up git work tree change" fi -# Regression: a failed `:l` / `:lf` must not be remembered for `:reload`, +# Regression: a failed `:lf` must not be remembered for `:reload`, # and an error in one loaded file must not drop later ones from the reload list. -cat > reloadA.nix < reloadB.nix <@g" \ - -e "/Added [0-9]* variables/{s@ [0-9]* @ @;n;d}" \ - -e '/\.\.\. and [0-9]* more; view with :ll/d' \ | grep -vF $'warning: you don\'t have Internet access; disabling some network-dependent features' \ ; } @@ -500,7 +287,10 @@ for test in $(cd "$testDir/repl"; echo *.in); do read -r -a flags < "$testDir/repl/$test.flags" fi - (cd "$testDir/repl"; set +x; runRepl "${flags[@]}" 2>&1) < "$in" > "$actual" || { + # Allow putting comments (lines starting with `# COM:`) in the test for + # documentation purposes. Regular comments are not skipped, since those are + # also interpreted by the repl. + (cd "$testDir/repl"; set +x; runRepl "${flags[@]}" 2>&1) < <(grep -Ev '^[[:space:]]*#[[:space:]]*COM:' "$in") > "$actual" || { echo "FAIL: $test (exit code $?)" >&2 badExitCode=1 } diff --git a/tests/functional/repl/add-lots-of-variables.expected b/tests/functional/repl/add-lots-of-variables.expected new file mode 100644 index 000000000000..4ba282263b47 --- /dev/null +++ b/tests/functional/repl/add-lots-of-variables.expected @@ -0,0 +1,32 @@ +Nix +Type :? for help. + +nix-repl> :a builtins.foldl' (x: y: x // y) {} (map (x: { ${builtins.toString x} = x; }) (builtins.genList (x: x) 23)) +Added 23 variables. +"0", "1", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "2", "20", "21", "22", "3", "4", "5", "6" +... and 3 more; view with :ll + +nix-repl> :ll +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 diff --git a/tests/functional/repl/add-lots-of-variables.in b/tests/functional/repl/add-lots-of-variables.in new file mode 100644 index 000000000000..77796389dfdd --- /dev/null +++ b/tests/functional/repl/add-lots-of-variables.in @@ -0,0 +1,2 @@ +:a builtins.foldl' (x: y: x // y) {} (map (x: { ${builtins.toString x} = x; }) (builtins.genList (x: x) 23)) +:ll diff --git a/tests/functional/repl/add-overwritten-symbol-env.expected b/tests/functional/repl/add-overwritten-symbol-env.expected new file mode 100644 index 000000000000..026b7cceb602 --- /dev/null +++ b/tests/functional/repl/add-overwritten-symbol-env.expected @@ -0,0 +1,13 @@ +Nix +Type :? for help. + +nix-repl> :a { a = "1"; } +Added 1 variables. +a + +nix-repl> :a { a = "2"; } +Added 1 variables. +a + +nix-repl> "result: ${a}" +"result: 2" diff --git a/tests/functional/repl/add-overwritten-symbol-env.in b/tests/functional/repl/add-overwritten-symbol-env.in new file mode 100644 index 000000000000..dbbf4967728d --- /dev/null +++ b/tests/functional/repl/add-overwritten-symbol-env.in @@ -0,0 +1,4 @@ +# COM: :a uses the newest version of a symbol +:a { a = "1"; } +:a { a = "2"; } +"result: ${a}" diff --git a/tests/functional/repl/add-variable-with-spaces.expected b/tests/functional/repl/add-variable-with-spaces.expected new file mode 100644 index 000000000000..dff5a0cbd2be --- /dev/null +++ b/tests/functional/repl/add-variable-with-spaces.expected @@ -0,0 +1,12 @@ +Nix +Type :? for help. + +nix-repl> :a { a = 1; b = 2; longerName = 3; "with spaces" = 4; } +Added 4 variables. +a, b, longerName, "with spaces" + +nix-repl> :reload + +nix-repl> :l ./attribute-set.nix +Added 4 variables. +a, b, longerName, "with spaces" diff --git a/tests/functional/repl/add-variable-with-spaces.in b/tests/functional/repl/add-variable-with-spaces.in new file mode 100644 index 000000000000..763a61901248 --- /dev/null +++ b/tests/functional/repl/add-variable-with-spaces.in @@ -0,0 +1,3 @@ +:a { a = 1; b = 2; longerName = 3; "with spaces" = 4; } +:reload +:l ./attribute-set.nix diff --git a/tests/functional/repl/attribute-set.nix b/tests/functional/repl/attribute-set.nix new file mode 100644 index 000000000000..7b2e71badd48 --- /dev/null +++ b/tests/functional/repl/attribute-set.nix @@ -0,0 +1,6 @@ +{ + a = 1; + b = 2; + longerName = 3; + "with spaces" = 4; +} diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected index d2a5bf328535..29ca40cee2cc 100644 --- a/tests/functional/repl/doc-comment-curried-args.expected +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc curriedArgs Function `curriedArgs`\ diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected index 357cf9986808..65fbc2bd5d80 100644 --- a/tests/functional/repl/doc-comment-formals.expected +++ b/tests/functional/repl/doc-comment-formals.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> "Note that this is not yet complete" "Note that this is not yet complete" diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected index 276de2e60b59..9fcf6169a5fc 100644 --- a/tests/functional/repl/doc-compact.expected +++ b/tests/functional/repl/doc-compact.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc compact Function `compact`\ diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected index a68188b25abf..bda53bbfa660 100644 --- a/tests/functional/repl/doc-constant.expected +++ b/tests/functional/repl/doc-constant.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc constant error: value does not have documentation diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected index 3bf1c40715b1..e01f8868ee7b 100644 --- a/tests/functional/repl/doc-floatedIn.expected +++ b/tests/functional/repl/doc-floatedIn.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc floatedIn Function `floatedIn`\ diff --git a/tests/functional/repl/doc-functor.expected b/tests/functional/repl/doc-functor.expected index 8b86fe913448..e5984098c344 100644 --- a/tests/functional/repl/doc-functor.expected +++ b/tests/functional/repl/doc-functor.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-functor.nix -Added variables. +Added 13 variables. +diverging, doubler, helper, helper2, helper3, lib, makeOverridable, makeVeryOverridable, multiplier, multiply, recursive, recursive2, square nix-repl> :doc multiplier Function `__functor`\ diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected index 437c09d2b319..676ef73135dd 100644 --- a/tests/functional/repl/doc-lambda-flavors.expected +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc nonStrict Function `nonStrict`\ diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected index 862697613be6..ef176cea883f 100644 --- a/tests/functional/repl/doc-measurement.expected +++ b/tests/functional/repl/doc-measurement.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc measurement Function `measurement`\ diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected index 21523e24c818..c2024b094918 100644 --- a/tests/functional/repl/doc-multiply.expected +++ b/tests/functional/repl/doc-multiply.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc multiply Function `multiply`\ diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected index 32ca9aef22ae..d2a452ba7bf3 100644 --- a/tests/functional/repl/doc-unambiguous.expected +++ b/tests/functional/repl/doc-unambiguous.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc unambiguous Function `unambiguous`\ diff --git a/tests/functional/repl/dollar-escaping.expected b/tests/functional/repl/dollar-escaping.expected new file mode 100644 index 000000000000..6e9d237f9162 --- /dev/null +++ b/tests/functional/repl/dollar-escaping.expected @@ -0,0 +1,5 @@ +Nix +Type :? for help. + +nix-repl> "$" + "{hi}" +"\${hi}" diff --git a/tests/functional/repl/dollar-escaping.in b/tests/functional/repl/dollar-escaping.in new file mode 100644 index 000000000000..39734f24b864 --- /dev/null +++ b/tests/functional/repl/dollar-escaping.in @@ -0,0 +1,2 @@ +# COM: Check dollar escaping https://github.com/NixOS/nix/issues/4909 +"$" + "{hi}" diff --git a/tests/functional/repl/file-a.nix b/tests/functional/repl/file-a.nix new file mode 100644 index 000000000000..f113eebfd9b9 --- /dev/null +++ b/tests/functional/repl/file-a.nix @@ -0,0 +1 @@ +{ fromA = 1; } diff --git a/tests/functional/repl/file-b.nix b/tests/functional/repl/file-b.nix new file mode 100644 index 000000000000..ddde63c16c42 --- /dev/null +++ b/tests/functional/repl/file-b.nix @@ -0,0 +1 @@ +{ fromB = 2; } diff --git a/tests/functional/repl/inherit-and-assignment.expected b/tests/functional/repl/inherit-and-assignment.expected new file mode 100644 index 000000000000..43ad52b5ef88 --- /dev/null +++ b/tests/functional/repl/inherit-and-assignment.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> x = { p = 10; } + +nix-repl> inherit (x) p; q = 20; + +nix-repl> p + q +30 diff --git a/tests/functional/repl/inherit-and-assignment.in b/tests/functional/repl/inherit-and-assignment.in new file mode 100644 index 000000000000..583b9096b5e8 --- /dev/null +++ b/tests/functional/repl/inherit-and-assignment.in @@ -0,0 +1,4 @@ +# COM: mixed bindings: inherit and assignment +x = { p = 10; } +inherit (x) p; q = 20; +p + q diff --git a/tests/functional/repl/inherit-current-scope.expected b/tests/functional/repl/inherit-current-scope.expected new file mode 100644 index 000000000000..d2b2f39d224a --- /dev/null +++ b/tests/functional/repl/inherit-current-scope.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> foo = 42 + +nix-repl> inherit foo + +nix-repl> foo +42 diff --git a/tests/functional/repl/inherit-current-scope.in b/tests/functional/repl/inherit-current-scope.in new file mode 100644 index 000000000000..d7eb0047afb2 --- /dev/null +++ b/tests/functional/repl/inherit-current-scope.in @@ -0,0 +1,4 @@ +# COM: inherit from current scope +foo = 42 +inherit foo +foo diff --git a/tests/functional/repl/inherit-missing-shows-pos.expected b/tests/functional/repl/inherit-missing-shows-pos.expected new file mode 100644 index 000000000000..dc02de2070f1 --- /dev/null +++ b/tests/functional/repl/inherit-missing-shows-pos.expected @@ -0,0 +1,13 @@ +Nix +Type :? for help. + +nix-repl> a = { x = 1; } + +nix-repl> inherit (a) y + +nix-repl> y +error: attribute 'y' missing + at «string»:1:13: + 1| inherit (a) y + | ^ + Did you mean x? diff --git a/tests/functional/repl/inherit-missing-shows-pos.in b/tests/functional/repl/inherit-missing-shows-pos.in new file mode 100644 index 000000000000..057b2e816e92 --- /dev/null +++ b/tests/functional/repl/inherit-missing-shows-pos.in @@ -0,0 +1,4 @@ +# COM: inherit error shows position (without spurious semicolon from retry) +a = { x = 1; } +inherit (a) y +y diff --git a/tests/functional/repl/inherit-multiple-attrs.expected b/tests/functional/repl/inherit-multiple-attrs.expected new file mode 100644 index 000000000000..e60df510a3cf --- /dev/null +++ b/tests/functional/repl/inherit-multiple-attrs.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> a = { x = 10; y = 20; } + +nix-repl> inherit (a) x y + +nix-repl> x + y +30 diff --git a/tests/functional/repl/inherit-multiple-attrs.in b/tests/functional/repl/inherit-multiple-attrs.in new file mode 100644 index 000000000000..0da0815b2d15 --- /dev/null +++ b/tests/functional/repl/inherit-multiple-attrs.in @@ -0,0 +1,4 @@ +# COM: inherit multiple attributes +a = { x = 10; y = 20; } +inherit (a) x y +x + y diff --git a/tests/functional/repl/inherit-with-semicolon.expected b/tests/functional/repl/inherit-with-semicolon.expected new file mode 100644 index 000000000000..37d4020104d5 --- /dev/null +++ b/tests/functional/repl/inherit-with-semicolon.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> a = { z = 99; } + +nix-repl> inherit (a) z; + +nix-repl> z +99 diff --git a/tests/functional/repl/inherit-with-semicolon.in b/tests/functional/repl/inherit-with-semicolon.in new file mode 100644 index 000000000000..d7a4972a2b3f --- /dev/null +++ b/tests/functional/repl/inherit-with-semicolon.in @@ -0,0 +1,4 @@ +# COM: inherit with semicolon +a = { z = 99; } +inherit (a) z; +z diff --git a/tests/functional/repl/inherit.expected b/tests/functional/repl/inherit.expected new file mode 100644 index 000000000000..9fde67b8cf46 --- /dev/null +++ b/tests/functional/repl/inherit.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> a = { b = 1; c = 2; } + +nix-repl> inherit (a) b + +nix-repl> b +1 diff --git a/tests/functional/repl/inherit.in b/tests/functional/repl/inherit.in new file mode 100644 index 000000000000..804bbbec17a8 --- /dev/null +++ b/tests/functional/repl/inherit.in @@ -0,0 +1,4 @@ +# COM: Test inherit statement support (issue #15053) +a = { b = 1; c = 2; } +inherit (a) b +b diff --git a/tests/functional/repl/list-loaded-nothing-loaded.expected b/tests/functional/repl/list-loaded-nothing-loaded.expected new file mode 100644 index 000000000000..1f9f97c984d2 --- /dev/null +++ b/tests/functional/repl/list-loaded-nothing-loaded.expected @@ -0,0 +1,5 @@ +Nix +Type :? for help. + +nix-repl> :ll +error: nothing has been loaded yet diff --git a/tests/functional/repl/list-loaded-nothing-loaded.in b/tests/functional/repl/list-loaded-nothing-loaded.in new file mode 100644 index 000000000000..f47ea2c888f9 --- /dev/null +++ b/tests/functional/repl/list-loaded-nothing-loaded.in @@ -0,0 +1 @@ +:ll diff --git a/tests/functional/repl/multiple-bindings-same-line.expected b/tests/functional/repl/multiple-bindings-same-line.expected new file mode 100644 index 000000000000..77062af5fe63 --- /dev/null +++ b/tests/functional/repl/multiple-bindings-same-line.expected @@ -0,0 +1,7 @@ +Nix +Type :? for help. + +nix-repl> a = 1; b = 2; + +nix-repl> a + b +3 diff --git a/tests/functional/repl/multiple-bindings-same-line.in b/tests/functional/repl/multiple-bindings-same-line.in new file mode 100644 index 000000000000..6dd1d62a0e43 --- /dev/null +++ b/tests/functional/repl/multiple-bindings-same-line.in @@ -0,0 +1,3 @@ +# COM: multiple bindings on one line +a = 1; b = 2; +a + b diff --git a/tests/functional/repl/nested-attr-path.expected b/tests/functional/repl/nested-attr-path.expected new file mode 100644 index 000000000000..59164fede4b9 --- /dev/null +++ b/tests/functional/repl/nested-attr-path.expected @@ -0,0 +1,7 @@ +Nix +Type :? for help. + +nix-repl> a.b.c = 1; + +nix-repl> a.b +{ c = 1; } diff --git a/tests/functional/repl/nested-attr-path.in b/tests/functional/repl/nested-attr-path.in new file mode 100644 index 000000000000..4862c3c34bf4 --- /dev/null +++ b/tests/functional/repl/nested-attr-path.in @@ -0,0 +1,3 @@ +# COM: nested attribute path +a.b.c = 1; +a.b diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected index 311855dae363..83ce76435783 100644 --- a/tests/functional/repl/pretty-print-idempotent.expected +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l pretty-print-idempotent.nix -Added variables. +Added 4 variables. +oneDeep, oneDeepList, twoDeep, twoDeepList nix-repl> oneDeep { homepage = "https://example.com"; } diff --git a/tests/functional/repl/printing.expected b/tests/functional/repl/printing.expected new file mode 100644 index 000000000000..f111939eb5b3 --- /dev/null +++ b/tests/functional/repl/printing.expected @@ -0,0 +1,59 @@ +Nix +Type :? for help. + +nix-repl> { a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +{ + a = { ... }; + l = [ ... ]; + n = 1234; + s = "string"; + x = { ... }; +} + +nix-repl> [ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ] +[ + 42 + 1 + "thingy" + { ... } + [ ... ] +] + +nix-repl> let x = { y = { a = 1; }; inherit x; }; in x +{ + x = «repeated»; + y = { ... }; +} + +nix-repl> :p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +{ + a = { b = 2; }; + n = 1234; + s = "string"; + x = { + y = { + z = { + y = «repeated»; + }; + }; + }; +} + +nix-repl> :p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ] +[ + 42 + 1 + "thingy" + { + a = 1; + b = { + a = 1; + b = «repeated»; + }; + } + [ + 1 + 2 + 3 + ] +] diff --git a/tests/functional/repl/printing.in b/tests/functional/repl/printing.in new file mode 100644 index 000000000000..f3ca2cc88fcc --- /dev/null +++ b/tests/functional/repl/printing.in @@ -0,0 +1,11 @@ +# COM: Test recursive printing and formatting +# COM: Normal output should print attributes in lexicographical order non-recursively +{ a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +# COM: Same for lists, but order is preserved +[ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ] +# COM: Same for let expressions +let x = { y = { a = 1; }; inherit x; }; in x +# COM: The :p command should recursively print sets, but prevent infinite recursion +:p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +# COM: Same for lists +:p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ] diff --git a/tests/functional/repl/reload-with-non-existent-file.expected b/tests/functional/repl/reload-with-non-existent-file.expected new file mode 100644 index 000000000000..e15be6e73085 --- /dev/null +++ b/tests/functional/repl/reload-with-non-existent-file.expected @@ -0,0 +1,24 @@ +Nix +Type :? for help. + +nix-repl> :l file-a.nix +Added 1 variables. +fromA + +nix-repl> :l ./does-not-exist.nix +error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist + +nix-repl> :l file-b.nix +Added 1 variables. +fromB + +nix-repl> :r +Loading "file-a.nix"... +Added 1 variables. +fromA +Loading "file-b.nix"... +Added 1 variables. +fromB + +nix-repl> fromA + fromB +3 diff --git a/tests/functional/repl/reload-with-non-existent-file.in b/tests/functional/repl/reload-with-non-existent-file.in new file mode 100644 index 000000000000..740b3be9a412 --- /dev/null +++ b/tests/functional/repl/reload-with-non-existent-file.in @@ -0,0 +1,5 @@ +:l file-a.nix +:l ./does-not-exist.nix +:l file-b.nix +:r +fromA + fromB From a953d61686126d5cd39e5430bbf89674e3662261 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 17:33:36 +0300 Subject: [PATCH 346/555] derivaton-builder: Reap recursive-nix daemon worker threads early Not reaping threads early on leads to resource exhaustion when handling a lot of connection (like in nix-ninja). Ideally this would all just be async coroutine code so that we didn't have to spawn threads to handle connections, but it's a long-term goal. Fixes ENOMEM errors when building nix via nix-ninja. --- src/libstore/unix/build/derivation-builder.cc | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 3e9ad5434754..9c0463791e70 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -41,6 +41,8 @@ #include #include #include +#include +#include #include "nix/util/strings.hh" #include "nix/util/signals.hh" @@ -226,10 +228,16 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder */ std::thread daemonThread; + struct DaemonWorkerState + { + std::thread thread; + ref done; + }; + /** * The daemon worker threads. */ - std::vector daemonWorkerThreads; + std::list daemonWorkerThreads; const StorePathSet & originalPaths() override { @@ -1201,19 +1209,41 @@ void DerivationBuilderImpl::startDaemon() debug("received daemon connection"); - auto workerThread = std::thread([store, remote{std::move(remote)}]() { + auto doneFlag = make_ref(); + + auto workerThread = std::thread([doneFlag, store, remote{std::move(remote)}]() { try { daemon::processConnection( store, FdSource(remote.get()), FdSink(remote.get()), NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (const Interrupted &) { debug("interrupted daemon connection"); - } catch (SystemError &) { + } catch (...) { + /* Swallow all exceptions to avoid crashing the the process (exceptions that escape from the thread + * trigger std::terminate()). */ ignoreExceptionExceptInterrupt(); } + + doneFlag->test_and_set(std::memory_order_relaxed); }); - daemonWorkerThreads.push_back(std::move(workerThread)); + daemonWorkerThreads.push_back( + DaemonWorkerState{ + .thread = std::move(workerThread), + .done = std::move(doneFlag), + }); + + /* Prune threads eagerly to free up resources. Ideally we'd also limit the number of concurrent workers. */ + for (auto it = daemonWorkerThreads.begin(), end = daemonWorkerThreads.end(); it != end;) { + auto & state = *it; + auto & thread = state.thread; + if (state.done->test(std::memory_order_relaxed) && thread.joinable()) { + thread.join(); + it = daemonWorkerThreads.erase(it); + } else { + ++it; + } + } } debug("daemon shutting down"); @@ -1242,9 +1272,7 @@ void DerivationBuilderImpl::stopDaemon() if (daemonThread.joinable()) daemonThread.join(); - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) + for (auto & [thread, doneFlag] : daemonWorkerThreads) thread.join(); daemonWorkerThreads.clear(); From 2acb40b2aa1c272dff4eb698921554c7ef948d30 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 May 2026 14:51:32 +0200 Subject: [PATCH 347/555] Don't destroy _fileTransfer on shutdown This is responsible for a lot of crash reports in Sentry (presumably due to destructor ordering issues). Since there is no cleanup done by this class that we care about, just leak it. --- src/libstore/filetransfer.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 6d97fb4e3f5d..a07c92fe0cb1 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1183,14 +1183,16 @@ ref makeCurlFileTransfer(const FileTransferSettings & settings return make_ref(settings); } +static auto * const _fileTransfer = new Sync>; + ref getFileTransfer() { - static ref fileTransfer = makeCurlFileTransfer(); + auto fileTransfer(_fileTransfer->lock()); - if (fileTransfer->state_.lock()->isQuitting()) - fileTransfer = makeCurlFileTransfer(); + if (!*fileTransfer || (*fileTransfer)->state_.lock()->isQuitting()) + *fileTransfer = makeCurlFileTransfer().get_ptr(); - return fileTransfer; + return ref(*fileTransfer); } ref makeFileTransfer(const FileTransferSettings & settings) From 8f40805cc63d9e55e5e8409bdf53baad91be1850 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Apr 2026 21:29:39 +0100 Subject: [PATCH 348/555] feat(libutil): add `memo(f0)` memoization combinator Memoizes a `fun`. Useful because "call by need" has its use cases outside the evaluator too. Motivating use case: libfetchers lazy attributes. --- src/libutil-tests/memo.cc | 46 ++++++++++++++++++++++++ src/libutil-tests/meson.build | 1 + src/libutil/include/nix/util/memo.hh | 40 +++++++++++++++++++++ src/libutil/include/nix/util/meson.build | 1 + 4 files changed, 88 insertions(+) create mode 100644 src/libutil-tests/memo.cc create mode 100644 src/libutil/include/nix/util/memo.hh diff --git a/src/libutil-tests/memo.cc b/src/libutil-tests/memo.cc new file mode 100644 index 000000000000..76be65260953 --- /dev/null +++ b/src/libutil-tests/memo.cc @@ -0,0 +1,46 @@ +#include +#include + +#include "nix/util/memo.hh" + +namespace nix { + +TEST(memo, computesOnce) +{ + int calls = 0; + fun f = memo([&calls]() -> int { + calls++; + return 42; + }); + EXPECT_EQ(f(), 42); + EXPECT_EQ(f(), 42); + EXPECT_EQ(f(), 42); + EXPECT_EQ(calls, 1); +} + +TEST(memo, copiesShareCache) +{ + int calls = 0; + fun f = memo([&calls]() -> int { + calls++; + return 7; + }); + auto g = f; + EXPECT_EQ(f(), 7); + EXPECT_EQ(g(), 7); + EXPECT_EQ(calls, 1); +} + +TEST(memo, worksWithString) +{ + int calls = 0; + fun f = memo([&calls]() -> std::string { + calls++; + return "hello"; + }); + EXPECT_EQ(f(), "hello"); + EXPECT_EQ(f(), "hello"); + EXPECT_EQ(calls, 1); +} + +} // namespace nix diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 6a86504ded42..527d5e3fbff5 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -79,6 +79,7 @@ sources = files( 'json-utils.cc', 'logging.cc', 'lru-cache.cc', + 'memo.cc', 'memory-source-accessor.cc', 'monitorfdhup.cc', 'nar-listing.cc', diff --git a/src/libutil/include/nix/util/memo.hh b/src/libutil/include/nix/util/memo.hh new file mode 100644 index 000000000000..ba09bac6eb75 --- /dev/null +++ b/src/libutil/include/nix/util/memo.hh @@ -0,0 +1,40 @@ +#pragma once +///@file + +#include "nix/util/fun.hh" + +#include +#include +#include + +namespace nix { + +/** + * Memoize a `fun`. + * + * Copies of the returned `fun` share the same cache. + * Thread-safe. + */ +template +fun memo(fun f) +{ + struct State + { + fun compute; + std::once_flag flag; + std::optional cached; + + explicit State(fun compute) + : compute(std::move(compute)) + { + } + }; + + auto state = std::shared_ptr(new State(std::move(f))); + return [state]() -> T { + std::call_once(state->flag, [&]() { state->cached.emplace(state->compute()); }); + return *state->cached; + }; +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 0f7a40df7a8c..cea6d48f5920 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -60,6 +60,7 @@ headers = [ config_pub_h ] + files( 'json-utils.hh', 'logging.hh', 'lru-cache.hh', + 'memo.hh', 'memory-source-accessor.hh', 'mounted-source-accessor.hh', 'muxable-pipe.hh', From ee78fe13ecc2e7d49d4763661760960b4c949362 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 21 Apr 2026 12:48:48 +0100 Subject: [PATCH 349/555] feat(libfetchers): add lazy attribute values Extend the Attr variant with a LazyAttr alternative: a deferred computation that is only evaluated when the attribute value is needed. This lets fetchers defer expensive work (like revCount) until an expression demands it. The existing getters and serialization transparently force any lazy attrs. --- src/libfetchers-tests/attrs.cc | 75 +++++++++++++++++++ src/libfetchers-tests/meson.build | 1 + src/libfetchers/attrs.cc | 35 ++++++--- src/libfetchers/include/nix/fetchers/attrs.hh | 26 ++++++- src/libflake/flake-primops.cc | 3 +- 5 files changed, 129 insertions(+), 11 deletions(-) create mode 100644 src/libfetchers-tests/attrs.cc diff --git a/src/libfetchers-tests/attrs.cc b/src/libfetchers-tests/attrs.cc new file mode 100644 index 000000000000..4d0cbbb9ef74 --- /dev/null +++ b/src/libfetchers-tests/attrs.cc @@ -0,0 +1,75 @@ +#include + +#include "nix/fetchers/attrs.hh" + +#include + +namespace nix::fetchers { + +TEST(LazyAttr, resolveToInt) +{ + Attrs attrs; + attrs.insert_or_assign( + "count", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(42); + }}))); + EXPECT_EQ(maybeGetIntAttr(attrs, "count"), 42); +} + +TEST(LazyAttr, resolveToString) +{ + Attrs attrs; + attrs.insert_or_assign( + "name", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("hello"); + }}))); + EXPECT_EQ(maybeGetStrAttr(attrs, "name"), "hello"); +} + +TEST(LazyAttr, resolveToBool) +{ + Attrs attrs; + attrs.insert_or_assign( + "flag", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return Explicit{true}; + }}))); + EXPECT_EQ(maybeGetBoolAttr(attrs, "flag"), true); +} + +TEST(LazyAttr, attrsToJSONForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "x", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(99); + }}))); + auto json = attrsToJSON(attrs); + EXPECT_EQ(json["x"], 99); +} + +TEST(LazyAttr, attrsToQueryForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "v", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("val"); + }}))); + auto query = attrsToQuery(attrs); + EXPECT_EQ(query.at("v"), "val"); +} + +TEST(LazyAttr, notCalledUntilForced) +{ + int calls = 0; + Attrs attrs; + attrs.insert_or_assign( + "lazy", LazyAttr(make_ref(LazyAttrComputation{.compute = [&calls]() -> ResolvedAttr { + calls++; + return uint64_t(1); + }}))); + EXPECT_EQ(calls, 0); + maybeGetIntAttr(attrs, "lazy"); + EXPECT_EQ(calls, 1); +} + +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index ba9774e956b9..8cbb42e21375 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -40,6 +40,7 @@ subdir('nix-meson-build-support/common') sources = files( 'access-tokens.cc', + 'attrs.cc', 'git-utils.cc', 'git.cc', 'input.cc', diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index cc9e72af460d..39e7a0fd5d6c 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -4,6 +4,18 @@ namespace nix::fetchers { +ResolvedAttr forceAttr(const Attr & attr) +{ + return std::visit( + overloaded{ + [](const LazyAttr & lazy) -> ResolvedAttr { return lazy->compute(); }, + [](const std::string & v) -> ResolvedAttr { return v; }, + [](uint64_t v) -> ResolvedAttr { return v; }, + [](const Explicit & v) -> ResolvedAttr { return v; }, + }, + attr); +} + Attrs jsonToAttrs(const nlohmann::json & json) { Attrs attrs; @@ -26,11 +38,12 @@ nlohmann::json attrsToJSON(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { json[attr.first] = v->t; } else unreachable(); @@ -43,7 +56,8 @@ std::optional maybeGetStrAttr(const Attrs & attrs, const std::strin auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump()); } @@ -61,7 +75,8 @@ std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not an integer", name); } @@ -79,7 +94,8 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if>(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if>(&resolved)) return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -96,11 +112,12 @@ StringMap attrsToQuery(const Attrs & attrs) { StringMap query; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, fmt("%d", *v)); - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, *v); - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); } else unreachable(); diff --git a/src/libfetchers/include/nix/fetchers/attrs.hh b/src/libfetchers/include/nix/fetchers/attrs.hh index 8a21b8ddbf69..60730d65d32b 100644 --- a/src/libfetchers/include/nix/fetchers/attrs.hh +++ b/src/libfetchers/include/nix/fetchers/attrs.hh @@ -3,6 +3,8 @@ #include "nix/util/types.hh" #include "nix/util/hash.hh" +#include "nix/util/ref.hh" +#include "nix/util/fun.hh" #include @@ -12,7 +14,24 @@ namespace nix::fetchers { -typedef std::variant> Attr; +/** + * The resolved (non-lazy) subset of attribute value types. + */ +using ResolvedAttr = std::variant>; + +/** + * A deferred attribute computation. Wrapping in `ref<>` gives + * pointer-identity equality/ordering, which is correct: two lazy + * attrs are equal iff they are the same computation. + */ +struct LazyAttrComputation +{ + fun compute; +}; + +using LazyAttr = ref; + +using Attr = std::variant, LazyAttr>; /** * An `Attrs` can be thought of a JSON object restricted or simplified @@ -21,6 +40,11 @@ typedef std::variant> Attr; */ typedef std::map Attrs; +/** + * Force a potentially lazy attribute to its resolved value. + */ +ResolvedAttr forceAttr(const Attr & attr); + Attrs jsonToAttrs(const nlohmann::json & json); nlohmann::json attrsToJSON(const Attrs & attrs); diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 66962a014879..5ee587c3e051 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -105,12 +105,13 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg for (const auto & [key, value] : attrs) { auto s = state.symbols.create(key); auto & vv = binds.alloc(s); + auto resolved = forceAttr(value); std::visit( overloaded{ [&vv, &state](const std::string & value) { vv.mkString(value, state.mem); }, [&vv](const uint64_t & value) { vv.mkInt(value); }, [&vv](const Explicit & value) { vv.mkBool(value.t); }}, - value); + resolved); } v.mkAttrs(binds); } From a08c14bf8b8dd91034a7fe7c1014b05aee12d772 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 21 Apr 2026 13:56:25 +0100 Subject: [PATCH 350/555] feat(libexpr): emit thunks for lazy fetcher attributes The "revCount" attribute is now emitted as a lazy attribute if libfetchers were to return it as such; as will be done in next commit. --- src/libexpr-tests/lazy-fetcher-attr.cc | 100 ++++++++++++++++++ src/libexpr-tests/meson.build | 1 + src/libexpr/include/nix/expr/fetch-tree.hh | 18 ++++ src/libexpr/include/nix/expr/meson.build | 1 + src/libexpr/primops/fetchTree.cc | 95 ++++++++++++++++- src/libfetchers/attrs.cc | 10 ++ src/libfetchers/include/nix/fetchers/attrs.hh | 5 + src/libflake/flake.cc | 1 + src/libflake/include/nix/flake/flake.hh | 8 -- 9 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 src/libexpr-tests/lazy-fetcher-attr.cc create mode 100644 src/libexpr/include/nix/expr/fetch-tree.hh diff --git a/src/libexpr-tests/lazy-fetcher-attr.cc b/src/libexpr-tests/lazy-fetcher-attr.cc new file mode 100644 index 000000000000..4c36424ecb55 --- /dev/null +++ b/src/libexpr-tests/lazy-fetcher-attr.cc @@ -0,0 +1,100 @@ +#include + +#include "nix/expr/fetch-tree.hh" +#include "nix/expr/tests/libexpr.hh" +#include "nix/fetchers/attrs.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/store/path.hh" + +namespace nix { + +class LazyFetcherAttrTest : public LibExprTest +{ +protected: + StorePath dummyPath() + { + return StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-test"}; + } +}; + +TEST_F(LazyFetcherAttrTest, nonLazyAttrProducesImmediateValue) +{ + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("revCount", uint64_t(5)); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 5); +} + +TEST_F(LazyFetcherAttrTest, lazyAttrProducesThunk) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(42); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + + // Not yet forced, so the lazy function should not have been called + EXPECT_EQ(calls, 0); + + // Force the thunk + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 42); + EXPECT_EQ(calls, 1); +} + +TEST_F(LazyFetcherAttrTest, lazyFunctionOnlyCalledOnAccess) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("lastModified", uint64_t(1000)); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(99); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + // Access lastModified, so should not trigger lazy revCount + auto * lmAttr = v.attrs()->get(state.symbols.create("lastModified")); + ASSERT_NE(lmAttr, nullptr); + state.forceValue(*lmAttr->value, noPos); + EXPECT_EQ(lmAttr->value->integer().value, 1000); + EXPECT_EQ(calls, 0); + + // Now access revCount + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 99); + EXPECT_EQ(calls, 1); +} + +} // namespace nix diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index 0b0a01c20654..d18d5a4830d5 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -51,6 +51,7 @@ sources = files( 'error_traces.cc', 'eval.cc', 'json.cc', + 'lazy-fetcher-attr.cc', 'main.cc', 'nix_api_expr.cc', 'nix_api_external.cc', diff --git a/src/libexpr/include/nix/expr/fetch-tree.hh b/src/libexpr/include/nix/expr/fetch-tree.hh new file mode 100644 index 000000000000..3eb8a01c0c5f --- /dev/null +++ b/src/libexpr/include/nix/expr/fetch-tree.hh @@ -0,0 +1,18 @@ +#pragma once + +#include "nix/expr/eval.hh" + +namespace nix { + +/** + * Convert a libfetchers `Input` to libexpr `Value`. + */ +void emitTreeAttrs( + EvalState & state, + const StorePath & storePath, + const fetchers::Input & input, + Value & v, + bool emptyRevFallback = false, + bool forceDirty = false); + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/meson.build b/src/libexpr/include/nix/expr/meson.build index 4213476fe73a..7334b42f8f17 100644 --- a/src/libexpr/include/nix/expr/meson.build +++ b/src/libexpr/include/nix/expr/meson.build @@ -20,6 +20,7 @@ headers = [ config_pub_h ] + files( 'eval-profiler.hh', 'eval-settings.hh', 'eval.hh', + 'fetch-tree.hh', 'function-trace.hh', 'gc-small-vector.hh', 'get-drvs.hh', diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index afd61e90fbb5..429009301db6 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,7 +1,9 @@ +#include "nix/expr/value.hh" #include "nix/fetchers/attrs.hh" #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" #include "nix/store/filetransfer.hh" @@ -19,6 +21,95 @@ namespace nix { +/** + * Adapter for putting libfetchers data into a thunk closure. + * Used as the argument to prim_forceLazyFetcherAttr in a lazy apply thunk. + */ +class LazyFetcherAttr : public ExternalValueBase, public gc_cleanup +{ + fetchers::LazyAttr lazy; + +public: + LazyFetcherAttr(fetchers::LazyAttr lazy) + : lazy(std::move(lazy)) + { + } + + fetchers::ResolvedAttr force() + { + return lazy->compute(); + } + +protected: + std::ostream & print(std::ostream & str) const override + { + unreachable(); + } + +public: + std::string showType() const override + { + unreachable(); + } + + std::string typeOf() const override + { + unreachable(); + } +}; + +/** + * Initialize a `Value` from a resolved fetcher attribute. + */ +static void resolvedAttrToValue(EvalState & state, Value & v, const fetchers::ResolvedAttr & resolved) +{ + std::visit( + overloaded{ + [&](const std::string & s) { v.mkString(s, state.mem); }, + [&](uint64_t n) { v.mkInt(n); }, + [&](const Explicit & b) { v.mkBool(b.t); }, + }, + resolved); +} + +/** + * internal primop: Force a LazyFetcherAttr external value. + */ +static void prim_forceLazyFetcherAttr(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + Value & arg = *args[0]; + + state.forceValue(arg, pos); + // We only construct this primop with LazyFetcherAttr preapplied. + assert(arg.type() == nExternal); + auto * ext = dynamic_cast(args[0]->external()); + assert(ext); + + resolvedAttrToValue(state, v, ext->force()); +} + +/** + * Emit a lazy thunk for a LazyAttr: mkApp(primop, externalValue). + */ +static void emitLazyAttrThunk(EvalState & state, const fetchers::LazyAttr & lazyAttr, Value & dest) +{ + // not user-callable (unregistered, internal) + static PrimOp forcePrimOp{ + .name = "__forceLazyFetcherAttr", + .arity = 1, + .impl = prim_forceLazyFetcherAttr, + .internal = true, + }; + + auto * vExt = state.allocValue(); + vExt->mkExternal(new LazyFetcherAttr(lazyAttr)); + + auto * vPrimOp = state.allocValue(); + vPrimOp->mkPrimOp(&forcePrimOp); + + dest.mkApp(vPrimOp, vExt); +} + void emitTreeAttrs( EvalState & state, const StorePath & storePath, @@ -51,7 +142,9 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem); } - if (auto revCount = input.getRevCount()) + if (auto revCount = maybeGetLazyAttr(input.attrs, "revCount")) + emitLazyAttrThunk(state, *revCount, attrs.alloc("revCount")); + else if (auto revCount = input.getRevCount()) attrs.alloc("revCount").mkInt(*revCount); else if (emptyRevFallback) attrs.alloc("revCount").mkInt(0); diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 39e7a0fd5d6c..f3ecc3a87bc0 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -51,6 +51,16 @@ nlohmann::json attrsToJSON(const Attrs & attrs) return json; } +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) + return {}; + if (auto v = std::get_if(&i->second)) + return *v; + return {}; +} + std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); diff --git a/src/libfetchers/include/nix/fetchers/attrs.hh b/src/libfetchers/include/nix/fetchers/attrs.hh index 60730d65d32b..8eede58086e5 100644 --- a/src/libfetchers/include/nix/fetchers/attrs.hh +++ b/src/libfetchers/include/nix/fetchers/attrs.hh @@ -45,6 +45,11 @@ typedef std::map Attrs; */ ResolvedAttr forceAttr(const Attr & attr); +/** + * Retrieve an attr, but only if it's a LazyAttr. + */ +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name); + Attrs jsonToAttrs(const nlohmann::json & json); nlohmann::json attrsToJSON(const Attrs & attrs); diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index e8dbf42f5451..03e68e1f995a 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -38,6 +38,7 @@ #include "nix/fetchers/input-cache.hh" #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index fd52dbebac5d..aa063a08e396 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -233,14 +233,6 @@ ref openEvalCache(EvalState & state, ref Date: Tue, 21 Apr 2026 14:08:14 +0100 Subject: [PATCH 351/555] feat(git): make revCount lazy revCount requires an expensive walk of the entire commit graph. Now it's wrapped in a LazyAttr so the cost is only paid when the Nix expression actually accesses `.revCount`. This also fixes fetching from shallow clones with fetchGit. Previously the eager revCount computation failed the whole fetch, but now `.outPath` and some other attributes succeed, while `.revCount` access fails (as expected for shallow repos). --- src/libfetchers/git.cc | 46 +++++++++++++++++++---------- tests/functional/fetchGitShallow.sh | 8 +++-- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 447b4a3694f7..6baf7d2e525c 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -13,6 +13,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/util/json-utils.hh" #include "nix/util/archive.hh" +#include "nix/util/memo.hh" #include "nix/util/mounted-source-accessor.hh" #include @@ -163,6 +164,13 @@ std::vector getPublicKeys(const Attrs & attrs) static const Hash nullRev{HashAlgorithm::SHA1}; +static LazyAttr makeLazyAttr(fun compute) +{ + return make_ref(LazyAttrComputation{ + .compute = memo(std::move(compute)), + }); +} + struct GitInputScheme : InputScheme { std::optional inputFromURL(const Settings & settings, const ParsedURL & url, bool requireTree) const override @@ -723,14 +731,12 @@ struct GitInputScheme : InputScheme } uint64_t getRevCount( - const Settings & settings, - const RepoInfo & repoInfo, - const std::filesystem::path & repoDir, - const Hash & rev) const + ref cache, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { - Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; + if (GitRepo::openRepo(repoDir, {})->isShallow()) + throw Error("'%s' is a shallow Git repository, so 'revCount' is not available", repoInfo.locationToArg()); - auto cache = settings.getCache(); + Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); @@ -745,6 +751,18 @@ struct GitInputScheme : InputScheme return revCount; } + LazyAttr lazyRevCount( + const Settings & settings, + const RepoInfo & repoInfo, + const std::filesystem::path & repoDir, + const Hash & rev) const + { + auto cache = settings.getCache(); + return makeLazyAttr([this, cache, repoInfo, repoDir, rev]() -> ResolvedAttr { + return getRevCount(cache, repoInfo, repoDir, rev); + }); + } + std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const { auto head = std::visit( @@ -891,13 +909,6 @@ struct GitInputScheme : InputScheme auto repo = GitRepo::openRepo(repoDir, {}); - auto isShallow = repo->isShallow(); - - if (isShallow && !getShallowAttr(input)) - throw Error( - "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", - repoInfo.locationToArg()); - // FIXME: check whether rev is an ancestor of ref? auto rev = *input.getRev(); @@ -911,7 +922,7 @@ struct GitInputScheme : InputScheme if (!getShallowAttr(input)) { /* Like lastModified, skip revCount if supplied by the caller. */ if (!input.attrs.contains("revCount")) - input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoDir, rev)); } printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); @@ -1033,8 +1044,11 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", rev.gitRev()); if (!getShallowAttr(input)) { - input.attrs.insert_or_assign( - "revCount", rev == nullRev ? 0 : getRevCount(settings, repoInfo, repoPath, rev)); + if (rev == nullRev) { + input.attrs.insert_or_assign("revCount", uint64_t(0)); + } else { + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoPath, rev)); + } } verifyCommit(input, repo); diff --git a/tests/functional/fetchGitShallow.sh b/tests/functional/fetchGitShallow.sh index 6b91d60cd9e3..67a8e3d3bd16 100644 --- a/tests/functional/fetchGitShallow.sh +++ b/tests/functional/fetchGitShallow.sh @@ -29,9 +29,11 @@ git -C "$TEST_ROOT/shallow-parent" commit -m "Branch commit" # Make a shallow clone (depth=1) git clone --depth 1 "file://$TEST_ROOT/shallow-parent" "$TEST_ROOT/shallow-clone" -# Test 1: Fetching a shallow repo shouldn't work by default, because we can't -# return a revCount. -(! nix eval --impure --raw --expr "(builtins.fetchGit { url = \"$TEST_ROOT/shallow-clone\"; ref = \"dev\"; }).outPath") +# Test 1: Fetching a shallow repo succeeds for outPath because revCount is lazy. +path1=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"$TEST_ROOT/shallow-clone\"; ref = \"dev\"; }).outPath") +[[ -d "$path1" ]] +# But accessing revCount on a shallow clone fails. +(! nix eval --impure --expr "(builtins.fetchGit { url = \"$TEST_ROOT/shallow-clone\"; ref = \"dev\"; }).revCount" 2>/dev/null) # Test 2: But you can request a shallow clone, which won't return a revCount. path=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow-clone\"; ref = \"dev\"; shallow = true; }).outPath") From 0c7b61dadd1b575bb8abc36e66a53e339988d200 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 30 Apr 2026 02:07:29 +0200 Subject: [PATCH 352/555] fix(flake): don't force revCount in fingerprint The flake eval cache fingerprint eagerly forced revCount via getRevCount(), which defeats lazy revCount on shallow clones. Since revCount is functionally determined by rev (already part of the fingerprint), we only need to include its *presence* in the fingerprint, not its value. Note that before these changes, Nix had a bug where it could produce a wrong revcount on shallow clones. It arguably should have inferred `shallow = true;` but it didn't and I'm not changing that behavior either. Leaving it at its default is more "pure" in a sense, but a case could certainly be made to let the CLI infer the value that works automagically. (Purity in the CLI is a spectrum; system attribute selection could be argued to be impure too, for instance.) --- src/libflake/flake.cc | 10 +++++++--- tests/functional/fetchGitShallow.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 03e68e1f995a..c34fa2d0a1ba 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -991,9 +991,13 @@ std::optional LockedFlake::getFingerprint(Store & store, const fetc /* Include revCount and lastModified because they're not necessarily implied by the content fingerprint (e.g. for - tarball flakes) but can influence the evaluation result. */ - if (auto revCount = flake.lockedRef.input.getRevCount()) - *fingerprint += fmt(";revCount=%d", *revCount); + tarball flakes) but can influence the evaluation result. + For revCount, we only include its presence (not its value) + because the value is functionally determined by rev, which + is already part of the fingerprint. This avoids forcing a + lazy revCount computation. */ + if (flake.lockedRef.input.attrs.contains("revCount")) + *fingerprint += ";hasRevCount"; if (auto lastModified = flake.lockedRef.input.getLastModified()) *fingerprint += fmt(";lastModified=%d", *lastModified); diff --git a/tests/functional/fetchGitShallow.sh b/tests/functional/fetchGitShallow.sh index 67a8e3d3bd16..0da36ee9bac9 100644 --- a/tests/functional/fetchGitShallow.sh +++ b/tests/functional/fetchGitShallow.sh @@ -65,3 +65,32 @@ fi # Verify that we can shallow fetch the worktree git -C "$TEST_ROOT/shallow-worktree" rev-list --count HEAD >/dev/null nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$TEST_ROOT/shallow-worktree\"; shallow = true; }).rev" + +# Test 5: nix build --dry-run on a shallow clone must not force revCount. +# The flake fingerprint must not eagerly evaluate revCount, because that +# would fail (or produce wrong results) on shallow clones. +# Nor should it generate a complete lock file that serializes the root node. +# We remove the parent repo to ensure that a future improvement that +# tries to fetch missing history can't paper over the issue. +createGitRepo "$TEST_ROOT/shallow-build-parent" +echo "" > "$TEST_ROOT/shallow-build-parent/file.txt" +git -C "$TEST_ROOT/shallow-build-parent" add file.txt +git -C "$TEST_ROOT/shallow-build-parent" commit -m "first" +cat > "$TEST_ROOT/shallow-build-parent/flake.nix" < \\\$out" ]; + }; + }; +} +EOF +git -C "$TEST_ROOT/shallow-build-parent" add flake.nix +git -C "$TEST_ROOT/shallow-build-parent" commit -m "add flake" +git clone --depth 1 "file://$TEST_ROOT/shallow-build-parent" "$TEST_ROOT/shallow-build-clone" +rm -rf "$TEST_ROOT/shallow-build-parent" +nix build --dry-run "git+file://$TEST_ROOT/shallow-build-clone" From ad649e34a3e630c3a91bab5217e422439edcee08 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 5 May 2026 00:29:59 +0200 Subject: [PATCH 353/555] chore: remove redundant comment --- src/libutil/include/nix/util/fun.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/include/nix/util/fun.hh b/src/libutil/include/nix/util/fun.hh index c480ffe71711..59a7cc87939c 100644 --- a/src/libutil/include/nix/util/fun.hh +++ b/src/libutil/include/nix/util/fun.hh @@ -1,6 +1,5 @@ #pragma once ///@file -// Tests in: src/libutil-tests/fun.cc #include #include From 7609abe60a6359dc349cf8ab645c910a5eae0293 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 13:11:19 +0200 Subject: [PATCH 354/555] Document how to use GC roots safely --- src/libstore/include/nix/store/store-api.hh | 41 +++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index bfd4ffce2181..b6114cce65e7 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -816,6 +816,47 @@ public: /** * Add a store path as a temporary root of the garbage collector. * The root disappears as soon as we exit. + * Before exiting, if you want to avoid the path being GC'ed, you either have to make it a permanent root using + * `LocalFSStore::addPermRoot()`, or make sure it's reachable from a permanent root (e.g. by adding it as a + * reference of a reachable path). + * + * To avoid races, you should call either this function or `LocalFSStore::addPermRoot()` *before* creating and using + * a store path, e.g. + * + * ```c++ + * auto path = store.computeStorePath(...); + * store->addTempRoot(path); + * if (!store->isValidPath(path)) + * store->addToStore(...); + * ``` + * + * By contrast, registering a root just before *using* a path is not sufficient to prevent GC races. For + * instance, don't do this: + * + * ```c++ + * store->addTempRoot(path); + * auto drv = store->readDerivation(path); + * ``` + * + * since the path may be GC'ed just before the call to `addTempRoot()`. + * + * Note that `addToStore()` implicitly calls `addTempRoot()`, so you don't need to call it yourself if you're + * calling `addToStore()` unconditionally. + * + * It is generally the responsibility of the caller of Nix APIs and CLI tools to ensure that paths are reachable by + * the garbage collector. For example, `buildPath(drvPath)` does not need to register *drvPath* as a GC root, since + * that's the responsibility of the caller, and it would be too late for `buildPath()` to do so anyway. Thus, this + * can race: + * ```console + * drv=$(nix-instantiate foo.nix) + * nix-store -r $drv + * ``` + * whereas this is safe: + * ```console + * nix-instantiate foo.nix --add-root ./drv + * nix-store -r ./drv + * ``` + * */ virtual void addTempRoot(const StorePath & path) { From b87d9e810b759a703c7d72c0cbe04414b2edcf22 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 13:11:38 +0200 Subject: [PATCH 355/555] Add TODO item --- src/libfetchers/fetch-to-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 3af2d4c83e88..09112a965584 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -82,6 +82,7 @@ std::pair fetchToStore2( auto [storePath, hash] = mode == FetchMode::DryRun ? [&]() { + // FIXME: we may have already computed this above. auto [storePath, hash] = store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2); debug( From e2490c72941e6eb34af2455e3bc459644be55755 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 19:42:50 +0200 Subject: [PATCH 356/555] StoreDirConfig::parseStorePath(): Don't crash on empty paths --- src/libstore/store-dir-config.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 61f82029d79f..962a3830f166 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -8,6 +8,8 @@ namespace nix { StorePath StoreDirConfig::parseStorePath(std::string_view path) const { + if (path.empty()) + throw BadStorePath("empty path is not a valid store path"); // On Windows, `/nix/store` is not a canonical path. More broadly it // is unclear whether this function should be using the native // notion of a canonical path at all. For example, it makes to From 8f3d7023cb4c168fa45dedac120efdb46794ed10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 19:43:01 +0200 Subject: [PATCH 357/555] canonPath(): Don't crash on empty paths --- src/libutil-tests/file-system.cc | 2 +- src/libutil/file-system.cc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index f290e1178090..417622a47cf2 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -114,7 +114,7 @@ TEST(canonPath, requiresAbsolutePath) ASSERT_ANY_THROW(canonPath("."sv)); ASSERT_ANY_THROW(canonPath(".."sv)); ASSERT_ANY_THROW(canonPath("../"sv)); - ASSERT_DEATH({ canonPath(""sv); }, "!path.empty\\(\\)"); + ASSERT_ANY_THROW(canonPath(""sv)); } /* ---------------------------------------------------------------------------- diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index b3087700d733..b6ad1e128046 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -95,7 +95,8 @@ absPath(const std::filesystem::path & path0, const std::filesystem::path * dir, std::filesystem::path canonPath(const std::filesystem::path & path, bool resolveSymlinks) { - assert(!path.empty()); + if (path.empty()) + throw Error("cannot canonicalise an empty path"); if (!path.is_absolute()) throw Error("not an absolute path: %s", PathFmt(path)); From 1a4d5782ffd33f6069295b3b1488bb402e34a9c2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 6 May 2026 03:02:45 +0300 Subject: [PATCH 358/555] Clean up and deduplicate some Logger code Puts some logger classes in anonymous namespaces. This also appeases Wweak-vtables and avoids making those symbols visible in the DSO. Also marks those classes as final and deduplicates storePathToNameWithoutDrvSuffix in the progress bar code. --- src/libmain/progress-bar.cc | 35 ++++++++++++++++++++++------------- src/libutil/tee-logger.cc | 7 ++++++- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index a8044a240d51..f81e90a06e7a 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -2,8 +2,10 @@ #include "nix/util/terminal.hh" #include "nix/util/sync.hh" #include "nix/util/signals.hh" -#include "nix/store/store-api.hh" +#include "nix/store/path.hh" +#include "nix/util/file-system.hh" #include "nix/store/names.hh" +#include "nix/util/util.hh" #include #include @@ -13,17 +15,19 @@ namespace nix { +namespace { + static std::string_view getS(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tString); + if (n >= fields.size() || fields[n].type != Logger::Field::tString) + throw Error("could not get expected log field of type 'string' at index %d", n); return fields[n].s; } static uint64_t getI(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tInt); + if (n >= fields.size() || fields[n].type != Logger::Field::tInt) + throw Error("could not get expected log field of type 'int' at index %d", n); return fields[n].i; } @@ -34,10 +38,17 @@ static std::string_view storePathToName(std::string_view path) return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1); } -class ProgressBar : public Logger +static std::string_view storePathToNameWithoutDrvSuffix(std::string_view path) { -private: + auto res = storePathToName(path); + if (hasSuffix(res, drvExtension)) + res.remove_suffix(drvExtension.size()); + return res; +} +class ProgressBar final : public Logger +{ +private: struct ActInfo { std::string s, lastLine, phase; @@ -223,9 +234,7 @@ class ProgressBar : public Logger state->activitiesByType[type].its.emplace(act, i); if (type == actBuild) { - std::string name(storePathToName(getS(fields, 0))); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); auto machineName = getS(fields, 1); if (machineName != "") @@ -250,9 +259,7 @@ class ProgressBar : public Logger } if (type == actPostBuildHook) { - auto name = storePathToName(getS(fields, 0)); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); i->name = DrvName(name).name; } @@ -687,6 +694,8 @@ class ProgressBar : public Logger } }; +} // namespace + std::unique_ptr makeProgressBar() { return std::make_unique(isTTY()); diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index 8433168a5a82..8e27714e3f40 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -2,10 +2,13 @@ namespace nix { -struct TeeLogger : Logger +namespace { + +class TeeLogger final : public Logger { std::vector> loggers; +public: TeeLogger(std::vector> && loggers) : loggers(std::move(loggers)) { @@ -94,6 +97,8 @@ struct TeeLogger : Logger } }; +} // namespace + std::unique_ptr makeTeeLogger(std::unique_ptr mainLogger, std::vector> && extraLoggers) { From a253942be956f946744c3da3f4e694934b56cc51 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 6 May 2026 03:07:07 +0300 Subject: [PATCH 359/555] libmain: Hide/unhide cursor in the progress bar Uses corresponding ANSI/VT escape sequences to hide/unhide the cursor when the output is a TTY. nom does the same AFAICT. --- src/libmain/progress-bar.cc | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index f81e90a06e7a..512ba4faad67 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -107,6 +107,18 @@ class ProgressBar final : public Logger std::unique_ptr interruptCallback; + void hideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25l"); + } + + void unhideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25h"); + } + public: ProgressBar(bool isTTY) @@ -116,6 +128,7 @@ class ProgressBar final : public Logger redraw("\rshutting down\e[K"); })) { + hideCursorIfNeeded(); state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); @@ -142,6 +155,7 @@ class ProgressBar final : public Logger if (state->active) { state->active = false; clearProgressDisplay(); + unhideCursorIfNeeded(); updateCV.notify_one(); quitCV.notify_one(); } @@ -159,8 +173,10 @@ class ProgressBar final : public Logger return; } - if (state->active) + if (state->active) { clearProgressDisplay(); + unhideCursorIfNeeded(); + } } void resume() override @@ -173,8 +189,10 @@ class ProgressBar final : public Logger state->suspensions--; } if (state->suspensions == 0) { - if (state->active) + if (state->active) { clearProgressDisplay(); + hideCursorIfNeeded(); + } state->haveUpdate = true; updateCV.notify_one(); } @@ -681,7 +699,9 @@ class ProgressBar final : public Logger return {}; invalidateRedrawCache(); std::cerr << fmt("\r\e[K%s ", msg); + unhideCursorIfNeeded(); auto s = trim(readLine(getStandardInput(), true)); + hideCursorIfNeeded(); if (s.size() != 1) return {}; draw(*state); From 4fa3a8cad2b651ab0bb3a30d78d9499c8ae16216 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 7 May 2026 00:41:57 +0300 Subject: [PATCH 360/555] libstore: Use member fileTransfer instead of global getFileTransfer() in the s3 store Ideally we'd get rid of the global file transfer object altogether and this moves us in that direction. fileTransfer was already a member of the HttpBinaryCacheStore, so just use that instead. In practice it pointed to the same singleton object though (it's primarily used for tests now). --- src/libstore/s3-binary-cache-store.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f6d300c0936f..54bf1d0ad7c3 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -322,7 +322,7 @@ std::string S3BinaryCacheStore::createMultipartUpload( std::move(headers->begin(), headers->end(), std::back_inserter(req.headers)); } - auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + auto result = fileTransfer->enqueueFileTransfer(req).get(); std::regex uploadIdRegex("([^<]+)"); std::smatch match; @@ -353,7 +353,7 @@ S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, req.data = {payload}; req.mimeType = "application/octet-stream"; - auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + auto result = fileTransfer->enqueueFileTransfer(req).get(); if (result.etag.empty()) { throw Error("S3 UploadPart response missing ETag for part %d", partNumber); @@ -374,7 +374,7 @@ void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_ req.uri = VerbatimURL(url); req.method = HttpMethod::Delete; - getFileTransfer()->enqueueFileTransfer(req).get(); + fileTransfer->enqueueFileTransfer(req).get(); } catch (...) { ignoreExceptionInDestructor(); } @@ -407,7 +407,7 @@ void S3BinaryCacheStore::completeMultipartUpload( req.data = {payload}; req.mimeType = "text/xml"; - getFileTransfer()->enqueueFileTransfer(req).get(); + fileTransfer->enqueueFileTransfer(req).get(); debug("S3 multipart upload completed: %d parts uploaded for '%s'", partEtags.size(), key); } From 220ccc746d156b5ea62c45b42e91a4b6f380f6ef Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 7 May 2026 01:26:05 +0200 Subject: [PATCH 361/555] fix: Restore fingerprint value for untrustworthy revCount values It was fine for trustworthy revCounts, but technically the functional dependency described does not always hold up, or we can not trust it to. - buggy tarball providers - merge conflicts in lock files or other kinds of chaos in "user space" --- src/libflake/flake.cc | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index c34fa2d0a1ba..bd3ea4acbc65 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -989,15 +989,24 @@ std::optional LockedFlake::getFingerprint(Store & store, const fetc *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); - /* Include revCount and lastModified because they're not - necessarily implied by the content fingerprint (e.g. for - tarball flakes) but can influence the evaluation result. - For revCount, we only include its presence (not its value) - because the value is functionally determined by rev, which - is already part of the fingerprint. This avoids forcing a - lazy revCount computation. */ - if (flake.lockedRef.input.attrs.contains("revCount")) - *fingerprint += ";hasRevCount"; + if (auto revCount = get(flake.lockedRef.input.attrs, "revCount")) { + if (std::get_if(revCount)) { + /* A lazy revCount is computed by the fetcher, so its + value is functionally determined by `rev`. We only + need to record its presence, not force its value. + + This means a lazy and a concrete revCount that would + resolve to the same value produce different + fingerprints, sacrificing some cache hits to avoid + the cost of forcing. */ + *fingerprint += ";hasRevCount"; + } else if (auto n = flake.lockedRef.input.getRevCount()) { + /* A concrete revCount comes from a lockfile or explicit + user input. The fetcher passes it through as-is, so + it can affect evaluation and must be fingerprinted. */ + *fingerprint += fmt(";revCount=%d", *n); + } + } if (auto lastModified = flake.lockedRef.input.getLastModified()) *fingerprint += fmt(";lastModified=%d", *lastModified); From 2a42376bbe96340cf5f94b96d881225b5366bf59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 8 May 2026 10:54:10 +0000 Subject: [PATCH 362/555] tests/nixos: serve fetchers-substitute cache via nginx nix-serve depends on the perl bindings, which are slated for removal, and nix-serve-ng adds an extra dependency just to serve a static binary cache over HTTP. Replace it with plain nginx serving a file:// binary cache populated via 'nix copy', which is simpler and only relies on components already in nixpkgs. :house: Remote-Dev: homespace --- tests/nixos/fetchers-substitute.nix | 83 +++++++++-------------------- 1 file changed, 24 insertions(+), 59 deletions(-) diff --git a/tests/nixos/fetchers-substitute.nix b/tests/nixos/fetchers-substitute.nix index b61a3929af80..0959512e1db6 100644 --- a/tests/nixos/fetchers-substitute.nix +++ b/tests/nixos/fetchers-substitute.nix @@ -11,22 +11,18 @@ "fetch-tree" ]; - networking.firewall.allowedTCPPorts = [ 5000 ]; + networking.firewall.allowedTCPPorts = [ 80 ]; - # TODO stop using this, because it has to depend on an older version of Nix that still has the perl bindings. - services.nix-serve = { + systemd.tmpfiles.rules = [ "d /var/cache/binary-cache 0755 root root -" ]; + + services.nginx = { enable = true; - secretKeyFile = - let - key = pkgs.writeTextFile { - name = "secret-key"; - text = '' - substituter:SerxxAca5NEsYY0DwVo+subokk+OoHcD9m6JwuctzHgSQVfGHe6nCc+NReDjV3QdFYPMGix4FMg0+K/TM1B3aA== - ''; - }; - in - "${key}"; + virtualHosts."substituter".root = "/var/cache/binary-cache"; }; + + environment.etc."nix/secret-key".text = '' + substituter:SerxxAca5NEsYY0DwVo+subokk+OoHcD9m6JwuctzHgSQVfGHe6nCc+NReDjV3QdFYPMGix4FMg0+K/TM1B3aA== + ''; }; nodes.importer = @@ -39,13 +35,12 @@ "nix-command" "fetch-tree" ]; - substituters = lib.mkForce [ "http://substituter:5000" ]; + substituters = lib.mkForce [ "http://substituter" ]; trusted-public-keys = lib.mkForce [ "substituter:EkFXxh3upwnPjUXg41d0HRWDzBoseBTINPiv0zNQd2g=" ]; }; }; - testScript = - { nodes }: # python + testScript = # python '' import json import os @@ -53,11 +48,11 @@ start_all() substituter.wait_for_unit("multi-user.target") + importer.wait_for_unit("multi-user.target") - ########################################## - # Test 1: builtins.fetchurl with substitution - ########################################## + binary_cache = "file:///var/cache/binary-cache?secret-key=/etc/nix/secret-key" + # builtins.fetchurl is substituted missing_file = "/only-on-substituter.txt" substituter.succeed(f"echo 'this should only exist on the substituter' > {missing_file}") @@ -75,11 +70,8 @@ file_store_path = json.loads(file_store_path_json) - substituter.succeed(f"nix store sign --key-file ${nodes.substituter.services.nix-serve.secretKeyFile} {file_store_path}") - - importer.wait_for_unit("multi-user.target") + substituter.succeed(f"nix copy --to '{binary_cache}' {file_store_path}") - print("Testing fetchurl with substitution...") importer.succeed(f""" nix-instantiate -vvvvv --eval --json --read-write-mode --expr ' builtins.fetchurl {{ @@ -88,26 +80,19 @@ }} ' """) - print("✓ fetchurl substitution works!") - - ########################################## - # Test 2: builtins.fetchTarball with substitution - ########################################## + # builtins.fetchTarball is substituted missing_tarball = "/only-on-substituter.tar.gz" - # Create a directory with some content substituter.succeed(""" mkdir -p /tmp/test-tarball echo 'Hello from tarball!' > /tmp/test-tarball/hello.txt echo 'Another file' > /tmp/test-tarball/file2.txt """) - - # Create a tarball substituter.succeed(f"tar czf {missing_tarball} -C /tmp test-tarball") - # For fetchTarball, we need to first fetch it without hash to get the store path, - # then compute the NAR hash of that path + # Fetch once without a hash to learn the store path, then derive the + # hashes the importer needs. tarball_store_path_json = substituter.succeed(f""" nix-instantiate --eval --json --read-write-mode --expr ' builtins.fetchTarball {{ @@ -118,22 +103,14 @@ tarball_store_path = json.loads(tarball_store_path_json) - # Get the NAR hash of the unpacked tarball in SRI format path_info_json = substituter.succeed(f"nix path-info --json-format 2 --json {tarball_store_path}").strip() path_info_dict = json.loads(path_info_json)["info"] - # narHash is already in SRI format tarball_hash_sri = path_info_dict[os.path.basename(tarball_store_path)]["narHash"] - print(f"Tarball NAR hash (SRI): {tarball_hash_sri}") - # Also get the old format hash for fetchTarball (which uses sha256 parameter) tarball_hash = substituter.succeed(f"nix-store --query --hash {tarball_store_path}").strip() - # Sign the tarball's store path - substituter.succeed(f"nix store sign --recursive --key-file ${nodes.substituter.services.nix-serve.secretKeyFile} {tarball_store_path}") + substituter.succeed(f"nix copy --to '{binary_cache}' {tarball_store_path}") - # Now try to fetch the same tarball on the importer - # The file doesn't exist locally, so it should be substituted - print("Testing fetchTarball with substitution...") result = importer.succeed(f""" nix-instantiate -vvvvv --eval --json --read-write-mode --expr ' builtins.fetchTarball {{ @@ -144,23 +121,14 @@ """) result_path = json.loads(result) - print(f"✓ fetchTarball substitution works! Result: {result_path}") - # Verify the content is correct - # fetchTarball strips the top-level directory if there's only one content = importer.succeed(f"cat {result_path}/hello.txt").strip() assert content == "Hello from tarball!", f"Content mismatch: {content}" - print("✓ fetchTarball content verified!") - - ########################################## - # Test 3: Verify fetchTree does NOT substitute (preserves metadata) - ########################################## - - print("Testing that fetchTree without __final does NOT use substitution...") - # fetchTree with just narHash (not __final) should try to download, which will fail - # since the file doesn't exist on the importer - exit_code = importer.fail(f""" + # fetchTree does NOT substitute non-final inputs: without __final it + # must perform the real fetch (to preserve metadata like lastModified), + # so it fails since the file only exists on the substituter. + output = importer.fail(f""" nix-instantiate --eval --json --read-write-mode --expr ' builtins.fetchTree {{ type = "tarball"; @@ -170,9 +138,6 @@ ' 2>&1 """) - # Should fail with "does not exist" since it tries to download instead of substituting - assert "does not exist" in exit_code or "Couldn't open file" in exit_code, f"Expected download failure, got: {exit_code}" - print("✓ fetchTree correctly does NOT substitute non-final inputs!") - print(" (This preserves metadata like lastModified from the actual fetch)") + assert "does not exist" in output or "Couldn't open file" in output, f"Expected download failure, got: {output}" ''; } From 67e442bbbf0ea04b206305b290463ae210d515b4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 15:57:32 +0300 Subject: [PATCH 363/555] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'https://releases.nixos.org/nixos/25.11/nixos-25.11.6495.e764fc9a4058/nixexprs.tar.xz?narHash=sha256-jEA8WggGKtMFeNeCKq3NK8cLEjJmG6/RLUElYYbBZ0E%3D' (2026-02-24) → 'https://releases.nixos.org/nixos/25.11/nixos-25.11.10470.0c88e1f2bdb9/nixexprs.tar.xz?narHash=sha256-amc4Y3GF3%2BanUi7IJeLVzf7hVqLb3ZqCGzYtkVyp7Qw%3D' (2026-05-05) --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 1212049c3bea..95b771a30c1a 100644 --- a/flake.lock +++ b/flake.lock @@ -60,11 +60,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1771903837, - "narHash": "sha256-jEA8WggGKtMFeNeCKq3NK8cLEjJmG6/RLUElYYbBZ0E=", - "rev": "e764fc9a405871f1f6ca3d1394fb422e0a0c3951", + "lastModified": 1778003029, + "narHash": "sha256-amc4Y3GF3+anUi7IJeLVzf7hVqLb3ZqCGzYtkVyp7Qw=", + "rev": "0c88e1f2bdb93d5999019e99cb0e61e1fe2af4c5", "type": "tarball", - "url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.6495.e764fc9a4058/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.10470.0c88e1f2bdb9/nixexprs.tar.xz" }, "original": { "type": "tarball", From 31f253b136d265bed12273f4a9de5388095936b4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:10:51 +0300 Subject: [PATCH 364/555] tests: Fix daemon compat tests for structured attrs --- tests/functional/structured-attrs.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index 321cb6107992..d0b41670967b 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -51,15 +51,17 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet " hacky=$(nix-instantiate --expr "$hackyExpr") nix derivation show "$hacky" | jq --exit-status '.derivations."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}' -# Test warning for non-object exportReferencesGraph in structured attrs -# shellcheck disable=SC2016 -expectStderr 0 nix-build --no-out-link --expr ' - with import ./config.nix; - mkDerivation { - name = "export-graph-non-object"; - __structuredAttrs = true; - exportReferencesGraph = [ "foo" "bar" ]; - builder = "/bin/sh"; - args = ["-c" "echo foo > ${builtins.placeholder "out"}"]; - } -' | grepQuiet "warning:.*exportReferencesGraph.*not a JSON object" +if isDaemonNewer "2.34pre"; then + # Test warning for non-object exportReferencesGraph in structured attrs + # shellcheck disable=SC2016 + expectStderr 0 nix-build --no-out-link --expr ' + with import ./config.nix; + mkDerivation { + name = "export-graph-non-object"; + __structuredAttrs = true; + exportReferencesGraph = [ "foo" "bar" ]; + builder = "/bin/sh"; + args = ["-c" "echo foo > ${builtins.placeholder "out"}"]; + } + ' | grepQuiet "warning:.*exportReferencesGraph.*not a JSON object" +fi From acadae1ee65637bc6003813456b5cfe37c38fe05 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:19:42 +0300 Subject: [PATCH 365/555] packaging: Use libgit2 >= 1.9.3 It's been a while since libgit2 has been released. No notable features in this release though - just bugfixes. Unstable has been updated in [1]. [1]: https://github.com/NixOS/nixpkgs/pull/517329 --- packaging/dependencies.nix | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index b44f6ae46c80..68e44a925f79 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -62,6 +62,21 @@ scope: { useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); }; + libgit2 = + if lib.versionAtLeast pkgs.libgit2.version "1.9.3" then + pkgs.libgit2 + else + # Grab newer libgit2. + pkgs.libgit2.overrideAttrs rec { + version = "1.9.3"; + src = pkgs.fetchFromGitHub { + owner = "libgit2"; + repo = "libgit2"; + tag = "v${version}"; + hash = "sha256-nJrRdPs86oGNL4W2CJb16oSUgfzYr9A2i5sw9BAehME="; + }; + }; + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. boost = (pkgs.boost.override { From bd2d65991100386983d9dbdd776e1afdb811383b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:26:22 +0300 Subject: [PATCH 366/555] packaging: Use mimalloc >= 3.3.2 --- packaging/dependencies.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 68e44a925f79..6a1873d9b3dd 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -17,16 +17,16 @@ scope: { inherit stdenv; mimalloc = - if lib.versionAtLeast pkgs.mimalloc.version "3.3.0" then + if lib.versionAtLeast pkgs.mimalloc.version "3.3.2" then pkgs.mimalloc else pkgs.mimalloc.overrideAttrs rec { - version = "3.3.0"; + version = "3.3.2"; src = pkgs.fetchFromGitHub { owner = "microsoft"; repo = "mimalloc"; tag = "v${version}"; - hash = "sha256-xy9gPihw3xvhnd6BrCYfMnnRp5dPSodynKRToYwxuzg="; + hash = "sha256-GZ37qQVDe9jgMb4Coe5oKvgaLTspZDlSkS5rdy1MfUU="; }; }; From c5aa96ef10ba7fffb581c5ea067a4f436ebe8283 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:29:15 +0300 Subject: [PATCH 367/555] packaging: Remove .broken override for libcurl Out nixpkgs is new enough now. --- packaging/dependencies.nix | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 6a1873d9b3dd..f531b709eec4 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -44,19 +44,13 @@ scope: { NIX_CFLAGS_COMPILE = "-DINITIAL_MARK_STACK_SIZE=1048576"; }); - curl = - (pkgs.curl.override { - http3Support = !pkgs.stdenv.hostPlatform.isWindows; - # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. - zstdSupport = true; - brotliSupport = true; - zlibSupport = true; - }).overrideAttrs - { - # TODO: Fix in nixpkgs. Static build with brotli is marked as broken, but it's not the case. - # Remove once https://github.com/NixOS/nixpkgs/pull/494111 lands in the 25.11 channel. - meta.broken = false; - }; + curl = pkgs.curl.override { + http3Support = !pkgs.stdenv.hostPlatform.isWindows; + # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. + zstdSupport = true; + brotliSupport = true; + zlibSupport = true; + }; libblake3 = pkgs.libblake3.override { useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); From 3fd52d9a32ab70251dbad20575cc42b985c8cd5b Mon Sep 17 00:00:00 2001 From: Felix Stupp Date: Fri, 8 May 2026 22:05:58 +0000 Subject: [PATCH 368/555] doc: add documentation to __addErrorContext primop This adds .args and .doc fields to the primop registration for __addErrorContext, making it appear in the builtins reference manual. --- src/libexpr/primops.cc | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 297b2bf234de..08b4fd76acc5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1047,7 +1047,35 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value ** a static RegisterPrimOp primop_addErrorContext( PrimOp{ .name = "__addErrorContext", + .args = {"context", "value"}, .arity = 2, + .doc = R"( + Evaluate *context*, which can be coerced to a string, + and append it to any error or stack traces displayed while evaluating *value*. + Then return *value*. + + This function is useful for providing helpful context in complex Nix expressions + when the evaluation of *value* fails. + The additional context is applied when evaluating *value* itself fails, + not when attributes or elements of *value* are evaluated. + + For example, the module system from nixpkgs uses this to show + the relevant information about the options that were evaluating + when an error occurs. + + ```nix-repl + nix-repl> addErrorContext "while evaluating foo" (throw "bar") + error: + … while evaluating foo + + … while calling the 'throw' builtin + at «string»:1:56: + 1| with builtins; addErrorContext "while evaluating foo" (throw "bar") + | ^ + + error: bar + ``` + )", // The normal trace item is redundant .addTrace = false, .impl = prim_addErrorContext, From cabb895f190c33fda1871eb110547191f3481a06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 May 2026 22:12:25 +0000 Subject: [PATCH 369/555] build(deps): bump cachix/install-nix-action from 31.10.5 to 31.10.6 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.10.5 to 31.10.6. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/ab739621df7a23f52766f9ccc97f38da6b7af14f...8aa03977d8d733052d78f4e008a241fd1dbf36b3) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.10.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49fff49b9737..2f294211ea74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,7 +189,7 @@ jobs: - name: Looking up the installer tarball URL id: installer-tarball-url run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 + - uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6 if: ${{ !matrix.rust-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} @@ -255,7 +255,7 @@ jobs: id: installer-tarball-url run: | echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 + - uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6 with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} From fcbe6cc5bffb374482b56d7b3445e2040a3972db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 May 2026 22:12:30 +0000 Subject: [PATCH 370/555] build(deps): bump aws-actions/configure-aws-credentials Bumps [aws-actions/configure-aws-credentials](https://github.com/aws-actions/configure-aws-credentials) from 5.1.1 to 6.1.1. - [Release notes](https://github.com/aws-actions/configure-aws-credentials/releases) - [Changelog](https://github.com/aws-actions/configure-aws-credentials/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-actions/configure-aws-credentials/compare/61815dcd50bd041e203e49132bacad1fd04d2708...d979d5b3a71173a29b74b5b88418bfda9437d885) --- updated-dependencies: - dependency-name: aws-actions/configure-aws-credentials dependency-version: 6.1.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/upload-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml index f00dce4a5c6b..f7da2f66364d 100644 --- a/.github/workflows/upload-release.yml +++ b/.github/workflows/upload-release.yml @@ -34,7 +34,7 @@ jobs: # get the same uberhack that nix-shell has to support it. echo "NIX_PATH=nixpkgs=$NIXPKGS_PATH" >> "$GITHUB_ENV" - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 + uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 with: role-to-assume: "arn:aws:iam::080433136561:role/nix-release" role-session-name: nix-release-oidc-${{ github.run_id }} From 5a870acfc9f5bccb9b50b2f6b1f541ecdfbfb1a6 Mon Sep 17 00:00:00 2001 From: znmz <267793835+znmz@users.noreply.github.com> Date: Mon, 11 May 2026 16:27:20 +0300 Subject: [PATCH 371/555] Fix typos in all .md, .sh, .nix files Also fixes one typo in each of the "remove_before_wrapper.py" "build-trace-entry-v3.yaml" "head.hbs" "sender.c" files --- doc/manual/remove_before_wrapper.py | 2 +- doc/manual/source/command-ref/env-common.md | 2 +- .../protocols/json/schema/build-trace-entry-v3.yaml | 2 +- doc/manual/source/release-notes/rl-2.30.md | 2 +- doc/manual/source/release-notes/rl-2.32.md | 2 +- doc/manual/source/store/file-system-object.md | 2 +- doc/manual/theme/head.hbs | 2 +- packaging/everything.nix | 2 +- scripts/create-darwin-volume.sh | 4 ++-- scripts/install-multi-user.sh | 4 ++-- tests/functional/binary-cache.sh | 2 +- tests/functional/build-delete.sh | 2 +- tests/functional/build.sh | 4 ++-- tests/functional/characterisation-test-infra.sh | 6 +++--- tests/functional/common/functions.sh | 2 +- tests/functional/dyn-drv/eval-outputOf.sh | 2 +- tests/functional/fetchTree-file.sh | 4 ++-- tests/functional/flakes/develop.sh | 10 +++++----- tests/functional/flakes/follow-paths.sh | 2 +- tests/functional/flakes/relative-paths.sh | 2 +- tests/functional/gc-closure.sh | 4 ++-- tests/functional/lang-gc.sh | 2 +- tests/functional/linux-sandbox.sh | 2 +- tests/functional/read-only-store.sh | 2 +- tests/functional/repl.sh | 2 +- tests/functional/simple.sh | 2 +- tests/functional/structured-attrs.sh | 2 +- tests/functional/suggestions.sh | 6 +++--- tests/nixos/ca-fd-leak/sender.c | 2 +- tests/nixos/fetch-git/test-cases/lfs/default.nix | 2 +- 30 files changed, 43 insertions(+), 43 deletions(-) diff --git a/doc/manual/remove_before_wrapper.py b/doc/manual/remove_before_wrapper.py index 6da4c19b0ce7..a0fcb6a55776 100644 --- a/doc/manual/remove_before_wrapper.py +++ b/doc/manual/remove_before_wrapper.py @@ -22,7 +22,7 @@ def main(): shutil.rmtree(output, ignore_errors=True) shutil.rmtree(output_temp, ignore_errors=True) - # Execute nix command with `--write-to` tempary output + # Execute nix command with `--write-to` temporary output nix_command_write_to = nix_command + ['--write-to', output_temp] subprocess.run(nix_command_write_to, check=True) diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index 7ea1d0e5aa99..cc7fe77eae56 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -160,7 +160,7 @@ When [`use-xdg-base-directories`] is enabled, the configuration directory is res Likewise for the state and cache directories. -## Miscellanous environment variables +## Miscellaneous environment variables - [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml index c3a27d2a6e0b..3ff606672cf5 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml @@ -40,7 +40,7 @@ additionalProperties: false title: Build Trace Key description: | A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. - This is the "key" part, refering to a derivation and output. + This is the "key" part, referring to a derivation and output. type: object required: - drvPath diff --git a/doc/manual/source/release-notes/rl-2.30.md b/doc/manual/source/release-notes/rl-2.30.md index 34d3e5bab4c6..5a65ed99af29 100644 --- a/doc/manual/source/release-notes/rl-2.30.md +++ b/doc/manual/source/release-notes/rl-2.30.md @@ -13,7 +13,7 @@ - Deprecate manually making structured attrs using the `__json` attribute [#13220](https://github.com/NixOS/nix/pull/13220) The proper way to create a derivation using [structured attrs] in the Nix language is by using `__structuredAttrs = true` with [`builtins.derivation`]. - However, by exploiting how structured attrs are implementated, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. + However, by exploiting how structured attrs are implemented, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. This sneaky alternative method is now deprecated, and may be disallowed in future versions of Nix. [structured attrs]: @docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs diff --git a/doc/manual/source/release-notes/rl-2.32.md b/doc/manual/source/release-notes/rl-2.32.md index 5d90da0c9ebd..c59ecd6c2456 100644 --- a/doc/manual/source/release-notes/rl-2.32.md +++ b/doc/manual/source/release-notes/rl-2.32.md @@ -8,7 +8,7 @@ - Derivation JSON format now uses store path basenames only [#13570](https://github.com/NixOS/nix/issues/13570) [#13980](https://github.com/NixOS/nix/pull/13980) - Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra paramater (the store directory). + Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra parameter (the store directory). We ultimately want to rectify this issue with all JSON formats to the extent allowed by our stability promises. To start with, we are changing the JSON format for derivations because the `nix derivation` commands are — in addition to being formally unstable — less widely used than other unstable commands. diff --git a/doc/manual/source/store/file-system-object.md b/doc/manual/source/store/file-system-object.md index 60cb3e572063..2ffad5f0ec94 100644 --- a/doc/manual/source/store/file-system-object.md +++ b/doc/manual/source/store/file-system-object.md @@ -19,7 +19,7 @@ Every file system object is one of the following: In general, Nix does not assign any semantics to symbolic links. Certain operations however, may make additional assumptions and attempt to use the target to find another file system object. - > See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. + > See [the Wikipedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. File system objects and their children form a tree. A bare file or symlink can be a root file system object. diff --git a/doc/manual/theme/head.hbs b/doc/manual/theme/head.hbs index e514a99777f7..40bfef7d2f8f 100644 --- a/doc/manual/theme/head.hbs +++ b/doc/manual/theme/head.hbs @@ -11,5 +11,5 @@ MathJax = { } }; - + diff --git a/packaging/everything.nix b/packaging/everything.nix index df7d57a85860..74629d684036 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -105,7 +105,7 @@ stdenv.mkDerivation (finalAttrs: { dontBuild = true; /** - `doCheck` controles whether tests are added as build gate for the combined package. + `doCheck` controls whether tests are added as build gate for the combined package. This includes both the unit tests and the functional tests, but not the integration tests that run in CI (the flake's `hydraJobs` and some of the `checks`). */ diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 7a61764d4f33..538a5e74d5ee 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -832,8 +832,8 @@ EOF # TODO: should probably alert the user if this is disabled? _sudo "to launch the Nix volume mounter" \ launchctl bootstrap system "$NIX_VOLUME_MOUNTD_DEST" || true - # TODO: confirm whether kickstart is necessesary? - # I feel a little superstitous, but it can guard + # TODO: confirm whether kickstart is necessary? + # I feel a little superstitious, but it can guard # against multiple problems (doesn't start, old # version still running for some reason...) _sudo "to launch the Nix volume mounter" \ diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index d4ea88b5ea6c..ae6625e1bc41 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -270,7 +270,7 @@ _diff() { printf -v CHANGED_GROUP_FORMAT "%b" "${GREEN}%>${RED}%<${ESC}" diff --changed-group-format="$CHANGED_GROUP_FORMAT" "$@" else - # simple colorized diff comatible w/ pre `--color` versions + # simple colorized diff compatible w/ pre `--color` versions diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" fi } @@ -961,7 +961,7 @@ configure_shell_profile() { cert_in_store() { # in a subshell # - change into the cert-file dir - # - get the phyiscal pwd + # - get the physical pwd # and test if this path is in the Nix store [[ "$(cd -- "$(dirname "$NIX_SSL_CERT_FILE")" && exec pwd -P)" == "$NIX_ROOT/store/"* ]] } diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 68263459337e..d5b81523b30d 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -44,7 +44,7 @@ cacheDir2="$TEST_ROOT/binary+cache" nix copy --to "file://$cacheDir2" "$outPath" && [[ -d "$cacheDir2" ]] basicDownloadTests() { - # No uploading tests bcause upload with force HTTP doesn't work. + # No uploading tests because upload with force HTTP doesn't work. # By default, a binary cache doesn't support "nix-env -qas", but does # support installation. diff --git a/tests/functional/build-delete.sh b/tests/functional/build-delete.sh index 66b14fd14384..e65615407aea 100755 --- a/tests/functional/build-delete.sh +++ b/tests/functional/build-delete.sh @@ -44,7 +44,7 @@ issue_6572_dependent_outputs() { # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. if [[ -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then - # Resolved derivations interferre with the deletion + # Resolved derivations interfere with the deletion nix-store --delete "${NIX_STORE_DIR}"/*.drv fi nix-store --delete "$(jq -r <"$TEST_ROOT"/a.json .[0].outputs.second)" diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 0e76f949f55d..9f86f8100324 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -53,7 +53,7 @@ nix build -f multiple-outputs.nix --json nothing-to-install --no-link | jq --exi (.outputs | keys == ["out"])) ' -# But not when it's overriden. +# But not when it's overridden. nix build -f multiple-outputs.nix --json e^a_a --no-link nix build -f multiple-outputs.nix --json e^a_a --no-link | jq --exit-status ' (.[0] | @@ -67,7 +67,7 @@ nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status ' (.outputs | keys == ["a_a", "b", "c"])) ' -# test buidling from non-drv attr path +# test building from non-drv attr path nix build -f multiple-outputs.nix --json 'e.a_a.outPath' --no-link | jq --exit-status ' (.[0] | diff --git a/tests/functional/characterisation-test-infra.sh b/tests/functional/characterisation-test-infra.sh index fecae29e8091..9651b4f7b6ba 100755 --- a/tests/functional/characterisation-test-infra.sh +++ b/tests/functional/characterisation-test-infra.sh @@ -19,14 +19,14 @@ cp "$TEST_ROOT/got" "$TEST_ROOT/expected" (( "$badDiff" == 0 )) ) -# matches empty, non-existant file is the same as empty file +# matches empty, non-existent file is the same as empty file echo -n > "$TEST_ROOT/got" ( diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" (( "$badDiff" == 0 )) ) -# doesn't matches non-empty, non-existant file is the same as empty file +# doesn't matches non-empty, non-existent file is the same as empty file echo Hi! > "$TEST_ROOT/got" ( diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" @@ -64,7 +64,7 @@ echo Bye! > "$TEST_ROOT/expected" (( "$badDiff" == 0 )) ) -# _NIX_TEST_ACCEPT matches empty, non-existant file not created +# _NIX_TEST_ACCEPT matches empty, non-existent file not created echo -n > "$TEST_ROOT/got" ( _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exists" diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index 771bbca785bc..0e571cc27a9a 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -101,7 +101,7 @@ killDaemon() { die "killDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." fi - # Don't fail trying to stop a non-existant daemon twice. + # Don't fail trying to stop a non-existent daemon twice. if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then return fi diff --git a/tests/functional/dyn-drv/eval-outputOf.sh b/tests/functional/dyn-drv/eval-outputOf.sh index 3681bd098899..13f7e7ba4c7e 100644 --- a/tests/functional/dyn-drv/eval-outputOf.sh +++ b/tests/functional/dyn-drv/eval-outputOf.sh @@ -34,7 +34,7 @@ testStaticHello () { (assert a == b; null))' } -# Test with a regular old input-addresed derivation +# Test with a regular old input-addressed derivation # # `builtins.outputOf` works without ca-derivations and doesn't create a # placeholder but just returns the output path. diff --git a/tests/functional/fetchTree-file.sh b/tests/functional/fetchTree-file.sh index 66be928c7b6b..a7c614b8ec0e 100755 --- a/tests/functional/fetchTree-file.sh +++ b/tests/functional/fetchTree-file.sh @@ -76,12 +76,12 @@ EOF # For backwards compatibility, flake inputs that correspond to the # old 'tarball' fetcher should still have their type set to 'tarball' assert (nodes.tarball_default_unpack.locked.type == "tarball"); - # Unless explicitely specified, the 'unpack' parameter shouldn’t appear here + # Unless explicitly specified, the 'unpack' parameter shouldn’t appear here # because that would break older Nix versions assert (!nodes.tarball_default_unpack.locked ? unpack); assert (nodes.tarball_default_unpack.locked.narHash == "$input_directory_hash"); - # Explicitely passing the unpack parameter should enforce the desired behavior + # Explicitly passing the unpack parameter should enforce the desired behavior assert (nodes.no_ext_explicit_unpack.locked.narHash == nodes.tarball_default_unpack.locked.narHash); assert (nodes.tarball_explicit_no_unpack.locked.narHash == nodes.no_ext_default_no_unpack.locked.narHash); diff --git a/tests/functional/flakes/develop.sh b/tests/functional/flakes/develop.sh index 0248adf49eaf..bb8850802742 100755 --- a/tests/functional/flakes/develop.sh +++ b/tests/functional/flakes/develop.sh @@ -66,7 +66,7 @@ echo "\$ENVVAR" EOF )" ]] -# Test wether `--keep-env-var` keeps the environment variable. +# Test whether `--keep-env-var` keeps the environment variable. ( expect='BAR' got="$(FOO='BAR' nix develop --ignore-env --keep-env-var FOO --no-write-lock-file .#hello <"$flakeFollowsA"/flake.nix < "$TEST_ROOT"/log grepQuietInverse 'error: renaming' "$TEST_ROOT"/log grepQuiet 'may not be deterministic' "$TEST_ROOT"/log diff --git a/tests/functional/read-only-store.sh b/tests/functional/read-only-store.sh index 8ccca2192af3..8bcf42cfc41d 100755 --- a/tests/functional/read-only-store.sh +++ b/tests/functional/read-only-store.sh @@ -4,7 +4,7 @@ source common.sh enableFeatures "read-only-local-store" -needLocalStore "cannot open store read-only when daemon has already opened it writeable" +needLocalStore "cannot open store read-only when daemon has already opened it writable" TODO_NixOS diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 9e752337f10d..cb181fbb6251 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -145,7 +145,7 @@ foo + baz ' "3" \ ./flake ./flake\#bar --experimental-features 'flakes' -# Test the `:reload` mechansim with flakes: +# Test the `:reload` mechanism with flakes: # - Eval `./flake#changingThing` # - Modify the flake # - Re-eval it diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index f6507d74182a..b7e8911930a4 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -20,7 +20,7 @@ text=$(cat "$outPath/hello") TODO_NixOS # Directed delete: $outPath is not reachable from a root, so it should -# be deleteable. +# be deletable. nix-store --delete "$outPath" [[ ! -e $outPath/hello ]] diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index d0b41670967b..32d647a772eb 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -26,7 +26,7 @@ TODO_NixOS # following line fails. # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` -# (which is an associative array in thsi case) being fine. +# (which is an associative array in this case) being fine. # shellcheck disable=SC2016 nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' diff --git a/tests/functional/suggestions.sh b/tests/functional/suggestions.sh index fbca93da8590..e4324f2dfafc 100755 --- a/tests/functional/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -30,7 +30,7 @@ EOF # Probable typo in the requested attribute path. Suggest some close possibilities NIX_BUILD_STDERR_WITH_SUGGESTIONS=$(! nix build .\#fob 2>&1 1>/dev/null) [[ "$NIX_BUILD_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ - fail "The nix build stderr should suggest the three closest possiblities" + fail "The nix build stderr should suggest the three closest possibilities" # None of the possible attributes is close to `bar`, so shouldn’t suggest anything NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) @@ -39,8 +39,8 @@ NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'"$system"'.fob' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ - fail "The evaluator should suggest the three closest possiblities" + fail "The evaluator should suggest the three closest possibilities" NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '({ foo }: foo) { foo = 1; fob = 2; }' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean foo?" ]] || \ - fail "The evaluator should suggest the three closest possiblities" + fail "The evaluator should suggest the three closest possibilities" diff --git a/tests/nixos/ca-fd-leak/sender.c b/tests/nixos/ca-fd-leak/sender.c index 639b88900228..f9fdd9405a52 100644 --- a/tests/nixos/ca-fd-leak/sender.c +++ b/tests/nixos/ca-fd-leak/sender.c @@ -62,6 +62,6 @@ int main(int argc, char ** argv) int buf; // Wait for the server to close the socket, implying that it has - // received the commmand. + // received the command. recv(sock, (void *) &buf, sizeof(int), 0); } diff --git a/tests/nixos/fetch-git/test-cases/lfs/default.nix b/tests/nixos/fetch-git/test-cases/lfs/default.nix index 289c3770911b..09b6fe44ffbd 100644 --- a/tests/nixos/fetch-git/test-cases/lfs/default.nix +++ b/tests/nixos/fetch-git/test-cases/lfs/default.nix @@ -172,7 +172,7 @@ f"did not set lfs, yet lfs-enrolled file is {file_size_default}b (>= 1KiB), probably bad default value" with subtest("Use as flake input"): - # May seem reduntant, but this has minor differences compared to raw + # May seem redundant, but this has minor differences compared to raw # fetchGit which caused failures before with TemporaryDirectory() as tempdir: client.succeed(f"mkdir -p {tempdir}") From 25295886ce8c4a8fa5ffd629540cac14a9736498 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 18:18:57 +0200 Subject: [PATCH 372/555] parseString(): Fix out-of-bounds read If the string isn't terminated, parseString() returns a string of size std::string::npos, which then causes an out-of-bounds read later. Fixes: ==47978== Invalid read of size 1 ==47978== at 0x4BEF70A: nix::expect(nix::(anonymous namespace)::StringViewStream&, char) (../src/libstore/derivations.cc:232) ==47978== by 0x4BEE3CA: parseDerivationOutput (../src/libstore/derivations.cc:383) ==47978== by 0x4BEE3CA: nix::parseDerivation(nix::StoreDirConfig const&, std::__cxx11::basic_string, std::allocator >&&, std::basic_string_view >, nix::ExperimentalFeatureSettings const&) (???:492) ==47978== by 0x3F3803: nix::DerivationTest_UnterminatedString_Test::TestBody() (../src/libstore-tests/derivation/external-formats.cc:27) ==47978== by 0x52AD3DD: void testing::internal::HandleExceptionsInMethodIfSupported(testing::Test*, void (testing::Test::*)(), char const*) (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x5298E3D: testing::Test::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x5298FCC: testing::TestInfo::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x529920E: testing::TestSuite::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x52A3996: testing::internal::UnitTestImpl::RunAllTests() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x52A3F74: testing::UnitTest::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x49DD52: RUN_ALL_TESTS (gtest.h:2334) ==47978== by 0x49DD52: main (???:16) --- src/libstore-tests/derivation/external-formats.cc | 8 ++++++++ src/libstore/derivations.cc | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index 6fee675e9849..e31b3e85442f 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -15,6 +15,14 @@ TEST_F(DerivationTest, BadATerm_version) parseDerivation(*store, readFile(goldenMaster("bad-version.drv")), "whatever", mockXpSettings), FormatError); } +TEST_F(DerivationTest, UnterminatedString) +{ + ASSERT_THROW( + parseDerivation( + *store, "Derive([(\"out\",\"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo", "bar", mockXpSettings), + FormatError); +} + TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { ASSERT_THROW( diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 8475532bca33..37c89461de25 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -223,6 +223,7 @@ static BackedStringView parseString(StringViewStream & str) size_t start = 0; size_t end = str.remaining.size(); const auto data = str.remaining.data(); + bool foundClose = false; while (start < end) { auto idx = str.remaining.find('"', start); if (idx == std::string_view::npos) { @@ -233,10 +234,13 @@ static BackedStringView parseString(StringViewStream & str) ; if ((idx - pos) % 2 == 0) { // even number of backslashes end = idx; + foundClose = true; break; } start = idx + 1; } + if (!foundClose) + throw FormatError("unterminated string in derivation"); start = 0; const auto content = str.remaining.substr(start, end); From e6dab09c2f4b87a62dc7b0022548fa7939f688cd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 May 2026 12:01:04 -0400 Subject: [PATCH 373/555] NAR listing: always serialize `executable` field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously `NarListing::Regular::to_json` only included `"executable"` when true, while `MemorySourceAccessor::File::Regular::to_json` always included it. This inconsistency makes it harder for other implementations to use a single JSON definition for both — e.g. with Rust's `#[serde(derive)]` there's no way to have `skip_serializing_if` vary by generic parameter. Always serializing the field can only help clients (they get more information), never hurt them (any correct client already handles both `true` and `false`). --- src/libutil-tests/data/nar-listing/deep.json | 1 + src/libutil/memory-source-accessor/json.cc | 3 +- tests/functional/binary-cache.sh | 22 +++- tests/functional/nar-access.sh | 110 +++++++++++++++++-- 4 files changed, 126 insertions(+), 10 deletions(-) diff --git a/src/libutil-tests/data/nar-listing/deep.json b/src/libutil-tests/data/nar-listing/deep.json index a7ed47c4c030..f58974f66e40 100644 --- a/src/libutil-tests/data/nar-listing/deep.json +++ b/src/libutil-tests/data/nar-listing/deep.json @@ -15,6 +15,7 @@ "type": "directory" }, "foo": { + "executable": false, "size": 15, "type": "regular" } diff --git a/src/libutil/memory-source-accessor/json.cc b/src/libutil/memory-source-accessor/json.cc index 84fbb71bb2ef..ff3808d3c363 100644 --- a/src/libutil/memory-source-accessor/json.cc +++ b/src/libutil/memory-source-accessor/json.cc @@ -51,8 +51,7 @@ void adl_serializer::to_json(json & j, const NarListing::Re { if (r.contents.fileSize) j["size"] = *r.contents.fileSize; - if (r.executable) - j["executable"] = true; + j["executable"] = r.executable; if (r.contents.narOffset) j["narOffset"] = *r.contents.narOffset; } diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 68263459337e..b76362bc2698 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -276,7 +276,27 @@ nix copy --to "file://$cacheDir"?write-nar-listing=1 "$outPath" diff -u \ <(jq -S < "$cacheDir/$(basename "$outPath" | cut -c1-32).ls") \ - <(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S) + <(jq -S <<'EOF' +{ + "version": 1, + "root": { + "type": "directory", + "entries": { + "bar": { + "type": "regular", + "executable": false, + "size": 4, + "narOffset": 232 + }, + "link": { + "type": "symlink", + "target": "xyzzy" + } + } + } +} +EOF + ) # Test debug info index generation. diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index cd419b4eefc8..f15a56d68867 100755 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -37,24 +37,120 @@ cp -r "$storePath" "$invalidPath" expect 1 nix store cat "$invalidPath/foo/baz" # Test --json. + +# Shallow listing of root (no --recursive) diff -u \ <(nix nar ls --json "$narFile" / | jq -S) \ - <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "foo": {}, + "foo-x": {}, + "qux": {}, + "zyx": {} + } +} +EOF + ) + +# Recursive listing of /foo from NAR (includes narOffset) diff -u \ <(nix nar ls --json -R "$narFile" /foo | jq -S) \ - <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "bar": { + "type": "regular", + "executable": false, + "size": 0, + "narOffset": 368 + }, + "baz": { + "type": "regular", + "executable": false, + "size": 0, + "narOffset": 552 + }, + "data": { + "type": "regular", + "executable": false, + "size": 58, + "narOffset": 736 + } + } +} +EOF + ) + +# Single file from NAR diff -u \ <(nix nar ls --json -R "$narFile" /foo/bar | jq -S) \ - <(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "regular", + "executable": false, + "size": 0, + "narOffset": 368 +} +EOF + ) + +# Shallow listing from store diff -u \ <(nix store ls --json "$storePath" | jq -S) \ - <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "foo": {}, + "foo-x": {}, + "qux": {}, + "zyx": {} + } +} +EOF + ) + +# Recursive listing from store (no narOffset) diff -u \ <(nix store ls --json -R "$storePath/foo" | jq -S) \ - <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "bar": { + "type": "regular", + "executable": false, + "size": 0 + }, + "baz": { + "type": "regular", + "executable": false, + "size": 0 + }, + "data": { + "type": "regular", + "executable": false, + "size": 58 + } + } +} +EOF + ) + +# Single file from store diff -u \ - <(nix store ls --json -R "$storePath/foo/bar"| jq -S) \ - <(echo '{"type":"regular","size":0}' | jq -S) + <(nix store ls --json -R "$storePath/foo/bar" | jq -S) \ + <(jq -S <<'EOF' +{ + "type": "regular", + "executable": false, + "size": 0 +} +EOF + ) # Test missing files. expect 1 nix store ls --json -R "$storePath/xyzzy" 2>&1 | grep 'does not exist' From 8ecf74430e3e4035444cff133a5e86db355738b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 19:05:10 +0200 Subject: [PATCH 374/555] Fix posix_fallocate() error case On error, it returns errno directly, not -1. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0bc7b6e1b6c8..a963f2d3cdee 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -218,7 +218,7 @@ LocalStore::LocalStore(ref config) #if HAVE_POSIX_FALLOCATE res = posix_fallocate(fd.get(), 0, gcSettings.reservedSize); #endif - if (res == -1) { + if (res != 0) { writeFull(fd.get(), std::string(gcSettings.reservedSize, 'X')); [[gnu::unused]] auto res2 = From 033ac553ccd2959c053b292da8e89a991ef31cfd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 20:10:28 +0200 Subject: [PATCH 375/555] Avoid size_t overflow --- src/libutil/serialise.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 5345883ceb70..0477cd2c9a8d 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -28,14 +28,14 @@ void BufferedSink::operator()(std::string_view data) while (!data.empty()) { /* Optimisation: bypass the buffer if the data exceeds the buffer size. */ - if (bufPos + data.size() >= bufSize) { + if (data.size() >= bufSize - bufPos) { flush(); writeUnbuffered(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer when it's full. */ - size_t n = bufPos + data.size() > bufSize ? bufSize - bufPos : data.size(); + size_t n = data.size() > bufSize - bufPos ? bufSize - bufPos : data.size(); memcpy(buffer.get() + bufPos, data.data(), n); data.remove_prefix(n); bufPos += n; From f61d9920d9206359ac12b0c080b2c8edca68cc7c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 20:10:59 +0200 Subject: [PATCH 376/555] Avoid calling lseek() with an unintended negative offset --- src/libutil/serialise.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0477cd2c9a8d..d989f1d09055 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -287,6 +287,8 @@ void FdSource::skip(size_t len) #ifndef _WIN32 /* If we can, seek forward in the file to skip the rest. */ if (isSeekable && len) { + if (len > static_cast(std::numeric_limits::max())) + throw Error("cannot skip %d bytes: exceeds maximum file offset", len); if (lseek(fd, len, SEEK_CUR) == -1) { if (errno == ESPIPE) isSeekable = false; From 7afb83f0b694b9036f245a7cd8fad722bceca972 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 20:29:18 +0200 Subject: [PATCH 377/555] readError(): Replace assertions by exceptions We shouldn't crash on input from the other side. --- src/libutil/serialise.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index d989f1d09055..7633732672c2 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -560,7 +560,8 @@ template StringSet readStrings(Source & source); Error readError(Source & source) { auto type = readString(source); - assert(type == "Error"); + if (type != "Error") + throw SerialisationError("unexpected error type '%s'", type); auto level = (Verbosity) readInt(source); [[maybe_unused]] auto name = readString(source); // removed auto msg = readString(source); @@ -569,11 +570,13 @@ Error readError(Source & source) .msg = HintFmt(msg), }; auto havePos = readNum(source); - assert(havePos == 0); + if (havePos != 0) + throw SerialisationError("deserializing error positions is not supported"); auto nrTraces = readNum(source); for (size_t i = 0; i < nrTraces; ++i) { havePos = readNum(source); - assert(havePos == 0); + if (havePos != 0) + throw SerialisationError("deserializing error positions is not supported"); info.traces.push_back(Trace{.hint = HintFmt(readString(source))}); } return Error(std::move(info)); From 22d1e6eef7eaaac25b322ac141bad07d25239357 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 30 Nov 2025 14:00:03 -0500 Subject: [PATCH 378/555] manual: More on stores, building and mounting in the file system Expand the manual on 1. Store paths: be more explicit about store path base names, introduce `{#digest}`, `{#name}`, and `{#base-name}` anchors, and add cross-links from `introduction.md`, `derivations.md`, `derivation-aterm.md`, `input-address.md`, and `rl-2.33.md`. 2. How store objects are exposed on the file system (new `os-file-system.md` page and new `{#exposing}` section of the store path page). 3. Building derivations: rewrite `building.md` with new sections on the build lifecycle, derivation resolution, file system setup (including `__structuredAttrs`/`passAsFile` file details), environment variables, and output processing. Try to be clearer about what *must* happen (spec, implementation agnostic), vs what Nix happens to do today. 4. Binary cache protocol: new `protocols/binary-cache/` section with an index page, a `.narinfo` format page (with field-by-field mapping to the JSON store object info schema), and move `nix-cache-info.md` into it. Link binary cache store types (HTTP, local, S3) back to the protocol page. 5. Store object metadata: new `{#metadata}` section in `store-object.md` linking to the JSON and narinfo formats. 6. Build trace: add `{#entry}` anchor, update JSON schema links. 7. Miscellaneous: fix typos in `os-file-system.md`, update `nix path-info --json-format` docs to cover v3, link `nix log` to store path base name and store object metadata, add `#store-directory` redirect, update stale links to match renamed anchors and moved pages. Co-authored-by: Eelco Dolstra Co-authored-by: Sergei Zimmerman --- doc/manual/redirects.json | 5 +- doc/manual/source/SUMMARY.md.in | 7 +- doc/manual/source/_redirects | 1 + doc/manual/source/command-ref/nix-hash.md | 20 +- .../source/command-ref/nix-prefetch-url.md | 2 +- doc/manual/source/glossary.md | 9 +- doc/manual/source/introduction.md | 2 +- .../source/language/advanced-attributes.md | 4 +- doc/manual/source/language/derivations.md | 2 +- .../binary-cache-substituter.md | 4 +- .../source/package-management/profiles.md | 3 +- .../source/protocols/binary-cache/index.md | 19 ++ .../source/protocols/binary-cache/narinfo.md | 42 +++ .../{ => binary-cache}/nix-cache-info.md | 4 +- .../source/protocols/derivation-aterm.md | 2 +- .../json/schema/build-trace-entry-v3.yaml | 6 +- .../json/schema/store-object-info-v3.yaml | 2 +- .../protocols/json/schema/store-path-v1.yaml | 2 +- doc/manual/source/protocols/nix32.md | 2 +- doc/manual/source/protocols/store-path.md | 6 +- doc/manual/source/release-notes/rl-2.33.md | 4 +- doc/manual/source/store/build-trace.md | 2 +- doc/manual/source/store/building.md | 246 +++++++++++++----- .../source/store/derivation/outputs/index.md | 2 +- .../store/derivation/outputs/input-address.md | 2 +- .../file-system-object/os-file-system.md | 40 +++ doc/manual/source/store/index.md | 31 ++- doc/manual/source/store/store-object.md | 6 + .../store/store-object/content-address.md | 2 +- doc/manual/source/store/store-path.md | 130 ++++++--- src/libstore/http-binary-cache-store.md | 2 +- .../include/nix/store/local-settings.hh | 2 +- src/libstore/local-binary-cache-store.md | 2 +- src/libstore/s3-binary-cache-store.md | 2 +- src/nix/flake.md | 4 +- src/nix/hash-convert.md | 4 +- src/nix/key-generate-secret.md | 2 +- src/nix/log.md | 14 +- src/nix/path-info.cc | 6 +- src/nix/path-info.md | 5 +- 40 files changed, 490 insertions(+), 162 deletions(-) create mode 100644 doc/manual/source/protocols/binary-cache/index.md create mode 100644 doc/manual/source/protocols/binary-cache/narinfo.md rename doc/manual/source/protocols/{ => binary-cache}/nix-cache-info.md (90%) create mode 100644 doc/manual/source/store/file-system-object/os-file-system.md diff --git a/doc/manual/redirects.json b/doc/manual/redirects.json index 0a6c71508006..329560ce786e 100644 --- a/doc/manual/redirects.json +++ b/doc/manual/redirects.json @@ -337,7 +337,7 @@ "string-literal": "string-literals.html" }, "language/derivations.html": { - "builder-execution": "../store/building.html#builder-execution" + "builder-execution": "../store/building.html" }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", @@ -363,6 +363,9 @@ "reverting": "contributing.html#reverting", "branches": "contributing.html#branches" }, + "store/store-path.html": { + "store-directory": "#store-directory-path" + }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", "package-attribute-set": "#package", diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 5a17426b9020..3efe9a38de23 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -19,9 +19,10 @@ - [Nix Store](store/index.md) - [File System Object](store/file-system-object.md) - [Content-Addressing File System Objects](store/file-system-object/content-address.md) + - [Exposing in OS File Systems](store/file-system-object/os-file-system.md) - [Store Object](store/store-object.md) - [Content-Addressing Store Objects](store/store-object/content-address.md) - - [Store Path](store/store-path.md) + - [Store Path and Store Directory](store/store-path.md) - [Store Derivation and Deriving Path](store/derivation/index.md) - [Derivation Outputs and Types of Derivations](store/derivation/outputs/index.md) - [Content-addressing derivation outputs](store/derivation/outputs/content-address.md) @@ -137,7 +138,9 @@ - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) - - [Nix Cache Info Format](protocols/nix-cache-info.md) + - [Binary Cache](protocols/binary-cache/index.md) + - [`nix-cache-info` Format](protocols/binary-cache/nix-cache-info.md) + - [`.narinfo` Format](protocols/binary-cache/narinfo.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Nix32 Encoding](protocols/nix32.md) - [C API](c-api.md) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 7e4557f7d595..be82fe872d60 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -47,6 +47,7 @@ /package-management/package-management /package-management 301! /package-management/s3-substituter /store/types/s3-binary-cache-store 301! +/protocols/nix-cache-info /protocols/binary-cache/nix-cache-info 301! /protocols/protocols /protocols 301! /json/* /protocols/json/:splat 301! diff --git a/doc/manual/source/command-ref/nix-hash.md b/doc/manual/source/command-ref/nix-hash.md index 7c17ce9095b4..c1a4251b0973 100644 --- a/doc/manual/source/command-ref/nix-hash.md +++ b/doc/manual/source/command-ref/nix-hash.md @@ -45,20 +45,20 @@ md5sum`. - `--base32` - Print the hash in a base-32 representation rather than hexadecimal. - This base-32 representation is more compact and can be used in Nix + Print the hash in [Nix32](@docroot@/protocols/nix32.md) representation rather than hexadecimal. + This representation is more compact and can be used in Nix expressions (such as in calls to `fetchurl`). - `--base64` - Similar to --base32, but print the hash in a base-64 representation, - which is more compact than the base-32 one. + Similar to `--base32`, but print the hash in a [Base64](https://en.wikipedia.org/wiki/Base64) representation, + which is more compact than the Nix32 one. - `--sri` - Print the hash in SRI format with base-64 encoding. + Print the hash in [SRI](@docroot@/glossary.md#gloss-sri) format with Base64 encoding. The type of hash algorithm will be prepended to the hash string, - followed by a hyphen (-) and the base-64 hash body. + followed by a hyphen (-) and the Base64 hash body. - `--truncate` @@ -71,18 +71,18 @@ md5sum`. - `--to-base16` - Don’t hash anything, but convert the base-32 hash representation + Don’t hash anything, but convert the [Nix32](@docroot@/protocols/nix32.md) hash representation *hash* to hexadecimal. - `--to-base32` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-32. + *hash* to [Nix32](@docroot@/protocols/nix32.md). - `--to-base64` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-64. + *hash* to Base64. - `--to-sri` @@ -134,7 +134,7 @@ $ nix-hash --type sha256 --flat test/world 5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 ``` -Converting between hexadecimal, base-32, base-64, and SRI: +Converting between hexadecimal, Nix32, Base64, and SRI: ```console $ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 diff --git a/doc/manual/source/command-ref/nix-prefetch-url.md b/doc/manual/source/command-ref/nix-prefetch-url.md index 8451778ad46d..86c20b9e1de4 100644 --- a/doc/manual/source/command-ref/nix-prefetch-url.md +++ b/doc/manual/source/command-ref/nix-prefetch-url.md @@ -32,7 +32,7 @@ Otherwise, the file is downloaded, and an error is signaled if the actual hash of the file does not match the specified hash. This command prints the hash on standard output. -The hash is printed using base-32 unless `--type md5` is specified, +The hash is printed using [Nix32](@docroot@/protocols/nix32.md) unless `--type md5` is specified, in which case it's printed using base-16. Additionally, if the option `--print-path` is used, the path of the downloaded file in the Nix store is also printed. diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index d7436a2052ee..112b0e3f8c0d 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -104,7 +104,7 @@ A derivation can be thought of as a [pure function](https://en.wikipedia.org/wiki/Pure_function) that produces new [store objects][store object] from existing store objects. - Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md#builder-execution). + Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md). This sandbox by default only allows reading from store objects specified as inputs, and only allows writing to designated [outputs][output] to be [captured as store objects](@docroot@/store/building.md#processing-outputs). A derivation is typically specified as a [derivation expression] in the [Nix language], and [instantiated][instantiate] to a [store derivation]. @@ -375,6 +375,13 @@ [path]: ./language/types.md#type-path [attribute name]: ./language/types.md#type-attrs +- [SRI]{#gloss-sri} + + [Subresource Integrity](https://www.w3.org/TR/SRI/) (SRI) is a [W3C specification](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) for integrity metadata. + Nix uses the SRI hash format (`-`) to specify content hashes in a way that is self-describing, since the hash algorithm is part of the format. + + [SRI]: #gloss-sri + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index 85de7982c917..33c8ba29faa9 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -49,7 +49,7 @@ builds correctly on your system, this is because you specified the dependency explicitly. This takes care of the build-time dependencies. Once a package is built, runtime dependencies are found by scanning -binaries for the hash parts of Nix store paths (such as `r8vvq9kq…`). +binaries for the [hash parts](@docroot@/store/store-path.md#digest) of Nix store paths (such as `r8vvq9kq…`). This sounds risky, but it works extremely well. ## Multi-user support diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index 67612029c8a4..cc2743dc5ca9 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -337,8 +337,8 @@ Here is more information on the `output*` attributes, and what values they may b This will specify the output hash of the single output of a [fixed-output derivation]. - The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). - The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of base-32 encoding. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](@docroot@/glossary.md#gloss-sri). + The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of Base32 encoding. > **Note** > diff --git a/doc/manual/source/language/derivations.md b/doc/manual/source/language/derivations.md index 2403183fc2d2..50aa525acbf4 100644 --- a/doc/manual/source/language/derivations.md +++ b/doc/manual/source/language/derivations.md @@ -165,7 +165,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > > for an Autoconf-style package. - The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. + The name of an output is combined with the name of the derivation to create the [name part](@docroot@/store/store-path.md#name) of the output's store path, unless it is `out`, in which case just the name of the derivation is used. > **Example** > diff --git a/doc/manual/source/package-management/binary-cache-substituter.md b/doc/manual/source/package-management/binary-cache-substituter.md index e6a772213d6d..bc2cdfb27ab2 100644 --- a/doc/manual/source/package-management/binary-cache-substituter.md +++ b/doc/manual/source/package-management/binary-cache-substituter.md @@ -19,7 +19,7 @@ whatever port you like: $ nix-serve -p 8080 ``` -To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) file on the client: +To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) file on the client: ```console $ curl http://avalon:8080/nix-cache-info @@ -28,7 +28,7 @@ WantMassQuery: 1 Priority: 30 ``` -When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) automatically if it doesn't exist. +When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) automatically if it doesn't exist. On the client side, you can tell Nix to use your binary cache using `--substituters`, e.g.: diff --git a/doc/manual/source/package-management/profiles.md b/doc/manual/source/package-management/profiles.md index 1d9e672a8def..53cf5061f834 100644 --- a/doc/manual/source/package-management/profiles.md +++ b/doc/manual/source/package-management/profiles.md @@ -11,8 +11,7 @@ in a directory another version might be stored in `/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2`. The long strings prefixed to the directory names are cryptographic hashes (to be -precise, 160-bit truncations of SHA-256 hashes encoded in a base-32 -notation) of *all* inputs involved in building the package — sources, +precise, 160-bit truncations of SHA-256 hashes encoded in [Nix32](@docroot@/protocols/nix32.md)) of *all* inputs involved in building the package — sources, dependencies, compiler flags, and so on. So if two packages differ in any way, they end up in different locations in the file system, so they don’t interfere with each other. Here is what a part of a typical Nix diff --git a/doc/manual/source/protocols/binary-cache/index.md b/doc/manual/source/protocols/binary-cache/index.md new file mode 100644 index 000000000000..d86d307ad9ee --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/index.md @@ -0,0 +1,19 @@ +# Binary Cache + +The binary cache format is an interface designed for exposing a store over HTTP. + +A binary cache consists of: + +- A [`nix-cache-info`](./nix-cache-info.md) file at the root with remote-side configuration. +- For each [store object](@docroot@/store/store-object.md): + - A [`.narinfo`](./narinfo.md) file containing the object's [metadata](@docroot@/store/store-object.md#metadata) and a (usually relative) URL to the corresponding compressed NAR. + - A possibly-compressed [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) containing the store object's file system data. +- For every entry in the [build trace](@docroot@/store/build-trace.md), a JSON file at `build-trace-v2//.doi`: + - the path encodes the [key](@docroot@/protocols/json/build-trace-entry.md#key) + - the contents are the [value](@docroot@/protocols/json/build-trace-entry.md#value). + +The following [store types](@docroot@/store/types/index.md) use the binary cache format: + +- [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md) — served over HTTP(S) +- [Local Binary Cache Store](@docroot@/store/types/local-binary-cache-store.md) — stored on the file system +- [S3 Binary Cache Store](@docroot@/store/types/s3-binary-cache-store.md) — stored in an AWS S3 bucket diff --git a/doc/manual/source/protocols/binary-cache/narinfo.md b/doc/manual/source/protocols/binary-cache/narinfo.md new file mode 100644 index 000000000000..e2e2efac0eeb --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/narinfo.md @@ -0,0 +1,42 @@ +# `.narinfo` Format + +A `.narinfo` file contains the [metadata of a store object](@docroot@/store/store-object.md#metadata) in the [binary cache](@docroot@/protocols/binary-cache/index.md) format. +It is a simple line-oriented format where each line is a `Key: Value` pair. +Some keys (e.g. `Sig`) may appear multiple times. + +The file is named `.narinfo`, where `` is the [hash part](@docroot@/store/store-path.md#digest) of the store object's [store path](@docroot@/store/store-path.md). + +The fields correspond to those documented in the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format: + +| `.narinfo` field | JSON field | Differences | +|---|---|---| +| `StorePath` | [`path`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_path) | Full [store path](@docroot@/store/store-path.md) rather than [store path base name](@docroot@/store/store-path.md#base-name) | +| `URL` | [`url`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_url) | | +| `Compression` | [`compression`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_compression) | Defaults to `bzip2` if omitted | +| `FileHash` | [`downloadHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadHash) | String-encoded hash rather than structured | +| `FileSize` | [`downloadSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadSize) | | +| `NarHash` | [`narHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narHash) | String-encoded hash rather than structured | +| `NarSize` | [`narSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narSize) | | +| `References` | [`references`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_references) | Space-separated [store path base names](@docroot@/store/store-path.md#base-name) rather than a JSON array | +| `Deriver` | [`deriver`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_deriver) | [Store path base name](@docroot@/store/store-path.md#base-name); `unknown-deriver` instead of `null` | +| `Sig` | [`signatures`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_signatures) | May appear multiple times rather than using an array | +| `CA` | [`ca`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_ca) | String-encoded [content address](@docroot@/store/store-object/content-address.md) rather than structured | + +## Example + + + +``` +StorePath: /nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +URL: nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz?sha256=1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3 +Compression: xz +FileHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +FileSize: 4029176 +NarHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +NarSize: 34878 +References: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +Deriver: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv +Sig: asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +Sig: qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +CA: fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh +``` diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/binary-cache/nix-cache-info.md similarity index 90% rename from doc/manual/source/protocols/nix-cache-info.md rename to doc/manual/source/protocols/binary-cache/nix-cache-info.md index e8351e1cebe8..3859b2a10ac4 100644 --- a/doc/manual/source/protocols/nix-cache-info.md +++ b/doc/manual/source/protocols/binary-cache/nix-cache-info.md @@ -1,6 +1,6 @@ -# Nix Cache Info Format +# `nix-cache-info` Format -The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/package-management/binary-cache-substituter.md) (e.g., `https://cache.example.com/nix-cache-info`). +The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/protocols/binary-cache/index.md) (e.g., `https://cache.example.com/nix-cache-info`). MIME type: `text/x-nix-cache-info` diff --git a/doc/manual/source/protocols/derivation-aterm.md b/doc/manual/source/protocols/derivation-aterm.md index 523678e663e3..778614eb1602 100644 --- a/doc/manual/source/protocols/derivation-aterm.md +++ b/doc/manual/source/protocols/derivation-aterm.md @@ -26,7 +26,7 @@ Derivations are serialised in one of the following formats: When derivation is encoded to a [store object] we make the following choices: -- The store path name is the derivation name with `.drv` suffixed at the end +- The store path [name](@docroot@/store/store-path.md#name) is the derivation name with `.drv` suffixed at the end Indeed, the ATerm format above does *not* contain the name of the derivation, on the assumption that a store path will also be provided out-of-band. diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml index 3ff606672cf5..2e825af6fb8a 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml @@ -4,7 +4,7 @@ title: Build Trace Entry description: | A record of a successful build outcome for a specific derivation output. - This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md). + This schema describes the JSON representation of an [entry](@docroot@/store/build-trace.md#entry) in a [build trace](@docroot@/store/build-trace.md). > **Warning** > @@ -39,7 +39,7 @@ additionalProperties: false key: title: Build Trace Key description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. This is the "key" part, referring to a derivation and output. type: object required: @@ -61,7 +61,7 @@ additionalProperties: false value: title: Build Trace Value description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. This is the "value" part, describing an output. type: object required: diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml index e0be716ef781..e0c1ffef134f 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml @@ -109,7 +109,7 @@ $defs: type: string title: Store Directory description: | - The [store directory](@docroot@/store/store-path.md#store-directory) this store object belongs to (e.g. `/nix/store`). + The [path to the store directory](@docroot@/store/store-path.md#store-directory-path) this store object belongs within (e.g. `/nix/store`). additionalProperties: false impure: diff --git a/doc/manual/source/protocols/json/schema/store-path-v1.yaml b/doc/manual/source/protocols/json/schema/store-path-v1.yaml index f1f58c2bf1ac..b1251c7426e9 100644 --- a/doc/manual/source/protocols/json/schema/store-path-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-path-v1.yaml @@ -24,7 +24,7 @@ description: | The format follows this pattern: `${digest}-${name}` - - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) + - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md) (20 hash bytes become 32 ASCII characters) - **name**: The package name and optional version/suffix information type: string diff --git a/doc/manual/source/protocols/nix32.md b/doc/manual/source/protocols/nix32.md index 72afe893ea24..d8da1e9952cf 100644 --- a/doc/manual/source/protocols/nix32.md +++ b/doc/manual/source/protocols/nix32.md @@ -1,6 +1,6 @@ # Nix32 Encoding -Nix32 is Nix's variant of base-32 encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. +Nix32 is Nix's variant of [Base32](https://en.wikipedia.org/wiki/Base32) encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. ## Alphabet diff --git a/doc/manual/source/protocols/store-path.md b/doc/manual/source/protocols/store-path.md index 1aa79615d1c8..bdce19d1d62c 100644 --- a/doc/manual/source/protocols/store-path.md +++ b/doc/manual/source/protocols/store-path.md @@ -18,11 +18,9 @@ where - `name` = the name of the store object. -- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory) +- `store-dir` = the [path of the store directory](@docroot@/store/store-path.md#store-directory-path) -- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. - - Nix uses a custom base-32 encoding called [Nix32](@docroot@/protocols/nix32.md). +- `digest` = [Nix32](@docroot@/protocols/nix32.md) representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. For the definition of the hash compression algorithm, please refer to section 5.1 of the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf). diff --git a/doc/manual/source/release-notes/rl-2.33.md b/doc/manual/source/release-notes/rl-2.33.md index bed697029389..668230693b8d 100644 --- a/doc/manual/source/release-notes/rl-2.33.md +++ b/doc/manual/source/release-notes/rl-2.33.md @@ -149,9 +149,9 @@ The new structured format follows the [JSON guidelines](@docroot@/development/js } ``` - The map from store path base names to store object info is nested under the `info` field. + The map from [store path base names](@docroot@/store/store-path.md#base-name) to store object info is nested under the `info` field. -- **Store path base names instead of full paths**: +- **[Store path base names](@docroot@/store/store-path.md#base-name) instead of full paths**: Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths. Combined with `storeDir`, the full path can be reconstructed. diff --git a/doc/manual/source/store/build-trace.md b/doc/manual/source/store/build-trace.md index a879d37d208d..cb9cb3099680 100644 --- a/doc/manual/source/store/build-trace.md +++ b/doc/manual/source/store/build-trace.md @@ -8,7 +8,7 @@ The *build trace* is a [memoization table](https://en.wikipedia.org/wiki/Memoization) for builds. It maps the inputs of builds to the outputs of builds. -Concretely, that means it maps [derivations][derivation] to maps of [output] names to [store objects][store object]. +Each *[entry]{#entry}* in the build trace maps a [derivation][derivation] to a map of [output] names to [store objects][store object]. In general the derivations used as a key should be [*resolved*](./resolution.md). A build trace with all-resolved-derivation keys is also called a *base build trace* for extra clarity. diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index 32e800129342..087413406487 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -1,72 +1,178 @@ # Building -## Normalizing derivation inputs +As discussed in the [main page on derivations](./derivation/index.md): -- Each input must be [realised] prior to building the derivation in question. +> A derivation is a specification for running an executable on precisely defined input to produce one or more [store objects][store object]. + +This page describes *building* a derivation, which is to say following the instructions in the derivation to actually run the executable. +Some elements of derivations are self-explanatory. +For example, the arguments specified in the derivation really are the arguments passed to the executable. +In other cases, however, there is additional common steps performed by Nix for all derivations --- mostly for setting up the build environment and collecting the built outputs. + +The chief design consideration for the building process is *determinism*. +Conventional operating systems are typically not designed with determinism in mind. +But determinism is needed to make Nix's build caching a transparent abstraction. + +> **Explanation** +> +> For example, no one wants to slightly modify a derivation, and then find that it no longer builds for an unrelated reason, because the original derivation *also* doesn't build anymore, but the cache hit on the original derivation was hiding this. +> We want builds that succeed once to continue succeeding, to encourage fearless modification of old build recipes. +> Determinism is what enables things that once worked to keep working. + +The life cycle of a build can be broken down into 3 parts: + +1. Spawn the builder process with the proper environment, including the correct process arguments, environment variables, and file system state. + +2. Wait for the builder process to exit and collect its exit status. + Exit code 0 means success; anything else is a build failure. + (Strictly speaking, Nix detects process exit by waiting for the standard output and error streams to close. + If a builder explicitly closes these streams without exiting, Nix will kill it, and deem the build a failure. + Processes should therefore exit *without* explicitly closing those standard streams, and let the exiting of the process close them implicitly.) + + Nix also logs the standard output and error of the process, but this is just for human convenience and does not influence the behavior of the system. + (Builder processes have no idea what the consumer of their standard output and error does with the pseudo-terminal master, only that they are indeed consumed so buffers do not fill up etc. and writes to each output standard stream will continue to succeed. + In practice, Nix will store the log in `/nix/var/log/nix`) + +3. Processing the outputs after the builder has exited. + + The builder process on exit should have left behind files for each output the derivation is supposed to produce. + The files must be processed to turn them into bona fide store objects. + If the processing succeeds, those store objects are associated with the derivation as (the results of) a successful build. + +Step (3) is done by Nix externally to the build itself, which is just steps (1) and (2). +In step (3), just inert data is processed, since the builder process has exited or been killed by then. +Step (1) however is best described not from Nix's perspective, but from the build process's perspective. + +> **Explanation** +> +> Ultimately, what matters for determinism is what the build process can observe: what resources (files, networking, etc.) it can see, what syscalls succeed or fail, etc. +> Nix can achieve this through many different sandboxing strategies (namespaces, VMs, chroots, ...), but the process shouldn't be able to tell them apart. +> We therefore specify building from the process's perspective, not Nix's perspective, to focus on *what*, not *how*. + +## What derivations can be built + +Actually only some derivations are ready to be built. +In particular, only [*resolved*](./resolution.md) derivations can be built. +That is to say, a derivation that depends on other derivations is not ready yet to be built, because some of those other derivations might not have yet been built. +If the other derivations are indeed all built, we can witness this fact by resolving the derivation, and converting all the derivation's input references into plain store paths. + +> **Note** +> +> Note that [input-addressing](derivation/outputs/input-address.md) derivations are improperly resolved. +> As discussed on the linked page, the current input-addressing algorithm does not respect resolution-equivalence of derivations (\\(\\sim_\mathrm{Drv}\\)). +> That means that if Nix properly resolved an input-addressed derivation, the resolved derivation would have different input addresses, violating expectations. +> Nix therefore improperly resolves the derivation, keeping its original input-addressed output paths, creating an invalid derivation that is both resolved and instructed to create the outputs at the originally expected paths. + +## Environment of the builder process + +This section describes how the [`builder`](./derivation/index.md#builder) is executed. + +> **Implementation detail** +> +> Nix prevents multiple [Nix instances][Nix instance] from performing the same build at the same time, for example by acquiring exclusive file locks. + +### File system + +The builder should have access to a limited file system where only certain objects are available. +The most important exposed files are the inputs (other store objects) of the (resolved) derivation. +Additionally, some other files are exposed. + +#### Store inputs + +The builder will be run against a file system in which the [store directory][store directory path] contains the [closure] of the inputs. +In particular, consider a store that just contains this closure. +That store is exposed to the file system according to the rules specified in the [Exposing Store Objects in OS File Systems](./store-path.md#exposing) documentation. +This precisely defines the file system layout of the store that should be visible to the builder process. + +> **Note** +> +> Historically, Nix exposed *at least* the following store contents to the builder, but also arbitrarily other store objects, due to limitations around operating systems' file system virtualization capabilities, and wanting to avoid copying or moving files. +> It still can do this in so-called *unsandboxed* builds. +> +> Such builds should be considered discouraged, but one that works less badly against non-mischievous derivations than might be expected. +> This is because store paths are relatively unpredictable, so a well-behaved program is unlikely to stumble upon a store object it wasn't supposed to know about. +> +> As operating systems developed better file system primitives, the need for disabling sandboxing has lessened greatly over the years, and this trend should continue into the future. + +The outputs are expected to be created in that store directory as if they were valid store objects. +(They are just files during builder execution, but during [processing outputs](#processing-outputs) they will be turned into proper store objects.) +The [environment variables](#env-vars) for each output indicate where the builder should write them; +Nix ensures that those paths do not yet exist when the builder is run. + +> **Note** +> +> In sandboxed builds, ensuring that the outputs do not exist in the store directory is trivial. +> In unsandboxed builds, it is harder in general. +> In the worst case, the derivation is in fact rewritten so different output paths are used instead, and then the outputs are rewritten back to the intended output paths after. +> In the content-addressing case rewriting would be needed either way, but in the input-addressing case, this is a significant degradation, as the point of input addressing is to avoid rewrites by knowing output paths in advance. [realised]: @docroot@/glossary.md#gloss-realise +[closure]: @docroot@/glossary.md#gloss-closure +[store directory path]: ./store-path.md#store-directory-path + +### Other file system state + +- The current working directory of the builder process will be a fresh temporary directory. + It is initially empty when the process starts except for a few input files: -- Once this is done, the derivation is *normalized*, replacing each input deriving path with its store path, which we now know from realising the input. + - If [`__structuredAttrs`](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs) is enabled: `.attrs.json` (the derivation attributes as JSON) and `.attrs.sh` (a Bash-compatible rendering of the same). + The environment variables `NIX_ATTRS_JSON_FILE` and `NIX_ATTRS_SH_FILE` point to these files, respectively. -## Builder Execution {#builder-execution} + - If [`passAsFile`](@docroot@/language/advanced-attributes.md#adv-attr-passAsFile) is used (only without `__structuredAttrs`): for each attribute name listed, a file `.attr-` where `` is the [Nix32](@docroot@/protocols/nix32.md)-encoded SHA-256 hash of the attribute name. + The environment variable `Path` points to the file containing the attribute's value. -The [`builder`](./derivation/index.md#builder) is executed as follows: + In sandboxed builds, this directory is at a deterministic path inside the sandbox (controlled by the [`sandbox-build-dir`](@docroot@/command-ref/conf-file.md#conf-sandbox-build-dir) setting, default `/build`). + See also the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for the host-side location. -- A temporary directory is created where the build will take place. The - current directory is changed to this directory. +- Basic device nodes for essential operations (null device, random number generation, standard streams as a pseudo terminal) - See the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for more information. + (A pseudo terminal would not be strictly necessary since the standard streams are passively logging, not there to facilitate interaction. + But it is still useful to entice programs to do nicer logging with e.g. colors etc.) -- The environment is cleared and set to the derivation attributes, as - specified above. +- On Linux: Process information via `/proc` -- In addition, the following variables are set: +- Minimal user and group identity information - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. +- A loopback-only network configuration with hostname set to `localhost` - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. +> **Note** +> +> Fixed-output derivations have access to additional operating system state to facilitate communication with the outside world, such as network name resolution and TLS certificate verification. +> This is necessary because these derivations are allowed to access the network, unlike regular derivations which are fully sandboxed. - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. +### Environment variables {#env-vars} - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. +The environment is cleared and set to the derivation attributes, as +specified above. - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). +For most derivations types this must contain at least: - - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` - is set to `true` for the derivation. A detailed explanation of this - behavior can be found in the - [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). +- For each output declared in `outputs`, the corresponding environment variable is set to point to the intended path in the Nix store for that output. + Each output path is a concatenation of the cryptographic hash of all build inputs, the `name` attribute and the output name. + (The output name is omitted if it's `out`.) - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) +In addition, the following variables are set: -- If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple [Nix instances][Nix instance] from performing the same - build at the same time. +- `NIX_BUILD_TOP` contains the path of the temporary directory for this build. -- A log of the combined standard output and error is written to - `/nix/var/log/nix`. +- Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the temporary directory. + This is to prevent the builder from accidentally writing temporary files anywhere else. + Doing so might cause interference by other processes. -- The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. +- `PATH` is set to `/path-not-set` to prevent shells from initialising it to their built-in default value. -- The temporary directory is removed (unless the `-K` option was - specified). +- `HOME` is set to `/homeless-shelter`. + (Without sandboxing, this discourages programs from using `/etc/passwd` or the like to find the user's home directory, which could cause impurity.) + Usually, when `HOME` is set, it is used as the location of the home directory, even if it points to a non-existent path. + +- `NIX_STORE` is set to the path of the top-level Nix [store directory path] (typically, `/nix/store`). + +- `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` is set to `true` for the derivation. + A detailed explanation of this behavior can be found in the [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). + +### Arguments + +The builder is passed the arguments specified by the derivation attribute `args`. ## Processing outputs @@ -74,28 +180,40 @@ If the builder exited successfully, the following steps happen in order to turn - **Normalize the file permissions** - Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Any possible `setuid` and `setgid` - bits are cleared. - - > **Note** - > - > Setuid and setgid programs are not currently supported by Nix. - > This is because the Nix archives used in deployment have no concept of ownership information, - > and because it makes the build result dependent on the user performing the build. + The files must conform to the model described in the [Exposing in OS file systems](./file-system-object/os-file-system.md) section. + For example, timestamps and permissions are canonicalised. - **Calculate the references** - Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. + Nix scans each output path for [references] to input store objects by looking for the [digest][store path digest] of each input. + (The name part and the [store directory path] are ignored when scanning; an input's hash part that is neither followed by a `-` nor proceeded by a `/` still scans as a reference.) + Since these are potential runtime dependencies, Nix will register them as references of the output store object they occur in. + + Nix also scans for references from one output to another in the same way, because outputs are allowed to refer to each other. + + The outputs' references must form a [directed acyclic graph](@docroot@/glossary.md#gloss-directed-acyclic-graph). + (This is not a special restriction for outputs; it is true for the references of all store objects in general.) + + In the case of derivations with output paths that are fixed in advance (i.e. [input-addressing] derivations, or [fixed content-addressing] derivations), the actual final store path to each output is used during the build if possible. + For [floating content-addressing] derivations, however, the final store path is not known in advance by definition. + Scratch store paths must therefore be used instead. + Scanning will use those scratch paths, but then any output-to-be that contains such a scanned scratch path must be rewritten to instead use the final (content-addressed) path of the output in question. - Nix also scans for references to other outputs' paths in the same way, because outputs are allowed to refer to each other. - If the outputs' references to each other form a cycle, this is an error, because the references of store objects much be acyclic. +At this point, the file system data is in the proper form, and the valid acyclic reference data for each output is also calculated, so the outputs are added to the store as proper store objects. +Additionally, those store objects (at least in the case that they are [content-addressed][content-addressing]) can be associated with the derivation in the [build trace] in the record for a successful build. +> **Implementation detail** +> +> Nix will normally clean up and remove the temporary build directory after every build, successful or unsuccessful. +> The builder doesn't know whether Nix does or not, however, as it will have exited before the build directory is cleaned up, and it will not see any old build directory if (after a failed build) it is run again. +> The [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) option can be specified to keep the build directory in the case of a failing build. +[references]: ./store-object.md#references +[store path digest]: ./store-path.md#digest +[store object]: ./store-object.md [Nix instance]: @docroot@/glossary.md#gloss-nix-instance +[content-addressing]: ./derivation/outputs/content-address.md +[input-addressing]: ./derivation/outputs/input-address.md +[fixed content-addressing]: ./derivation/outputs/content-address.md#fixed +[floating content-addressing]: ./derivation/outputs/content-address.md#floating +[build trace]: ./build-trace.md diff --git a/doc/manual/source/store/derivation/outputs/index.md b/doc/manual/source/store/derivation/outputs/index.md index ca2ce6665b04..9b46405cdf58 100644 --- a/doc/manual/source/store/derivation/outputs/index.md +++ b/doc/manual/source/store/derivation/outputs/index.md @@ -9,7 +9,7 @@ The outputs specification is a map, from names to specifications for individual ## Output Names {#outputs} -Output names can be any string which is also a valid [store path](@docroot@/store/store-path.md) name. +Output names can be any string which is also a valid [store path name](@docroot@/store/store-path.md#name). The name mapped to each output specification is not actually the name of the output. In the general case, the output store object has name `derivationName + "-" + outputSpecName`, not any other metadata about it. However, an output spec named "out" describes and output store object whose name is just the derivation name. diff --git a/doc/manual/source/store/derivation/outputs/input-address.md b/doc/manual/source/store/derivation/outputs/input-address.md index 3fd20f17d724..6df9b94961e8 100644 --- a/doc/manual/source/store/derivation/outputs/input-address.md +++ b/doc/manual/source/store/derivation/outputs/input-address.md @@ -16,7 +16,7 @@ Concretely, this would cause a "mass rebuild" whenever any fetching detail chang To solve this problem, we compute output hashes differently, so that certain output hashes become identical. We call this concept quotient hashing, in reference to quotient types or sets. -So how do we compute the hash part of the output paths of an input-addressed derivation? +So how do we compute the [hash part](@docroot@/store/store-path.md#digest) of the output paths of an input-addressed derivation? This is done by the function `hashQuotientDerivation`, shown below. First, a word on inputs. diff --git a/doc/manual/source/store/file-system-object/os-file-system.md b/doc/manual/source/store/file-system-object/os-file-system.md new file mode 100644 index 000000000000..6ce21f7788c9 --- /dev/null +++ b/doc/manual/source/store/file-system-object/os-file-system.md @@ -0,0 +1,40 @@ +# Exposing File System Objects in real operating system file systems + +Nix's [file system object] data model is minimal. +All the various other bits and pieces of real world filesystem interfaces, such as [extended file attributes](https://en.wikipedia.org/wiki/Extended_file_attributes), are specifically ignored to reduce our interface surface and the reproducibility issues associated with a larger interface. +In the view of Nix's developers, the types of simple, fine-grained batch jobs (typically, building software) that Nix specializes in simply don't benefit enough from that extra complexity for it to be worth the costs of supporting it. + +But to actually be used by software, file system objects need to be made available through the operating system's file system. +This is sometimes called "mounting" or "exposing" the file system object, though do note it may or may not be implemented with what the operating system calls "mounting". + +[file system object]: ../file-system-object.md + +## Metadata normalization + +File systems typically contain other metadata that is outside Nix's data model. +To avoid this other metadata being a side channel and source of nondeterminism, Nix is careful to normalize to fixed values. +For example, on Unix, the following metadata normalization occurs: + +- The creation and last modification timestamps on all files are set to Unix Epoch 1s (00:00:01 1/1/1970 UTC) + +- The group is set to the [default group](@docroot@/command-ref/conf-file.md#conf-build-users-group) + +- The Unix mode of the file to 0444 or 0555 (i.e., read-only, with execute permission enabled if the file was originally executable). + +- Any possible `setuid` and `setgid` bits are cleared. + + > **Note** + > + > `setuid` and `setgid` programs are not currently supported by Nix. + > These special file system permissions are in general a security footgun, and with data owned by different users in different stores, it would especially be a hazard when copying store objects between stores. + > + > This restriction has not proved to be onerous in practice. + > For example, NixOS uses so called setuid-wrappers which are outside the store. + +> **Explanation** +> +> As discussed before, Nix essentially shares its file system object data model with other tools like Git. +> But those tools tend to ignore this metadata in both directions --- when reading files, like Nix, but when writing files, timestamps are set organically, and the user is free to set other special permissions (`setuid`, `setgid`, sticky, etc.) however they like. +> Normalizing, and not just ignoring, this metadata is therefore what distinguishes Nix from these other tools more than the file system object data model itself. +> +> Nix's approach is motivated by deterministic building. Whereas Git can assume that humans running commands will simply ignore timestamps etc. as appropriate, understanding they are local and ephemeral, Nix aims to run software that was not necessarily designed with Nix in mind, and is unaware of whatever sandboxing/virtualization is in place. diff --git a/doc/manual/source/store/index.md b/doc/manual/source/store/index.md index f1e8f1402988..d063fc4fdc05 100644 --- a/doc/manual/source/store/index.md +++ b/doc/manual/source/store/index.md @@ -2,4 +2,33 @@ The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. -There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). +Concretely, albeit using concepts that are only defined in the rest of the chapter, a store consists of: + +- A set of [store objects][store object], the immutable file system data. + + This can also be looked at as a map from [store paths][store path] to store objects. + +- A set of [derivations][derivation], instructions for building store objects. + + This can also be looked at as a map from [store paths][store path] to derivations. + Since store paths to derivations always end in `.drv`, and store paths to other store objects never do, the two maps can also be combined into one. + Derivations can also be encoded as store objects too. + +- A [build trace], a record of which derivations have been built and what they produced. + + > **Warning** + > + > The concept of a build trace is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) + > and subject to change. + +There are [multiple types of Nix stores][store type] with different capabilities, such as the default one on the [local file system][local store] (`/nix/store`) or [binary caches][binary cache]. + +[store object]: ./store-object.md +[store path]: ./store-path.md +[derivation]: ./derivation/index.md +[build trace]: ./build-trace.md + +[store type]: ./types/index.md +[local store]: ./types/local-store.md +[binary cache]: ./types/http-binary-cache-store.md diff --git a/doc/manual/source/store/store-object.md b/doc/manual/source/store/store-object.md index 170d6246cfe7..eb84ab84370d 100644 --- a/doc/manual/source/store/store-object.md +++ b/doc/manual/source/store/store-object.md @@ -66,3 +66,9 @@ A store can only contain a store object if it also contains all the store object > > The "closure property" isn't meant to prohibit, for example, [lazy loading](https://en.wikipedia.org/wiki/Lazy_loading) of store objects. > However, the "closure property" and immutability in conjunction imply that any such lazy loading ought to be deterministic. + +### Store Object Metadata {#metadata} + +[Store implementations](@docroot@/store/types/index.md) currently associate more information than described above with a store object. +Quite arguably some of this information doesn't belong here, because it conflates concerns. +For details see the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format or the [narinfo](@docroot@/protocols/binary-cache/narinfo.md) format. diff --git a/doc/manual/source/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md index 94d94ec6d4ae..282b7545a231 100644 --- a/doc/manual/source/store/store-object/content-address.md +++ b/doc/manual/source/store/store-object/content-address.md @@ -9,7 +9,7 @@ In particular, the content-addressing scheme will ensure that the digest of the - file system object graph (the root one and its children, if it has any) - references -- [store directory](../store-path.md#store-directory) +- [store directory path](../store-path.md#store-directory-path) - name of the store object, and not any other information, which would not be an intrinsic property of that store object. diff --git a/doc/manual/source/store/store-path.md b/doc/manual/source/store/store-path.md index 04bdfec004c2..43037f7baac6 100644 --- a/doc/manual/source/store/store-path.md +++ b/doc/manual/source/store/store-path.md @@ -1,72 +1,136 @@ -# Store Path +# Store Path and Store Directory -> **Example** -> -> `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` -> -> A rendered store path +Nix's [store object] and [file system object] data models are minimal and abstract. +But to actually be used by software, store objects need to be made available through the operating system's file system. + +This is done by exposing all the store objects in a single *[store directory][store directory path]*. +Every entry in that directory is a *[store path base name]* pointing to a store object. +Store objects exposed in this way can then be referenced by *[store paths][store path]*. + +[store object]: ./store-object.md +[file system object]: ./file-system-object.md +[store path]: #store-path +[store path base name]: #base-name +[store directory path]: #store-directory-path + +## Store Path Base Name {#base-name} -Nix implements references to [store objects](./store-object.md) as *store paths*. +Nix implements references to store objects as *store path base names*. -Think of a store path as an [opaque], [unique identifier]: -The only way to obtain store path is by adding or building store objects. -A store path will always reference exactly one store object. +Think of a store path base name as an [opaque], [unique identifier]: +The only way to obtain a store path base name is by adding or building store objects. +A store path base name will always reference exactly one store object. [opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type [unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier -Store paths are pairs of +Store path base names are pairs of -- A 20-byte digest for identification -- A symbolic name for people to read +- A 20-byte [digest]{#digest} for identification +- A symbolic [name]{#name} for people to read > **Example** > > - Digest: `q06x3jll2yfzckz2bzqak089p43ixkkq` > - Name: `firefox-33.1` -To make store objects accessible to operating system processes, stores have to expose store objects through the file system. +A store path base name is rendered to a string as the concatenation of -A store path is rendered to a file system path as the concatenation of - -- [Store directory](#store-directory) (typically `/nix/store`) -- Path separator (`/`) -- Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) +- Digest rendered in [Nix32], a variant of [Base32] (20 hash bytes become 32 ASCII characters) - Hyphen (`-`) - Name > **Example** > > ``` -> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 -> |--------| |------------------------------| |----------| -> store directory digest name +> q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |------------------------------| |----------| +> digest name > ``` -Exactly how the digest is calculated depends on the type of store path. +[Nix32]: @docroot@/protocols/nix32.md +[Base32]: https://en.wikipedia.org/wiki/Base32 + +Exactly how the digest is calculated depends on the type of store object being referenced. Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). -## Store Directory - -Every [Nix store](./index.md) has a store directory. +## Store Directory Path -Not every store can be accessed through the file system. -But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). +Every [Nix store] has a store directory path. +This is an absolute, lexically canonical (not containing any `..`, `.`, or similar) path which points to the directory where all store objects are to be found. -[file system objects]: ./file-system-object.md - -This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +[Nix store]: ./index.md > **Note** > > The store directory defaults to `/nix/store`, but is in principle arbitrary. -It is important which store a given store object belongs to: +## Store Path + +A store path is the pair of a store directory path and a [store path base name]. +It is rendered to a file system path as the concatenation of + +- [Store directory path] (typically `/nix/store`) +- Path separator (`/`) +- The [store path base name] + +> **Example** +> +> ``` +> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +When we have fixed a given store, or given store directory path (that all the stores in use share), the abstract syntax for store paths and the abstract syntax for store path base names coincide: the store directory path is known from context, so only the other two fields vary from one store path to the next. + +## Exposing Store Objects in OS File Systems {#exposing} + +Not every store can be accessed through the file system. +But if the store has a file system representation, the following should be true: + +- The store directory path is canonical: no prefix of the path (i.e. path of the first *n* path segments) points to a symlink. + In other words, the store directory can be looked up from the store directory path without following any symlinks. + (This condition is a separate condition in addition to the "lexical canonicity" described above, which is a property of just the path itself. + This (regular) "canonicity" is a property about the path and the filesystem it navigates jointly.) + + > **Note** + > + > The [`allow-symlinked-store`](@docroot@/command-ref/conf-file.md#conf-allow-symlinked-store) setting can be used to relax this requirement. + +- The store directory path in fact points to a directory. + +- The store directory contains, for every store object in the store, the [file system object] of that store object at the (rendered) [store path base name]. + The permissions and other metadata for these files in the store directory is in the normal form described in [Exposing in OS file systems](./file-system-object/os-file-system.md). + +The above properties mean that the following file accesses will work. +Suppose we have a store available on the file system per the above rules, and `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` is the store path base name of a store object in that store. + +- Suppose that the store directory (path) is `/foo/bar`. + Then, `/foo/bar/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` exists and is the file system object of that store object. + +- Suppose that we don't know what the store directory path of the store is, but we do have a capability `storeDir` to the store directory on the file system. + (This would be a "file descriptor" on Unix, or a "file handle" on Windows.) + Then (using the Unix notation for this): + ``` + openat(storeDir, "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1", O_NOFOLLOW) + ``` + will succeed (so long as the file system object is not a symlink), and the yielded capability will point to the file system object of that store object. + + (The behavior for symlinks is harder to specify because of limitations in POSIX.) + +## Relocating store objects + +The inclusion of the store directory path in the full rendered store path means that the full rendered store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +(And actually, all of the currently-supported ways of computing the digest of a store path also depend on the store directory path, as described in the [specification of store path digests](@docroot@/protocols/store-path.md). +So this is also true even just for store path base names, in general.) + +It is therefore important to consider which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. Nix can only guarantee referential integrity if store paths do not cross store boundaries. -Therefore one can only copy store objects to a different store if +One can only copy store objects to a different store if - The source and target stores' directories match diff --git a/src/libstore/http-binary-cache-store.md b/src/libstore/http-binary-cache-store.md index 20c26d0c2caf..03dd350ec518 100644 --- a/src/libstore/http-binary-cache-store.md +++ b/src/libstore/http-binary-cache-store.md @@ -2,7 +2,7 @@ R"( **Store URL format**: `http://...`, `https://...` -This store allows a binary cache to be accessed via the HTTP +This store allows a [binary cache](@docroot@/protocols/binary-cache/index.md) to be accessed via the HTTP protocol. )" diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index 7381b5b8e766..dc4322753d7f 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -196,7 +196,7 @@ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllo 0, "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation. + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#env-vars) of a derivation. The `builder` executable can use this variable to control its own maximum amount of parallelism. - From 87c3c3251c4a1ac39bac6772342bca00cceb18f7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 21:48:22 +0300 Subject: [PATCH 513/555] Fix truncated store hash part in the manual introduction, document FreeBSD support --- doc/manual/source/introduction.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index 85de7982c917..51f9fb45d2e0 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -10,7 +10,7 @@ as /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1/ -where `b6gvzjyb2pg0…` is a unique identifier for the package that +where `q06x3jll2yfz…` is a unique identifier for the package that captures all its dependencies (it’s a cryptographic hash of the package’s build dependency graph). This enables many powerful features. @@ -174,7 +174,7 @@ the package: ## Portability -Nix runs on Linux and macOS. +Nix runs on Linux, macOS and FreeBSD. ## NixOS From 3d445e7bc597cd78d28a3c0f74fc9bc5b7e8ba03 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 21:48:52 +0300 Subject: [PATCH 514/555] Bring prerequisites-source.md up to date This has bitrotten a lot and needed a lot of updates since the meson migration. I've removed version bounds that we don't check in the build system, which have had likely bitrotten too. Also start to follow SEMBR in these docs. --- .../installation/prerequisites-source.md | 126 +++++++----------- 1 file changed, 49 insertions(+), 77 deletions(-) diff --git a/doc/manual/source/installation/prerequisites-source.md b/doc/manual/source/installation/prerequisites-source.md index 640032c5176b..e98067348ee6 100644 --- a/doc/manual/source/installation/prerequisites-source.md +++ b/doc/manual/source/installation/prerequisites-source.md @@ -1,80 +1,52 @@ # Prerequisites - - GNU Autoconf () and the - autoconf-archive macro collection - (). These are - needed to run the bootstrap script. - - - GNU Make. - - - Bash Shell. The `./configure` script relies on bashisms, so Bash is - required. - - - A version of GCC or Clang that supports C++23. - - - `pkg-config` to locate dependencies. If your distribution does not - provide it, you can get it from - . - - - The OpenSSL library to calculate cryptographic hashes. If your - distribution does not provide it, you can get it from - . - - - The `libbrotlienc` and `libbrotlidec` libraries to provide - implementation of the Brotli compression algorithm. They are - available for download from the official repository - . - - - cURL and its library. If your distribution does not provide it, you - can get it from . - - - The SQLite embedded database library, version 3.6.19 or higher. If - your distribution does not provide it, please install it from - . - - - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce - the evaluator’s memory consumption (optional). - - To enable it, install - `pkgconfig` and the Boehm garbage collector, and pass the flag - `--enable-gc` to `configure`. - - - The `boost` library of version 1.66.0 or higher. It can be obtained - from the official web site . - - - The `editline` library of version 1.14.0 or higher. It can be - obtained from the its repository - . - - - The `libsodium` library for verifying cryptographic signatures - of contents fetched from binary caches. - It can be obtained from the official web site - . - - - Recent versions of Bison and Flex to build the parser. (This is - because Nix needs GLR support in Bison and reentrancy support in - Flex.) For Bison, you need version 2.6, which can be obtained from - the [GNU FTP server](ftp://alpha.gnu.org/pub/gnu/bison). For Flex, - you need version 2.5.35, which is available on - [SourceForge](http://lex.sourceforge.net/). Slightly older versions - may also work, but ancient versions like the ubiquitous 2.5.4a - won't. - - - The `libseccomp` is used to provide syscall filtering on Linux. This - is an optional dependency and can be disabled passing a - `--disable-seccomp-sandboxing` option to the `configure` script (Not - recommended unless your system doesn't support `libseccomp`). To get - the library, visit . - - - On 64-bit x86 machines only, `libcpuid` library - is used to determine which microarchitecture levels are supported + This list and lower version bounds are maintained on best-effort basis. When in doubt, check the `meson.build` files. + + - Meson build system (). + + - Ninja (). + + - A version of GCC or Clang that supports C++23 (anything newer than Clang 19 or GCC 14 is likely to work). + + - `pkg-config` to locate dependencies. + If your distribution does not provide it, you can get it from . + + - The OpenSSL library to calculate cryptographic hashes. + If your distribution does not provide it, you can get it from . + + - The `libbrotlienc` and `libbrotlidec` libraries to provide implementation of the Brotli compression algorithm. + They are available for download from the official repository . + + - cURL library. + If your distribution does not provide it, you can get it from . + + - The SQLite embedded database library, version 3.6.19 or higher. + If your distribution does not provide it, please install it from . + + - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce the evaluator’s memory consumption (optional). + To enable it, install `pkgconfig` and the Boehm garbage collector, and pass the option `-Dlibexpr:gc=enabled` to `meson setup`. + + - The `boost` library of version 1.87.0 or higher. + It can be obtained from the official web site . + + - The `editline` library of version 1.14.0 or higher. + It can be obtained from the its repository . + + - The `libsodium` library for verifying cryptographic signatures of contents fetched from binary caches. + It can be obtained from the official web site . + + - Recent versions of Bison and Flex to build the parser. + (This is because Nix needs C++ template support in Bison and reentrancy support in Flex.) + + - The `libseccomp` is used to provide syscall filtering on Linux. + This is an optional dependency and can be disabled passing a `-Dlibstore:seccomp-sandboxing=disabled` option to the `meson setup` command + (Not recommended unless your system doesn't support `libseccomp`). + To get the library, visit . + + - On 64-bit x86 machines only, `libcpuid` library is used to determine which microarchitecture levels are supported (e.g., as whether to have `x86_64-v2-linux` among additional system types). - The library is available from its homepage - . - This is an optional dependency and can be disabled - by providing a `--disable-cpuid` to the `configure` script. - - - Unless `meson setup build -Dunit-tests=false` is specified, GoogleTest (GTest) and - RapidCheck are required, which are available at - and - respectively. + The library is available from its homepage . + This is an optional dependency and can be disabled by providing a `-Dlibutil:cpuid=disabled` option to `meson setup` script. + + - Unless `meson setup build -Dunit-tests=false` is specified, GoogleTest (GTest) and RapidCheck are required, which are available at + and respectively. From 5d2600a01dfc0e722ddbb66d9ae3b4b65cec41ae Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Wed, 17 Jun 2026 13:49:11 -0400 Subject: [PATCH 515/555] Fix exceptions not being caught in nix::Pid::~Pid Using function try blocks with destructors causes the exception to be rethrown instead of properly caught. --- src/libutil/unix/processes.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 7accf2e1759e..32d59535883a 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -50,11 +50,13 @@ Pid::Pid(pid_t pid) } Pid::~Pid() -try { - if (pid != -1) - kill(/*allowInterrupts=*/false); -} catch (...) { - ignoreExceptionInDestructor(); +{ + try { + if (pid != -1) + kill(/*allowInterrupts=*/false); + } catch (...) { + ignoreExceptionInDestructor(); + } } void Pid::operator=(pid_t pid) From c37e17c280934a6ca667984151516b0bb6212de4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 20:35:42 +0300 Subject: [PATCH 516/555] Increase the sourceToSink/sinkToSource coroutine stack sizes We had too little headroom previously, since the default stack size if 128KiB and half of that is typically consumed by a 64KiB buffer for copying between Source/Sink. 512KiB should be more than enough for the limit that we have (64 levels in NARs , which usually also bounds the recursion depth with some constant factor). --- src/libutil/archive.cc | 3 +-- src/libutil/serialise.cc | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 8d9b833c29aa..56172bf32b9a 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -36,8 +36,7 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); /* Maximum directory nesting depth for dumpPath()/parseDump(). Bounds stack usage so deep trees cannot overflow the (possibly coroutine) - stack these run on. Chosen to fit comfortably in the default 128 KiB - boost coroutine stack. */ + stack these run on. */ static constexpr size_t narMaxDepth = 64; PathFilter defaultPathFilter = [](const std::string &) { return true; }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0529c2ca2f89..eb1209843179 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -369,6 +369,13 @@ void StringSource::skip(size_t len) pos += len; } +/* 512KiB is a conservative estimate for deeply nested NARs, which are limited + to 64 levels. We also tend to allocate rather large buffers on the stack, so + we should leave plenty of headroom. Note that no evaluation is supposed to + happen on sourceToSink/sinkToSource coroutine stacks (for Boehm GC reasons), + which requires much more stack space. */ +static constexpr size_t defaultCoroutineStackSize = 512 * 1024; + std::unique_ptr sourceToSink(fun reader) { struct SourceToSink : FinishSink @@ -392,8 +399,9 @@ std::unique_ptr sourceToSink(fun reader) cur = in; if (!coro) { - coro = - coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { + coro = coro_t::push_type( + boost::coroutines2::protected_fixedsize_stack(defaultCoroutineStackSize), + [&](coro_t::pull_type & yield) { LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { yield(); @@ -450,8 +458,9 @@ std::unique_ptr sinkToSource(fun writer, fun eof) { bool hasCoro = coro.has_value(); if (!hasCoro) { - coro = - coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { + coro = coro_t::pull_type( + boost::coroutines2::protected_fixedsize_stack(defaultCoroutineStackSize), + [&](coro_t::push_type & yield) { /* Feed the consumer in chunks, instead of on each write to avoid excessive context switching. parseDump does lots of small writes to the sink, which we should From 9586b80af2c75d9ffefc01402a4e15c63847ebbd Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 17 Jun 2026 21:04:34 +0300 Subject: [PATCH 517/555] Print attributes in repl commands (:env or :st) in lexicographic order Depending on how nix is built, the order of variables in the environments can differ [1], due to static initialisation order. Technically, in non-unity builds the order is fully undefined due to the static initialisation order fiasco. I recently added tests for this functionality and they started failing in nixpkgs, because it doesn't use unity builds. [1]: https://github.com/NixOS/nixpkgs/pull/532575 --- src/libexpr/eval.cc | 27 ++++++++++++------- .../repl/debugger-fail-throw-env-1.expected | 2 +- .../repl/debugger-fail-throw-env-2.expected | 2 +- .../repl/debugger-okay-ignore-try.expected | 8 +++--- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 81ed77c90593..069d7e2c5190 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -667,27 +667,34 @@ std::optional EvalState::getDoc(Value & v) return {}; } +static StaticEnv::Vars lexicographicOrder(const SymbolTable & st, StaticEnv::Vars vars) +{ + std::ranges::sort(vars, [&st](const auto & lhs, const auto & rhs) { + return std::string_view(st[lhs.first]) < std::string_view(st[rhs.first]); + }); + return vars; +} + // just for the current level of StaticEnv, not the whole chain. -void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) +static void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) { std::cout << ANSI_MAGENTA; - for (auto & i : se.vars) - std::cout << st[i.first] << " "; + for (auto & [name, displacement] : lexicographicOrder(st, se.vars)) + std::cout << st[name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } // just for the current level of Env, not the whole chain. -void printWithBindings(const SymbolTable & st, const Env & env) +static void printWithBindings(const SymbolTable & st, const Env & env) { if (!env.values[0]->isThunk()) { std::cout << "with: "; std::cout << ANSI_MAGENTA; - auto j = env.values[0]->attrs()->begin(); - while (j != env.values[0]->attrs()->end()) { - std::cout << st[j->name] << " "; - ++j; - } + auto * bindings = env.values[0]->attrs(); + /* TODO: Don't print the whole attribute set, since it can be quite large. */ + for (const Attr * attr : bindings->lexicographicOrder(st)) + std::cout << st[attr->name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } @@ -708,7 +715,7 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::cout << ANSI_MAGENTA; // for the top level, don't print the double underscore ones; // they are in builtins. - for (auto & i : se.vars) + for (auto & i : lexicographicOrder(st, se.vars)) if (!hasPrefix(st[i.first], "__")) std::cout << st[i.first] << " "; std::cout << ANSI_NORMAL; diff --git a/tests/functional/repl/debugger-fail-throw-env-1.expected b/tests/functional/repl/debugger-fail-throw-env-1.expected index 865aa58eaf12..921395b8f928 100644 --- a/tests/functional/repl/debugger-fail-throw-env-1.expected +++ b/tests/functional/repl/debugger-fail-throw-env-1.expected @@ -8,7 +8,7 @@ Env level 0 static: _ Env level 1 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :quit error: diff --git a/tests/functional/repl/debugger-fail-throw-env-2.expected b/tests/functional/repl/debugger-fail-throw-env-2.expected index 251da649f3ec..3eb47bdf53aa 100644 --- a/tests/functional/repl/debugger-fail-throw-env-2.expected +++ b/tests/functional/repl/debugger-fail-throw-env-2.expected @@ -15,7 +15,7 @@ Env level 2 static: x Env level 3 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :bt diff --git a/tests/functional/repl/debugger-okay-ignore-try.expected b/tests/functional/repl/debugger-okay-ignore-try.expected index 23a5f476ac41..7d4e72fdaeab 100644 --- a/tests/functional/repl/debugger-okay-ignore-try.expected +++ b/tests/functional/repl/debugger-okay-ignore-try.expected @@ -29,7 +29,7 @@ Env level 0 static: someFailingExpr tried Env level 1 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: undefined variable 'bricked' @@ -54,7 +54,7 @@ Env level 1 static: someFailingExpr tried Env level 2 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: infinite recursion encountered @@ -85,7 +85,7 @@ Env level 1 static: someFailingExpr tried Env level 2 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: infinite recursion encountered @@ -110,7 +110,7 @@ Env level 1 static: someFailingExpr tried Env level 2 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: infinite recursion encountered From 1ba3bc047c6d0494f13c4138268bca8392c1ad4c Mon Sep 17 00:00:00 2001 From: Adam Dinwoodie Date: Wed, 17 Jun 2026 20:12:52 +0100 Subject: [PATCH 518/555] worker-settings: speed factor is non-integer The value of the speed factor when specifying a remote builder can be any positive number. Correct the comments / documentation to reflect that fact. --- src/libstore/include/nix/store/worker-settings.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/include/nix/store/worker-settings.hh b/src/libstore/include/nix/store/worker-settings.hh index 32c2b3684269..d78ca420f776 100644 --- a/src/libstore/include/nix/store/worker-settings.hh +++ b/src/libstore/include/nix/store/worker-settings.hh @@ -185,7 +185,7 @@ public: 4. The maximum number of builds that Nix executes in parallel on the machine. Typically this should be equal to the number of CPU cores. - 5. The “speed factor”, indicating the relative speed of the machine as a positive integer. + 5. The “speed factor”, indicating the relative speed of the machine as a positive integer or decimal number. If there are multiple machines of the right type, Nix prefers the fastest, taking load into account. 6. A comma-separated list of supported [system features](#conf-system-features). From f88c306340939391bdec8e3cd9d753c4f6f5ccf2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 23:09:28 +0300 Subject: [PATCH 519/555] Unbreak the formatting of the builtins in the manual 2a71fbf41017b4b9ad87f7c6fa5506b80743e1a7 broke the manual. Also the documentation was quite strange in some places (and slightly inaccurate). Using inline documentation is much more readable and doesn't have the formatting footguns. --- src/libexpr/primops.cc | 199 ++++++----------------------------------- 1 file changed, 28 insertions(+), 171 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 951c6e74607b..1a837a1b96ff 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3090,11 +3090,7 @@ static RegisterPrimOp primop_attrNames({ alphabetically sorted list. For instance, `builtins.attrNames { y = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`. - # Time Complexity - - - O(n log n), where: - - n = number of attributes in the set + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrNames, }); @@ -3128,11 +3124,7 @@ static RegisterPrimOp primop_attrValues({ Return the values of the attributes in the set *set* in the order corresponding to the sorted attribute names. - # Time Complexity - - - O(n log n), where: - - n = number of attributes in the set + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrValues, }); @@ -3159,9 +3151,7 @@ static RegisterPrimOp primop_getAttr({ the `.` operator, since *s* is an expression rather than an identifier. - # Time Complexity - - O(log n) where n = number of attributes in the set + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_getAttr, }); @@ -3251,9 +3241,7 @@ static RegisterPrimOp primop_hasAttr({ `false` otherwise. This is a dynamic version of the `?` operator, since *s* is an expression rather than an identifier. - # Time Complexity - - O(log n) where n = number of attributes in the set + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_hasAttr, }); @@ -3314,12 +3302,7 @@ static RegisterPrimOp primop_removeAttrs({ evaluates to `{ y = 2; }`. - # Time Complexity - - O(n + k log k) where: - - n = number of attributes in input set - k = number of attribute names to remove + Has `O(n + k log k)` time complexity, where `n` is number of attributes in the *set* and `k` is the size of *list*. )", .impl = prim_removeAttrs, }); @@ -3408,9 +3391,7 @@ static RegisterPrimOp primop_listToAttrs({ { foo = 123; bar = 456; } ``` - # Time Complexity - - O(n log n) where n = number of list elements + Has `O(n log n)` time complexity, where `n` is size of the list. )", .impl = prim_listToAttrs, }); @@ -3487,12 +3468,7 @@ static RegisterPrimOp primop_intersectAttrs({ Return a set consisting of the attributes in the set *e2* which have the same name as some attribute in *e1*. - # Time Complexity - - O(n * log m) where: - - n = number of attributes in the smaller set - m = number of attributes in the larger set + Has `O(n log m)` time complexity, where `n` and `m` are the sizes of the smallest and largest set respectively. )", .impl = prim_intersectAttrs, }); @@ -3533,12 +3509,7 @@ static RegisterPrimOp primop_catAttrs({ evaluates to `[1 2]`. - # Time Complexity - - O(n * log m) where: - - n = list length - m = number of attributes per set + Has `O(n)` time complexity, where `n` is the size of the *list*. )", .impl = prim_catAttrs, }); @@ -3583,9 +3554,7 @@ static RegisterPrimOp primop_functionArgs({ the function. Plain lambdas are not included, e.g. `functionArgs (x: ...) = { }`. - # Time Complexity - - O(n) where n = number of formal arguments + Has constant time complexity. )", .impl = prim_functionArgs, }); @@ -3619,13 +3588,9 @@ static RegisterPrimOp primop_mapAttrs({ evaluates to `{ a = 10; b = 20; }`. - # Time Complexity - - O(n) where: - - n = number of attributes - - Calls to `f` are performed afterwards, when needed. + Has `O(n)` time complexity, where `n` is the size of the *attrset*. + Note that no calls to *f* are performed by the builtin. + The function *f* is called on demand when a resulting attribute value is evaluated. )", .impl = prim_mapAttrs, }); @@ -3714,12 +3679,7 @@ static RegisterPrimOp primop_zipAttrsWith({ } ``` - # Time Complexity - - O(N * log k) where: - - N = total attributes across all sets - k = number of unique keys across all sets + Has `O(n log n)` time complexity, where `n` is the number of attributes across all sets. )", .impl = prim_zipAttrsWith, }); @@ -3787,9 +3747,7 @@ static RegisterPrimOp primop_head({ isn’t a list or is an empty list. You can test whether a list is empty by comparing it with `[]`. - # Time Complexity - - O(1) + Has constant time complexity. )", .impl = prim_head, }); @@ -3821,10 +3779,6 @@ static RegisterPrimOp primop_tail({ > This function should generally be avoided since it's inefficient: > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. - - # Time Complexity - - O(n) where n = list length (copies n-1 elements) )", .impl = prim_tail, }); @@ -3860,13 +3814,9 @@ static RegisterPrimOp primop_map({ evaluates to `[ "foobar" "foobla" "fooabc" ]`. - # Time Complexity - - O(n) where: - - n = list length - - Calls to `f` are performed afterwards when needed. + Has `O(n)` time complexity, where `n` is the size of the *list*. + Note that no calls to *f* are performed by the builtin, but *f* itself is evaluated and its type is checked eagerly. + The function *f* is called on demand when a resulting list element is evaluated. )", .impl = prim_map, }); @@ -3916,13 +3866,7 @@ static RegisterPrimOp primop_filter({ .doc = R"( Return a list consisting of the elements of *list* for which the function *f* returns `true`. - - # Time Complexity - - O(n * T_f) (eager; predicate is forced) where: - - n = list length - T_f = predicate evaluation time + Has linear time complexity in the size of the input *list*. )", .impl = prim_filter, }); @@ -3946,15 +3890,7 @@ static RegisterPrimOp primop_elem({ .doc = R"( Return `true` if a value equal to *x* occurs in the list *xs*, and `false` otherwise. - - # Time Complexity - - O(n * T) (worst case) where: - - n = list length - T = time to compare two elements - - returns early if the elements is found + Short-circuits and does not evaluate elements that occur in the list after the first match. )", .impl = prim_elem, }); @@ -3972,12 +3908,6 @@ static RegisterPrimOp primop_concatLists({ .args = {"lists"}, .doc = R"( Concatenate a list of lists into a single list. - - # Time Complexity - - O(N) where: - - N = total number of elements across all lists )", .impl = prim_concatLists, }); @@ -3994,10 +3924,6 @@ static RegisterPrimOp primop_length({ .args = {"e"}, .doc = R"( Return the length of the list *e*. - - # Time Complexity - - O(1) )", .impl = prim_length, }); @@ -4063,12 +3989,7 @@ static RegisterPrimOp primop_foldlStrict({ but lacks these benefits. See also [Nixpkgs `lib.foldl`](https://nixos.org/manual/nixpkgs/unstable/#function-library-lib.lists.foldl). - # Time Complexity - - O(n * T_op) where: - - n = list length - T_op = `op` call evaluation time + Has linear time complexity in the size of the list. )", .impl = prim_foldlStrict, }); @@ -4107,15 +4028,7 @@ static RegisterPrimOp primop_any({ .doc = R"( Return `true` if the function *pred* returns `true` for at least one element of *list*, and `false` otherwise. - - # Time Complexity - - O(n * T_pred) where: - - - n = `list` length - - T_pred = `pred` call evaluation time - - returns early when `pred` returns `true` + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `true`. )", .impl = prim_any, }); @@ -4131,15 +4044,7 @@ static RegisterPrimOp primop_all({ .doc = R"( Return `true` if the function *pred* returns `true` for all elements of *list*, and `false` otherwise. - - # Time Complexity - - O(n * T_f) where: - - - n = list length - - T_f = predicate evaluation time - - returns early when `pred` returns `false` + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `false`. )", .impl = prim_all, }); @@ -4179,16 +4084,7 @@ static RegisterPrimOp primop_genList({ returns the list `[ 0 1 4 9 16 ]`. - # Time Complexity - - Complexity of `genList generator n`: O(n) - - Complexity of `deepSeq (genList generator n)`: O(n * T_f) - - where: - - n = requested length - T_f = `generator` call evaluation time + Has linear time complexity. )", .impl = prim_genList, }); @@ -4300,15 +4196,8 @@ static RegisterPrimOp primop_sort({ If the *comparator* violates any of these properties, then `builtins.sort` reorders elements in an unspecified manner. - # Time Complexity - - O(n log n * T_cmp), where: - - n = `list` length - T_cmp = `comparator` call evaluation time - - Uses an adaptive sort that exploits existing sorted runs in the input, - down to O(n * T_cmp) when the list is already sorted. + Runs in `O(n log n)` time on average, where `n` is the size of the *list*. + Uses an adaptive sort that exploits existing sorted runs in the input, down to `O(n)` when the list is already sorted. )", .impl = prim_sort, }); @@ -4371,12 +4260,7 @@ static RegisterPrimOp primop_partition({ { right = [ 23 42 ]; wrong = [ 1 9 3 ]; } ``` - # Time Complexity - - O(n * T_pred) where: - - n = list length - T_pred = `pred` call evaluation time + Runs in linear time in the size of the *list*. )", .impl = prim_partition, }); @@ -4431,13 +4315,7 @@ static RegisterPrimOp primop_groupBy({ { b = [ "bar" "baz" ]; f = [ "foo" ]; } ``` - # Time Complexity - - O(N * T_f + N * log k) where: - - N = number of `list` elements - T_f = `f` call evaluation time - k = number of unique groups + Has `O(n log n)` time complexity, where `n` is the size of the input *list*. )", .impl = prim_groupBy, }); @@ -4480,14 +4358,6 @@ static RegisterPrimOp primop_concatMap({ .doc = R"( This function is equivalent to `builtins.concatLists (map f list)` but is more efficient. - - # Time Complexity - - O(k * T_f + N) where: - - k = length of input list - T_f = time to call `f` on an element - N = total number of elements returned by `f` calls )", .impl = prim_concatMap, }); @@ -5191,13 +5061,6 @@ static RegisterPrimOp primop_concatStringsSep({ Concatenate a list of strings with a separator between each element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] == "usr/local/bin"`. - - # Time Complexity - - O(n + m) (amortized) where: - - n = number of list elements - m = total length of output string )", .impl = prim_concatStringsSep, }); @@ -5283,13 +5146,7 @@ static RegisterPrimOp primop_replaceStrings({ evaluates to `"fabir"`. - # Time Complexity - - O(n * k * c) (worst case) where: - - n = length of input string - k = number of replacement patterns - c = average length of patterns in 'from' list + Has `O(n k)` time complexity, where `n` is the length of *s* and `k` is the number of replacements. )", .impl = prim_replaceStrings, }); From b308d8cea8f79fb2d0cee2c5732524f3c68ba4ea Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 17 Jun 2026 22:50:23 +0200 Subject: [PATCH 520/555] doc: Generalize the JSON guidelines to "Data Modeling Guidelines" They were kind of the same thing and we've been applying them more broadly on occasion. --- doc/manual/source/SUMMARY.md.in | 2 +- doc/manual/source/_redirects | 3 ++- .../{json-guideline.md => data-modeling.md} | 11 ++++++++--- doc/manual/source/release-notes/rl-2.23.md | 4 ++-- doc/manual/source/release-notes/rl-2.33.md | 2 +- src/libmain/include/nix/main/common-args.hh | 4 ++-- 6 files changed, 16 insertions(+), 10 deletions(-) rename doc/manual/source/development/{json-guideline.md => data-modeling.md} (86%) diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 5a17426b9020..a73446a99021 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -149,7 +149,7 @@ - [Debugging](development/debugging.md) - [Documentation](development/documentation.md) - [CLI guideline](development/cli-guideline.md) - - [JSON guideline](development/json-guideline.md) + - [Data Modeling Guidelines](development/data-modeling.md) - [C++ style guide](development/cxx.md) - [Static Analysis](development/static-analysis.md) - [Experimental Features](development/experimental-features.md) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 7e4557f7d595..17aba776a376 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -27,7 +27,8 @@ /contributing/documentation /development/documentation 301! /contributing/experimental-features /development/experimental-features 301! /contributing/cli-guideline /development/cli-guideline 301! -/contributing/json-guideline /development/json-guideline 301! +/contributing/json-guideline /development/data-modeling 301! +/development/json-guideline /development/data-modeling 301! /contributing/cxx /development/cxx 301! /expressions/expression-language /language/ 301! diff --git a/doc/manual/source/development/json-guideline.md b/doc/manual/source/development/data-modeling.md similarity index 86% rename from doc/manual/source/development/json-guideline.md rename to doc/manual/source/development/data-modeling.md index 309b4b3a06e4..a9d131147588 100644 --- a/doc/manual/source/development/json-guideline.md +++ b/doc/manual/source/development/data-modeling.md @@ -1,7 +1,12 @@ -# JSON guideline +# Data Modeling Guidelines -Nix consumes and produces JSON in a variety of contexts. -These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. +Nix consumes and produces JSON and attribute sets in a variety of contexts. +These guidelines ensure consistent practices for our interfaces, for ease of use, and so that experience in one part carries over to another. + +For these guidelines, we will use JSON terminology, but they apply equally well to new attribute set interfaces (primops, etc.). +Note that these are guidelines first and foremost. Exceptions include: +- Feature testing: e.g., it is OK to do `builtins?frobnicate`. +- Compatibility: we generally do not change stable interfaces just to make them comply. New replacements can be added with care. ## Extensibility diff --git a/doc/manual/source/release-notes/rl-2.23.md b/doc/manual/source/release-notes/rl-2.23.md index b358a0fdc3c3..92e5f4599440 100644 --- a/doc/manual/source/release-notes/rl-2.23.md +++ b/doc/manual/source/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/json-guideline.md). + The JSON format for derivations has been slightly revised to better conform to our [data modeling guidelines](@docroot@/development/data-modeling.md). In particular, the hash algorithm and content addressing method of content-addressed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. @@ -89,7 +89,7 @@ This makes records of this sort more self-describing, and easier to consume programmatically. We will follow this design principle going forward; - the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + the [data modeling guidelines](@docroot@/development/data-modeling.md) in the contributing section have been updated accordingly. - Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) diff --git a/doc/manual/source/release-notes/rl-2.33.md b/doc/manual/source/release-notes/rl-2.33.md index bed697029389..cc5781f37314 100644 --- a/doc/manual/source/release-notes/rl-2.33.md +++ b/doc/manual/source/release-notes/rl-2.33.md @@ -135,7 +135,7 @@ This is the legacy format, preserved for backwards compatibility: ### Version 2 (`--json-format 2`) -The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes: +The new structured format follows the [data modeling guidelines](@docroot@/development/data-modeling.md) with the following changes: - **Nested structure with top-level metadata**: diff --git a/src/libmain/include/nix/main/common-args.hh b/src/libmain/include/nix/main/common-args.hh index d67fc2ad0c47..b20df3a99ec9 100644 --- a/src/libmain/include/nix/main/common-args.hh +++ b/src/libmain/include/nix/main/common-args.hh @@ -81,8 +81,8 @@ struct MixPrintJSON : virtual Args * This is a template to avoid accidental coercions from `string` to `json` in the caller, * to avoid mistakenly passing an already serialized JSON to this function. * - * It is not recommended to print a JSON string - see the JSON guidelines - * about extensibility, https://nix.dev/manual/nix/development/development/json-guideline.html - + * It is not recommended to print a JSON string - see the data modeling guidelines + * about extensibility, https://nix.dev/manual/nix/development/development/data-modeling.html - * but you _can_ print a sole JSON string by explicitly coercing it to * `nlohmann::json` first. */ From 5a3d3986e6af6a6b43027ef0c767f3d39d855505 Mon Sep 17 00:00:00 2001 From: Tom Hunze Date: Sat, 4 Apr 2026 12:18:03 +0200 Subject: [PATCH 521/555] nix develop: use `runPhase` to run phases, fall back to `runHook` Using `runHook`, `nix develop --phase ` still executes the generic `Phase` when `Phase` is defined when calling `stdenv.mkDerivation` [1]. `runPhase` was specifically added to avoid this problem and to be more consistent with `genericBuild` behavior [2]. [1] https://github.com/NixOS/nix/issues/6202 [2] https://github.com/NixOS/nixpkgs/pull/230874 --- src/nix/develop.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 50b248bcb884..3105efa82536 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -605,7 +605,13 @@ struct CmdDevelop : Common, MixEnvironment // FIXME: foundMakefile is set by buildPhase, need to get // rid of that. script += fmt("foundMakefile=1\n"); - script += fmt("runHook %1%Phase\n", *phase); + script += + fmt("if declare -f runPhase >/dev/null; then\n" + " runPhase %1%Phase\n" + "else\n" + " runHook %1%Phase\n" + "fi\n", + *phase); } else if (!command.empty()) { From 2662dc95aff7b9f77ddf00421d9354de2c57d284 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 18 Jun 2026 23:52:40 +0300 Subject: [PATCH 522/555] Remove UnkeyedValidPathInfo::id This doesn't seem to be used anywhere now. It got added initially in 762cee72ccd860e72c7b639a1dd542ac0f298bb2, where it was used in registerValidPaths. But nowadays we do queryValidPathId for that use-case unconditionally. That can be improved to avoid some unnecessary SQLite queries in case we know the primary id, but it can be done in a much more local way that's not exposed in the interface. --- src/libstore/include/nix/store/path-info.hh | 8 -------- src/libstore/local-store.cc | 4 +--- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index 7ba30359b547..0dd19b81ecf1 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -90,14 +90,6 @@ struct UnkeyedValidPathInfo */ uint64_t narSize = 0; - /** - * internal use only: SQL primary key for on-disk store objects with - * `LocalStore`. - * - * @todo Remove, layer violation - */ - uint64_t id = 0; - /** * Whether the path is ultimately trusted, that is, it's a * derivation output that was built locally. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e4a4ffc988a8..dd5d325239d2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -814,8 +814,6 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s auto info = std::make_shared(path, UnkeyedValidPathInfo(*this, narHash)); - info->id = id; - info->registrationTime = useQueryPathInfo.getInt(2); auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); @@ -836,7 +834,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s info->ca = ContentAddress::parseOpt(s); /* Get the references. */ - auto useQueryReferences(state.stmts->QueryReferences.use().apply(info->id)); + auto useQueryReferences(state.stmts->QueryReferences.use().apply(id)); while (useQueryReferences.next()) info->references.insert(parseStorePath(useQueryReferences.getStr(0))); From f0f0d950795ef2ea2ac5989ab75b000364c2016d Mon Sep 17 00:00:00 2001 From: Michael Wang <41721295+zwang20@users.noreply.github.com> Date: Mon, 8 Jun 2026 21:13:49 +1000 Subject: [PATCH 523/555] Fix formatting in documentation --- doc/manual/source/command-ref/nix-collect-garbage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md index 763179b8ee18..07229255e7cd 100644 --- a/doc/manual/source/command-ref/nix-collect-garbage.md +++ b/doc/manual/source/command-ref/nix-collect-garbage.md @@ -62,9 +62,9 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. - - [`--max-freed`](#opt-max-freed) *bytes* +- [`--max-freed`](#opt-max-freed) *bytes* - + Keep deleting paths until at least *bytes* bytes have been deleted, then stop. The argument *bytes* can be followed by the From 09c8a1b9aac0b041136dd178be6db6666d6fdd75 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Mon, 22 Jun 2026 11:13:27 -0500 Subject: [PATCH 524/555] Fix boost format error when diff hook fails Signed-off-by: Lisanna Dettwyler --- src/libstore/unix/build/derivation-builder.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 94bff9c2bc01..3b02dfc4f172 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -137,8 +137,7 @@ static void handleDiffHook( .gid = gid, .chdir = "/"}); if (!statusOk(diffRes.first)) - throw ExecError( - diffRes.first, "diff-hook program %s %2%", PathFmt(diffHook), statusToString(diffRes.first)); + throw ExecError(diffRes.first, "diff-hook program %s %s", PathFmt(diffHook), statusToString(diffRes.first)); if (diffRes.second != "") printError(chomp(diffRes.second)); From 4df375be378db8ebf5f25ef9549e2b76a1db83e1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 20 Jun 2026 01:41:36 +0300 Subject: [PATCH 525/555] Add test for broken positions on repl :load-file This is my SNAFU from a091a8100a8. I neglected the fact that :l and a bit of other things also nuke the cache. The following commit will have the fix. (cherry picked from commit acfc7d845176c7c6a514ff8527be7266850d495c) --- tests/functional/repl/file-b.nix | 9 ++++++- .../functional/repl/load-and-reload.expected | 21 ++++++++++++++++ ...on-existent-file.in => load-and-reload.in} | 6 +++-- .../reload-with-non-existent-file.expected | 24 ------------------- 4 files changed, 33 insertions(+), 27 deletions(-) create mode 100644 tests/functional/repl/load-and-reload.expected rename tests/functional/repl/{reload-with-non-existent-file.in => load-and-reload.in} (70%) delete mode 100644 tests/functional/repl/reload-with-non-existent-file.expected diff --git a/tests/functional/repl/file-b.nix b/tests/functional/repl/file-b.nix index ddde63c16c42..7416216c9f51 100644 --- a/tests/functional/repl/file-b.nix +++ b/tests/functional/repl/file-b.nix @@ -1 +1,8 @@ -{ fromB = 2; } +{ + fromBFails = throw "b"; + fromB = 2; + /** + Some documentation. + */ + funcFromB = x: x; +} diff --git a/tests/functional/repl/load-and-reload.expected b/tests/functional/repl/load-and-reload.expected new file mode 100644 index 000000000000..1642987fe6bc --- /dev/null +++ b/tests/functional/repl/load-and-reload.expected @@ -0,0 +1,21 @@ +Nix +Type :? for help. + +nix-repl> :l file-b.nix +Added 3 variables. +fromB, fromBFails, funcFromB + +nix-repl> :l ./does-not-exist.nix +error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist + +nix-repl> :p fromBFails +error: + … while calling the 'throw' builtin + at «string»:1:18: + 1| fromBFails + | ^ + + error: b + +nix-repl> :doc funcFromB +error: basic_string::substr: __pos (which is 3) > this->size() (which is 0) diff --git a/tests/functional/repl/reload-with-non-existent-file.in b/tests/functional/repl/load-and-reload.in similarity index 70% rename from tests/functional/repl/reload-with-non-existent-file.in rename to tests/functional/repl/load-and-reload.in index 740b3be9a412..a8b94a265bf2 100644 --- a/tests/functional/repl/reload-with-non-existent-file.in +++ b/tests/functional/repl/load-and-reload.in @@ -1,5 +1,7 @@ -:l file-a.nix -:l ./does-not-exist.nix :l file-b.nix +:l ./does-not-exist.nix +:p fromBFails +:doc funcFromB +:l file-a.nix :r fromA + fromB diff --git a/tests/functional/repl/reload-with-non-existent-file.expected b/tests/functional/repl/reload-with-non-existent-file.expected deleted file mode 100644 index e15be6e73085..000000000000 --- a/tests/functional/repl/reload-with-non-existent-file.expected +++ /dev/null @@ -1,24 +0,0 @@ -Nix -Type :? for help. - -nix-repl> :l file-a.nix -Added 1 variables. -fromA - -nix-repl> :l ./does-not-exist.nix -error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist - -nix-repl> :l file-b.nix -Added 1 variables. -fromB - -nix-repl> :r -Loading "file-a.nix"... -Added 1 variables. -fromA -Loading "file-b.nix"... -Added 1 variables. -fromB - -nix-repl> fromA + fromB -3 From dbcc91a1b39f65621da5fe502cc08a0036c3f0b3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 23 Jun 2026 00:27:06 +0300 Subject: [PATCH 526/555] Revert "libexpr: Clear PosTable contents in EvalState::resetFileCache" This reverts commit a091a8100a8587185e579d4cff04381e8e074f12. (cherry picked from commit e6cf0d2dcab3d8b675b817399f715a3fd5337cb4) --- src/libexpr/eval.cc | 1 - .../functional/repl/load-and-reload.expected | 29 ++++++++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 069d7e2c5190..29ecec69ee64 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1189,7 +1189,6 @@ void EvalState::resetFileCache() fileEvalCache->clear(); inputCache->clear(); lookupPathResolved->clear(); - positions.clear(); rootFS->invalidateCache(); } diff --git a/tests/functional/repl/load-and-reload.expected b/tests/functional/repl/load-and-reload.expected index 1642987fe6bc..7f953e28613a 100644 --- a/tests/functional/repl/load-and-reload.expected +++ b/tests/functional/repl/load-and-reload.expected @@ -11,11 +11,32 @@ error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist nix-repl> :p fromBFails error: … while calling the 'throw' builtin - at «string»:1:18: - 1| fromBFails - | ^ + at /path/to/tests/functional/repl/file-b.nix:2:16: + 1| { + 2| fromBFails = throw "b"; + | ^ + 3| fromB = 2; error: b nix-repl> :doc funcFromB -error: basic_string::substr: __pos (which is 3) > this->size() (which is 0) +Function `funcFromB`\ + … defined at /path/to/tests/functional/repl/file-b.nix:7:15 + + +Some documentation. + +nix-repl> :l file-a.nix +Added 1 variables. +fromA + +nix-repl> :r +Loading "file-b.nix"... +Added 3 variables. +fromB, fromBFails, funcFromB +Loading "file-a.nix"... +Added 1 variables. +fromA + +nix-repl> fromA + fromB +3 From 1c7a37c6d70285f415c96a85484dce9188d3a522 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 23 Jun 2026 01:01:55 +0300 Subject: [PATCH 527/555] Align readline/editline final prompt behavior Aligning these different behaviours seems fraught, so this is like an easier solution. See: https://hydra.nixos.org/build/331681294/log (cherry picked from commit d8e330749a23511794b6310d076086d754b21c78) --- src/libcmd/include/nix/cmd/repl-interacter.hh | 4 +++ src/libcmd/repl-interacter.cc | 28 ++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/libcmd/include/nix/cmd/repl-interacter.hh b/src/libcmd/include/nix/cmd/repl-interacter.hh index 7cba481059c9..9b9a03c3e423 100644 --- a/src/libcmd/include/nix/cmd/repl-interacter.hh +++ b/src/libcmd/include/nix/cmd/repl-interacter.hh @@ -1,8 +1,10 @@ #pragma once /// @file +#include "nix/util/file-descriptor.hh" #include "nix/util/finally.hh" #include "nix/util/fun.hh" +#include "nix/util/terminal.hh" #include "nix/util/types.hh" #include #include @@ -39,6 +41,8 @@ public: class ReadlineLikeInteracter : public virtual ReplInteracter { std::filesystem::path historyFile; + bool isInteractive = nix::isTTY(getStandardInput()); + public: ReadlineLikeInteracter(std::filesystem::path historyFile) : historyFile(std::move(historyFile)) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 81240af7f547..5f2417dc1331 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -203,8 +203,25 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT setupSignals(); #endif - char * s = readline(promptForType(promptType)); - Finally doFree([&]() { free(s); }); + + /* Buffer for the non-interactive input. */ + std::string buffer; + const char * s = nullptr; + char * rl = nullptr; + + /* Use plain std::getline for non-interactive mode, which we also use for + testing purposes. readline/editline seem to disagree too much about how + to handle final prompts etc., so it's easier to bypass those. The tests + are mostly about testing the core repl logic, not input handling. */ + if (isInteractive) { + rl = ::readline(promptForType(promptType)); + s = rl; + } else { + s = std::getline(std::cin, buffer) ? buffer.c_str() : nullptr; + } + + Finally doFree([&]() { ::free(rl); }); + #ifndef _WIN32 // TODO use more signals.hh for this restoreSignals(); #endif @@ -215,15 +232,12 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return true; } - // editline doesn't echo the input to the output when non-interactive, unlike readline - // this results in a different behavior when running tests. The echoing is - // quite useful for reading the test output, so we add it here. + /* Echo the prompt into the output if run in non-interactive mode, somewhat + for the purposes of characterisation tests. */ if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") { -#if !USE_READLINE // This is probably not right for multi-line input, but we don't use that // in the characterisation tests, so it's fine. std::cout << promptForType(promptType) << s << std::endl; -#endif } if (!s) From 07ffc599aa22780040173c9cc1368effbc24731b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 23 Jun 2026 21:08:37 +0300 Subject: [PATCH 528/555] Fix #15916 The core of the issue is that the trampoline goal reported an arbitrary exitCode, which without --keep-going ends up cancelling goals and we end up dying with an assert in Goal::amDone(). See: https://github.com/NixOS/nix/pull/16042#issuecomment-4760250433 (cherry picked from commit f2c1d45dfc54a263198613c9499c3715aaef4edc) --- .../build/derivation-trampoline-goal.cc | 79 ++++++++++++++++--- tests/functional/meson.build | 1 + .../multiple-outputs-substitute-failure.sh | 28 +++++++ 3 files changed, 97 insertions(+), 11 deletions(-) create mode 100755 tests/functional/multiple-outputs-substitute-failure.sh diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index edf8d1e86ebc..14864ed052f8 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -2,6 +2,9 @@ #include "nix/store/build/worker.hh" #include "nix/store/derivations.hh" +#include +#include + namespace nix { DerivationTrampolineGoal::DerivationTrampolineGoal( @@ -144,11 +147,14 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation }, wantedOutputs.raw); + /* Must have at least one wanted output. This is assumed below. */ + assert(!resolvedWantedOutputs.empty()); + Goals concreteDrvGoals; /* Build this step! */ - auto sharedDrv = make_ref(std::move(drv)); + auto sharedDrv = make_ref(std::move(drv)); for (auto & output : resolvedWantedOutputs) { auto g = upcast_goal(worker.makeDerivationGoal(drvPath, sharedDrv, output, buildMode, false)); @@ -157,20 +163,71 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation concreteDrvGoals.insert(std::move(g)); } - // Copy on purpose - co_await await(Goals(concreteDrvGoals)); + co_await await(concreteDrvGoals); trace("outer build done"); - auto & g = *concreteDrvGoals.begin(); - buildResult = g->buildResult; - if (auto * successP = buildResult.tryGetSuccess()) - for (auto & g2 : concreteDrvGoals) - if (auto * successP2 = g2->buildResult.tryGetSuccess()) - for (auto && [x, y] : successP2->builtOutputs) - successP->builtOutputs.insert_or_assign(x, y); + if (nrFailed != 0) { + auto gi = std::ranges::find_if(concreteDrvGoals, [](const GoalPtr & goal) -> bool { + auto exitCode = goal->exitCode; + /* Note that without --keep-going waitees might be cancelled before + we are woken up. */ + return exitCode != ecBusy && exitCode != ecSuccess; + }); + + const Goal * g = gi->get(); + assert(gi != concreteDrvGoals.end() && "expected a failing goal"); + auto exitCode = g->exitCode; + const auto * failure = g->buildResult.tryGetFailure(); + assert(failure && "failing goal does not report a failed build result"); + + /* Report the exit status of *some* failing goal. This might not be strictly + correct, since multiple subgoals can fail independently, but this should be + a good enough heuristic without --keep-going. */ + co_return doneFailure(exitCode, *failure); + } + + SingleDrvOutputs outputs; + + auto successes = std::views::transform(concreteDrvGoals, [](const GoalPtr & a) -> const BuildResult::Success & { + auto * success = a->buildResult.tryGetSuccess(); + assert(success && "goal succeeded, but some waitees do not report a successful status"); + return *success; + }); + + for (const auto & success : successes) + std::ranges::copy(success.builtOutputs, std::inserter(outputs, outputs.end())); + + auto statuses = successes | std::views::transform(&BuildResult::Success::status); + + /* Aggregate the status code. If some outputs we already valid, but we had + to build/substitute the other ones, report it as the smallest common + denominator. */ + auto compareSuccesses = [](auto a, auto b) { + /* This is technically an identity mapping of the underlying values, but + it would be worse to rely on the enum ordering here. */ + auto toPriority = [](auto st) { + using enum BuildResult::Success::Status; + switch (st) { + case Built: + return 0; + case Substituted: + return 1; + case AlreadyValid: + return 2; + case ResolvesToAlreadyValid: + return 3; + default: + unreachable(); + } + }; + return toPriority(a) < toPriority(b); + }; - co_return amDone(g->exitCode); + co_return doneSuccess({ + .status = std::ranges::min(statuses, compareSuccesses), + .builtOutputs = std::move(outputs), + }); } } // namespace nix diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 7d5110ebc4da..8fdaf4ffe018 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -129,6 +129,7 @@ suites = [ 'logging.sh', 'make-content-addressed.sh', 'misc.sh', + 'multiple-outputs-substitute-failure.sh', 'multiple-outputs.sh', 'nar-access.sh', 'nars.sh', diff --git a/tests/functional/multiple-outputs-substitute-failure.sh b/tests/functional/multiple-outputs-substitute-failure.sh new file mode 100755 index 000000000000..fd4a7ec50b15 --- /dev/null +++ b/tests/functional/multiple-outputs-substitute-failure.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# See https://github.com/NixOS/nix/issues/15916 + +source common.sh + +TODO_NixOS # Requires substituting from a local binary cache, enable when we sign paths in NixOS tests +needLocalStore "'--no-require-sigs' can’t be used with the daemon" + +BINARY_CACHE=file://$cacheDir + +readarray -t outPaths < <(nix build -f multiple-outputs.nix 'independent^*' --no-link --print-out-paths) +[[ ${#outPaths[@]} -eq 2 ]] +nix copy --to "$BINARY_CACHE" "${outPaths[@]}" +for p in "${outPaths[@]}"; do + [[ $p == *-second ]] && secondOut=$p +done + +# Corrupt the second output, so that the substitution partially succeeds. +secondNarInfoFile="$cacheDir/$(basename "$secondOut" | cut -c1-32).narinfo" +sed -i 's|^NarHash:.*|NarHash: sha256:0000000000000000000000000000000000000000000000000000|' "$secondNarInfoFile" + +clearStore +clearCacheCache + +# Note that using "^*" matters here. We want all wanted outputs for the same goal. +expect 1 nix build -j 0 -f multiple-outputs.nix "independent^*" --no-link \ + --substituters "$BINARY_CACHE" --no-require-sigs --substitute 2>&1 | grepQuiet "hash mismatch" From 05d02b88ad73c806a1182a9f2238db081a08a011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 23 Jun 2026 10:01:53 +0000 Subject: [PATCH 529/555] nix-store --register-validity: fix ENOENT under chroot stores With --store local?root=, registerValidity() passed the logical store path to canonicalisePathMetaData(), which then tried to lstat a path that does not exist on the host filesystem and failed with ENOENT. Use toRealPath() so the chroot-prefixed real path is canonicalised instead. ensureLocalStore() is already required at the end of this function, so hoisting the downcast does not narrow the set of accepted stores. (cherry picked from commit d4248c87a4aac4df91282190abe235d55d80b21b) --- src/nix/nix-store/nix-store.cc | 5 +++-- tests/functional/chroot-store.sh | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index e90c7768d9d4..2bb56eb51bca 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -570,6 +570,7 @@ static void opDumpDB(Strings opFlags, Strings opArgs) static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) { + auto localStore = ensureLocalStore(); ValidPathInfos infos; while (1) { @@ -587,7 +588,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) /* !!! races */ if (canonicalise) canonicalisePathMetaData( - store->printStorePath(info->path), + localStore->toRealPath(info->path), {NIX_WHEN_SUPPORT_ACLS(settings.getLocalSettings().ignoredAcls)}); if (!hashGiven) { HashResult hash = hashPath( @@ -601,7 +602,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) } } - ensureLocalStore()->registerValidPaths(infos); + localStore->registerValidPaths(infos); } static void opLoadDB(Strings opFlags, Strings opArgs) diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index cbb80c8710ad..d0f41ee87218 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -60,6 +60,15 @@ PATH7=$(nix path-info --store "local://$TEST_ROOT/x%2Bchroot" "$CORRECT_PATH") # Path gets decoded. [[ ! -d "$TEST_ROOT/x%2Bchroot" ]] +# Regression test: `nix-store --register-validity` against a chroot store must +# canonicalise the real (chroot-prefixed) path, not the logical store path +# (which does not exist on the host filesystem). +regPath=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-reg +touch "$TEST_ROOT/x/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-reg" +[[ ! -e "$regPath" ]] +(echo "$regPath" && echo && echo 0) | nix-store --store "local?root=$TEST_ROOT/x" --register-validity +nix-store --store "$TEST_ROOT/x" --check-validity "$regPath" + # Ensure store info trusted works with local store nix --store "$TEST_ROOT/x" store info --json | jq -e '.trusted' From 779ed6e72fd821077a5c36f59946eedbc524da30 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Tue, 23 Jun 2026 17:07:26 -0500 Subject: [PATCH 530/555] Set MADV_DONTDUMP for bump allocator This should drastically reduce the time needed to generate a coredump. Resolves #16057 Signed-off-by: Lisanna Dettwyler (cherry picked from commit b06d8e6e8b0db0016fe74938c871cb42e1b2f4a3) --- src/libutil/bump-memory-resource.cc | 10 +++++++++- src/libutil/include/nix/util/bump-memory-resource.hh | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libutil/bump-memory-resource.cc b/src/libutil/bump-memory-resource.cc index c892b1652d80..938bd2b84e1b 100644 --- a/src/libutil/bump-memory-resource.cc +++ b/src/libutil/bump-memory-resource.cc @@ -1,4 +1,6 @@ #include "nix/util/bump-memory-resource.hh" +#include "nix/util/environment-variables.hh" +#include "nix/util/error.hh" #include "nix/util/file-system.hh" #include "nix/util/alignment.hh" #include "nix/util/logging.hh" @@ -57,7 +59,7 @@ static bool canOvercommit(std::size_t reserveSize) #endif // _WIN32 -BumpMemoryResource::BumpMemoryResource(std::size_t reserveSize, std::pmr::memory_resource * upstream) +BumpMemoryResource::BumpMemoryResource(std::size_t reserveSize, std::pmr::memory_resource * upstream, bool dontDump) : upstreamResource(upstream) { #ifndef _WIN32 @@ -83,6 +85,12 @@ BumpMemoryResource::BumpMemoryResource(std::size_t reserveSize, std::pmr::memory base = p; capacity = reserveSize; + +# ifdef MADV_DONTDUMP + static const bool dumpEverything = getEnv("_NIX_CORE_DUMP_EVERYTHING").value_or("0") == "1"; + if (!dumpEverything && dontDump && ::madvise(p, reserveSize, MADV_DONTDUMP)) + throw SysError("calling madvise"); +# endif #endif } diff --git a/src/libutil/include/nix/util/bump-memory-resource.hh b/src/libutil/include/nix/util/bump-memory-resource.hh index 15b160f35fe8..9ae6be048588 100644 --- a/src/libutil/include/nix/util/bump-memory-resource.hh +++ b/src/libutil/include/nix/util/bump-memory-resource.hh @@ -39,7 +39,8 @@ public: explicit BumpMemoryResource( std::size_t reserveSize = defaultReserveSize, - std::pmr::memory_resource * upstream = std::pmr::new_delete_resource()); + std::pmr::memory_resource * upstream = std::pmr::new_delete_resource(), + bool dontDump = true); BumpMemoryResource(BumpMemoryResource &&) = delete; BumpMemoryResource(const BumpMemoryResource &) = delete; From 08a209269a27fe087f1aef731e179043d8310089 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:13:04 +0000 Subject: [PATCH 531/555] build(deps): bump nixpkgs in the flake-inputs group Bumps the flake-inputs group with 1 update: [nixpkgs](https://github.com/NixOS/nixpkgs). Updates `nixpkgs` from `bd0ff2d` to `714a5f8` - [Commits](https://github.com/NixOS/nixpkgs/commits) --- updated-dependencies: - dependency-name: nixpkgs dependency-version: 714a5f8c4ead6b31148d829288440ed033ccc041 dependency-type: direct:production dependency-group: flake-inputs ... Signed-off-by: dependabot[bot] (cherry picked from commit d19f68cf337e1e0666c8a3ecd4b678a484a83ae1) --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 85432d06e26a..f1e2fad80273 100644 --- a/flake.lock +++ b/flake.lock @@ -60,11 +60,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1780902259, - "narHash": "sha256-YMnBf9lk/LYgvqfmSSJuOGigtRs5Lsy26pJHVlR9yMY=", - "rev": "bd0ff2d3eac24699c3664d5966b9ef36f388e2ca", + "lastModified": 1782535326, + "narHash": "sha256-r4TA57SL7nvj1R+GY/FCLwFU48w9IixTQlzBTIYkt8E=", + "rev": "714a5f8c4ead6b31148d829288440ed033ccc041", "type": "tarball", - "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.1550.bd0ff2d3eac2/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.3494.714a5f8c4ead/nixexprs.tar.xz" }, "original": { "type": "tarball", From 9dc4d7654a2f0dc9e97e8f490b6d40fde65c6a4e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 26 Jun 2026 15:03:05 +0300 Subject: [PATCH 532/555] installer: Bail out early if running on macOS < 14.0 (cherry picked from commit 9205295fbd46e741c598acb4fced5de0920e9067) --- scripts/install-nix-from-tarball.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh index f17e4c2af3b9..73f389b6ff16 100644 --- a/scripts/install-nix-from-tarball.sh +++ b/scripts/install-nix-from-tarball.sh @@ -28,14 +28,15 @@ fi OS="$(uname -s)" -# macOS support for 10.12.6 or higher +# Since nixpkgs 25.11 the minimum deployment target is macOS 14.0 if [ "$OS" = "Darwin" ]; then + # shellcheck disable=SC2034 IFS='.' read -r macos_major macos_minor macos_patch << EOF $(sw_vers -productVersion) EOF - if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then + if [ "$macos_major" -lt 14 ]; then # patch may not be present; command substitution for simplicity - echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" + echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 14.0 or higher" exit 1 fi fi From 128ce54597975dd8b5a48892fe0edeae0dfaf947 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jun 2026 17:09:58 +0200 Subject: [PATCH 533/555] SourceAccessor::readFile(sink): Remove default implementation This was defined in terms of the non-virtual string variant of readFile(), which is defined in terms of the sink variant. So this could never work. (cherry picked from commit 0db6bc69c1a38bcbbe24a9ff09b963701ea5c917) --- src/libutil/include/nix/util/source-accessor.hh | 3 ++- src/libutil/source-accessor.cc | 7 ------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 0c713fcdb518..0a325de16856 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -81,7 +81,8 @@ public: * @note subclasses of `SourceAccessor` need to implement at least * one of the `readFile()` variants. */ - virtual void readFile(const CanonPath & path, Sink & sink, fun sizeCallback = [](uint64_t size) {}); + virtual void + readFile(const CanonPath & path, Sink & sink, fun sizeCallback = [](uint64_t size) {}) = 0; virtual bool pathExists(const CanonPath & path); diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 8ae914375418..a7cc160a6dc5 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -71,13 +71,6 @@ std::string SourceAccessor::readFile(const CanonPath & path) return std::move(sink.s); } -void SourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) -{ - auto s = readFile(path); - sizeCallback(s.size()); - sink(s); -} - Hash SourceAccessor::hashPath(const CanonPath & path, PathFilter & filter, HashAlgorithm ha) { HashSink sink(ha); From 90823254bdefba32e8737f97da7acec2cb1d23d5 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 29 Jun 2026 02:39:57 +0300 Subject: [PATCH 534/555] libexpr: Handle lazy paths in builtins.storePath better MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building nix.dev manual I ran into an issue with the current handling of lazy paths: … while calling the 'import' builtin at /nix/store/8kpx53qi52yhjai1vdw8zpa95iqa61bv-source/default.nix:167:19: 166| 167| flake = import (outPath + "/flake.nix"); | ^ 168| … while realising the context of a path … while calling the 'storePath' builtin at /nix/store/8kpx53qi52yhjai1vdw8zpa95iqa61bv-source/default.nix:115:15: 114| # If it's already a store path, don't copy it again. 115| builtins.storePath src | ^ 116| else error: path '/nix/store/5fn5lshxlh5zb8w1a747apqlqlbi3j0x-source' is required, but there is no substituter that can build it This also has the benefit of not relying on nix::canonPath(), which is one of the bits of I/O not funneled through the rootFS accessor. The responsible code is in flake-compat [1]. With a path we lack the correct context but it points to a "lazy" path in the store. Funneling I/O into the accessor seems like the correct solution here. [1]: https://github.com/NixOS/flake-compat/blob/5edf11c44bc78a0d334f6334cdaf7d60d732daab/default.nix#L147-L156 (cherry picked from commit 933f3140b1e73de2b909902b0e251c7da4031607) --- src/libexpr/primops.cc | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1a837a1b96ff..46ef49a36759 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1961,25 +1961,21 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V .debugThrow(); NixStringContext context; - auto path = - state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'") - .path; - /* Here we are leaving the realm of the rootFS accessor and must actually fetch to the store. - TODO: This could probably get optimised to avoid the fetching altogether to short-circuit when the path - is already mounted on storeFS. */ - state.ensureLazyPathsCopied(context); + SourcePath sourcePath = state.coerceToPath( + pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'"); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ - if (!state.store->isStorePath(path.abs())) - path = CanonPath(canonPath(path.abs(), true).string()); - if (!state.store->isInStore(path.abs())) - state.error("path '%1%' is not in the Nix store", path).atPos(pos).debugThrow(); - auto path2 = state.store->toStorePath(path.abs()).first; - if (!settings.readOnlyMode) - state.store->ensurePath(path2); - context.insert(NixStringContextElem::Opaque{.path = path2}); - v.mkString(path.abs(), context, state.mem); + if (!state.store->isStorePath(sourcePath.path.abs())) + sourcePath = sourcePath.resolveSymlinks(SymlinkResolution::Full); + if (!state.store->isInStore(sourcePath.path.abs())) + state.error("path '%1%' is not in the Nix store", sourcePath).atPos(pos).debugThrow(); + auto storePath = state.store->toStorePath(sourcePath.path.abs()).first; + if (!state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath))) && !settings.readOnlyMode) + state.store->ensurePath(storePath); + context.insert(NixStringContextElem::Opaque{.path = storePath}); + v.mkString(sourcePath.path.abs(), context, state.mem); } static RegisterPrimOp primop_storePath({ From 199ad670792f9a887e0215ea92b07fec833abd96 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 26 Jun 2026 22:36:12 +0300 Subject: [PATCH 535/555] Fix LocalStoreAccessor::maybeLstat Paths that are not valid store path names (i.e. `foo` or `bar`) can't ever exist in the store, and the correct semantics for it is to return `std::nullopt` instead of throwing. Also noticed a lot of bugs in RemoteFSAccessor, but that's for a later patch. There are also lots of missing overrides of pathExists in SourceAccessor implementations, so I wonder whether that should just be made non-virtual. The evaluator doesn't benefit from potentially optimised pathExists, because MountedSourceAccessor and various other combinators don't propagate them. Plus we should typically still query the lstat to do positive caching (CachingSourceAccessor). See https://github.com/NixOS/nix/issues/16017. (cherry picked from commit 25dbbaf41697511ff09428b89191b2316fb21f59) --- src/libstore/dummy-store.cc | 3 ++ .../include/nix/store/remote-fs-accessor.hh | 2 + src/libstore/local-fs-store.cc | 37 +++++++++++++++---- src/libstore/remote-fs-accessor.cc | 2 + .../include/nix/util/source-accessor.hh | 7 ++++ tests/functional/flakes/follow-paths.sh | 2 +- tests/functional/lang.sh | 2 +- .../functional/lang/eval-okay-pathexists.nix | 3 ++ 8 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index b364c90e160f..6238c8890499 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -69,6 +69,8 @@ class WholeStoreViewAccessor : public SourceAccessor }); if (!res) + /* The accessor is truly empty, i.e. without any file at root so + any subsequent operation with it will fail. */ res = &emptyAccessor; return callback(*res, path); @@ -107,6 +109,7 @@ class WholeStoreViewAccessor : public SourceAccessor DirEntries readDirectory(const CanonPath & path) override { + /* FIXME: Special-case the root directory to read the whole store, not just an empty root. */ return callWithAccessorForPath( path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.readDirectory(path); }); } diff --git a/src/libstore/include/nix/store/remote-fs-accessor.hh b/src/libstore/include/nix/store/remote-fs-accessor.hh index fa7f5fc28052..26fae0d63dd2 100644 --- a/src/libstore/include/nix/store/remote-fs-accessor.hh +++ b/src/libstore/include/nix/store/remote-fs-accessor.hh @@ -33,6 +33,8 @@ public: /** * @return nullptr if the store does not contain any object at that path. + * + * @todo This actually doesn't return nullptr, but throws on invalid paths. */ std::shared_ptr accessObject(const StorePath & path); diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 5be0e5b0673b..a6fd24bf34e6 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -49,13 +49,33 @@ struct LocalStoreAccessor : SourceAccessor { } - void requireStoreObject(const CanonPath & path) + void requireStoreObject(const StorePath & storePath) { - auto [storePath, rest] = store->toStorePath(store->storeDir + path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); } + static StorePath getStoreObjectPath(const CanonPath & path) + { + /* See special handling of isRoot() in maybeLstat. */ + if (path.isRoot()) + throw BadStorePath("path '%1%' is not a valid store path", path); + return StorePath(*path.begin()); + } + + static std::optional maybeGetStoreObjectPath(const CanonPath & path) + try { + return getStoreObjectPath(path); + } catch (BadStorePath &) { + /* FIXME: Stop using exceptions for control flow. */ + return std::nullopt; + } + + void requireStoreObject(const CanonPath & path) + { + requireStoreObject(getStoreObjectPath(path)); + } + std::optional maybeLstat(const CanonPath & path) override { /* Also allow `path` to point to the entire store, which is @@ -63,7 +83,13 @@ struct LocalStoreAccessor : SourceAccessor if (path.isRoot()) return Stat{.type = tDirectory}; - requireStoreObject(path); + /* Querying existence should not fail for things like + `/nix/store/foo.nix`. The store cannot contain such files (unless + some weird impurities sneak in, but that's UB from nix's PoV). */ + auto maybeStorePath = maybeGetStoreObjectPath(path); + if (!maybeStorePath) + return std::nullopt; + requireStoreObject(*maybeStorePath); return accessor->maybeLstat(path); } @@ -123,11 +149,6 @@ struct LocalStoreAccessor : SourceAccessor { return accessor->getLastModified(); } - - bool pathExists(const CanonPath & path) override - { - return accessor->pathExists(path); - } }; } // namespace diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 0129be46eb9b..acf6a4680de4 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -39,6 +39,8 @@ std::optional RemoteFSAccessor::maybeLstat(const CanonPath { if (path.isRoot()) return Stat{.type = tDirectory}; + /* FIXME: Correctly handle invalid names (return nullopt) and don't fail on + non-existent paths. */ auto res = fetch(path); return res.first->maybeLstat(res.second); } diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 0a325de16856..cc904a720d2f 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -84,6 +84,13 @@ public: virtual void readFile(const CanonPath & path, Sink & sink, fun sizeCallback = [](uint64_t size) {}) = 0; + /** + * @brief Check whether a file exists at @p path. + * + * @todo Consider making this non-virtual, since the evaluator uses + * maybeLstat as an indication that a file exists always (for positive + * caching purposes). + */ virtual bool pathExists(const CanonPath & path); enum Type { diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 143d0c2255e2..39e87348f140 100755 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -131,7 +131,7 @@ EOF git -C "$flakeFollowsA" add flake.nix expect 1 nix flake lock "$flakeFollowsA" 2>&1 | grep '/flakeB.*is forbidden in pure evaluation mode' -expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep "'flakeB' is too short to be a valid store path" +expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep '/flakeB.*does not exist' # Test relative non-flake inputs. cat > "$flakeFollowsA"/flake.nix <&1 | grepQuiet Hello diff --git a/tests/functional/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix index 022b22feae53..a5a3875da173 100644 --- a/tests/functional/lang/eval-okay-pathexists.nix +++ b/tests/functional/lang/eval-okay-pathexists.nix @@ -32,3 +32,6 @@ builtins.pathExists (./lib.nix) && builtins.pathExists ./symlink-resolution/foo/overlays/overlay.nix && builtins.pathExists ./symlink-resolution/broken && builtins.pathExists (builtins.toString ./symlink-resolution/foo/overlays + "/.") +&& builtins.pathExists "${builtins.storeDir}" +&& !builtins.pathExists "${builtins.storeDir}/foo" +&& !builtins.pathExists "${builtins.storeDir}/foo/bar" From 18ed6c101d0d652d44b952f646f6dce790fc5799 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Jul 2026 04:08:22 +0300 Subject: [PATCH 536/555] libstore: Fix libcurl thread wakeup with curl >= 8.21 Since https://github.com/curl/curl/commit/2a2104f3cff44bb28bb570a093be52bbeeed8f23 libcurl now swallows events in curl_multi_perform, so there's a chance that we miss a wakeup. This is somewhat reproducible on my machine by doing nix flake prefetch https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz. Another option is to bring back our own wakeup pipe, but that's less portable. Ideally this regression would be fixed in libcurl too... (cherry picked from commit 0d0c3335045ebc7383312265082e93928cd7f77a) --- src/libstore/filetransfer.cc | 63 ++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index d03af296b9b6..c58751e96bf3 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -989,6 +989,8 @@ struct curlFileTransfer : public FileTransfer private: bool quitting = false; public: + bool work = false; + void quit() { quitting = true; @@ -1036,12 +1038,14 @@ struct curlFileTransfer : public FileTransfer void stopWorkerThread() { /* Signal the worker thread to exit. */ - state_.lock()->quit(); - wakeupMulti(); + auto state(state_.lock()); + state->quit(); + wakeupMulti(*state); } - void wakeupMulti() + void wakeupMulti(State & state) { + state.work = true; if (auto ec = ::curl_multi_wakeup(curlm.get())) throw curlMultiError(ec); } @@ -1091,25 +1095,12 @@ struct curlFileTransfer : public FileTransfer } } - /* Wait for activity, including wakeup events. */ - long maxSleepTimeMs = items.empty() ? 10000 : 100; - auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() - ? std::max( - 0, - (int) std::chrono::duration_cast( - nextWakeup - std::chrono::steady_clock::now()) - .count()) - : maxSleepTimeMs; - - int numfds = 0; - mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds); - if (mc != CURLM_OK) - throw curlMultiError(mc); - nextWakeup = std::chrono::steady_clock::time_point(); std::vector> incoming; + std::vector> unpause; auto now = std::chrono::steady_clock::now(); + bool haveWork; { auto state(state_.lock()); @@ -1131,7 +1122,9 @@ struct curlFileTransfer : public FileTransfer break; } } + unpause = std::exchange(state->unpause, {}); quit = state->isQuitting(); + haveWork = std::exchange(state->work, false); } for (auto & item : incoming) { @@ -1142,14 +1135,10 @@ struct curlFileTransfer : public FileTransfer items[item->req] = item; } - /* NOTE: Unpausing may invoke callbacks to flush all buffers. */ - auto unpause = [&]() { - auto state(state_.lock()); - auto res = state->unpause; - state->unpause.clear(); - return res; - }(); + if (quit) + break; + /* NOTE: Unpausing may invoke callbacks to flush all buffers. */ for (auto & item : unpause) { /* The transfer might have completed (failed) between it getting enqueued for unpause and by the time the worker thread picked @@ -1159,6 +1148,26 @@ struct curlFileTransfer : public FileTransfer continue; static_cast(*ptr).unpause(); } + + /* Wait for activity, including wakeup events. */ + long maxSleepTimeMs = items.empty() ? 10000 : 100; + auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() + ? std::max( + 0, + (int) std::chrono::duration_cast( + nextWakeup - std::chrono::steady_clock::now()) + .count()) + : maxSleepTimeMs; + + /* Since https://github.com/curl/curl/commit/2a2104f3cff44bb28bb570a093be52bbeeed8f23 (8.21), + curl_multi_perform seems to swallow queued up events ¯\_(ツ)_/¯. */ + if (haveWork) + sleepTimeMs = 0; + + int numfds = 0; + mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds); + if (mc != CURLM_OK) + throw curlMultiError(mc); } debug("download thread shutting down"); @@ -1195,9 +1204,9 @@ struct curlFileTransfer : public FileTransfer throw nix::Error("cannot enqueue download request because the download thread is shutting down"); state->incoming.push(item); item->enqueued = true; /* Now any exceptions should be reported via the callback. */ + wakeupMulti(*state); } - wakeupMulti(); return ItemHandle(item.get_ptr()); } @@ -1217,7 +1226,7 @@ struct curlFileTransfer : public FileTransfer { auto state(state_.lock()); state->unpause.push_back(std::move(item)); - wakeupMulti(); + wakeupMulti(*state); } void unpauseTransfer(ItemHandle handle) override From 99b98e5b55f1d697dac0bebe9f2097e1d15c765a Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 23 Mar 2026 17:04:25 -0400 Subject: [PATCH 537/555] cli: error when remounting store without a private mount namespace Previously, when unshare(CLONE_NEWNS) failed and the store was read-only, Nix warned but still remounted the store writable on the host mount table. This silently affected other processes sharing the namespace. Now it throws an error, since proceeding would mutate shared state. (cherry picked from commit f218c7e8ee1b488921e78e0309eee1a22872366b) --- src/libstore/local-store.cc | 36 +--------- .../include/nix/util/linux-namespaces.hh | 22 +++--- src/libutil/linux/linux-namespaces.cc | 67 ++++++++++++++++++- src/nix/main.cc | 11 +-- 4 files changed, 83 insertions(+), 53 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index dd5d325239d2..5a1c161c0739 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -37,9 +37,7 @@ #endif #ifdef __linux__ -# include -# include -# include +# include "nix/util/linux-namespaces.hh" #endif #ifdef __CYGWIN__ @@ -649,37 +647,7 @@ void LocalStore::makeStoreWritable() #ifdef __linux__ if (!isRootUser()) return; - /* Check if /nix/store is on a read-only mount. */ - struct statvfs stat; - if (statvfs(config->realStoreDir.get().c_str(), &stat) != 0) - throw SysError("getting info about the Nix store mount point"); - - if (stat.f_flag & ST_RDONLY) { - /* In a user namespace, mount flags like `nodev` and `nosuid` are - locked and dropping them causes `EPERM`, so here we translate each - `statvfs` flag to the corresponding `mount` flag individually. */ - unsigned long flags = MS_REMOUNT | MS_BIND; - if (stat.f_flag & ST_NODEV) - flags |= MS_NODEV; - if (stat.f_flag & ST_NOSUID) - flags |= MS_NOSUID; - if (stat.f_flag & ST_NOEXEC) - flags |= MS_NOEXEC; - if (stat.f_flag & ST_NOATIME) - flags |= MS_NOATIME; - if (stat.f_flag & ST_NODIRATIME) - flags |= MS_NODIRATIME; - if (stat.f_flag & ST_RELATIME) - flags |= MS_RELATIME; - if (stat.f_flag & ST_SYNCHRONOUS) - flags |= MS_SYNCHRONOUS; -# ifdef ST_NOSYMFOLLOW - if (stat.f_flag & ST_NOSYMFOLLOW) - flags |= MS_NOSYMFOLLOW; -# endif - if (mount(0, config->realStoreDir.get().c_str(), "none", flags, 0) == -1) - throw SysError("remounting %s writable", PathFmt(config->realStoreDir.get())); - } + remountReadOnlyWritable(config->realStoreDir.get()); #endif } diff --git a/src/libutil/linux/include/nix/util/linux-namespaces.hh b/src/libutil/linux/include/nix/util/linux-namespaces.hh index 8f7ffa8df48d..0b6bef7bcabb 100644 --- a/src/libutil/linux/include/nix/util/linux-namespaces.hh +++ b/src/libutil/linux/include/nix/util/linux-namespaces.hh @@ -1,21 +1,27 @@ #pragma once ///@file -#include - -#include "nix/util/types.hh" +#include namespace nix { /** - * Save the current mount namespace. Ignored if called more than - * once. + * Save the parent mount namespace and enter a private one via + * `unshare(CLONE_NEWNS)`. + */ +void tryEnterPrivateMountNamespace(); + +/** + * Remount `path` writable if its mount is read-only, leaving + * already-writable mounts untouched. This throws if we aren't in a + * private mount namespace, since remounting would leak into the + * host mount table. */ -void saveMountNamespace(); +void remountReadOnlyWritable(const std::filesystem::path & path); /** - * Restore the mount namespace saved by saveMountNamespace(). Ignored - * if saveMountNamespace() was never called. + * Restore the parent mount namespace saved when we entered a private + * one. Ignored if `tryEnterPrivateMountNamespace()` never succeeded. */ void restoreMountNamespace(); diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index 26a7479050ad..f0416307d126 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -7,6 +7,7 @@ #include #include +#include namespace nix { @@ -90,8 +91,11 @@ bool mountAndPidNamespacesSupported() static AutoCloseFD fdSavedMountNamespace; static AutoCloseFD fdSavedRoot; +static bool havePrivateMountNs = false; -void saveMountNamespace() +/* Save the current mount namespace so restoreMountNamespace() can return + to it later. Ignored if called more than once. */ +static void saveMountNamespace() { static std::once_flag done; std::call_once(done, []() { @@ -103,12 +107,69 @@ void saveMountNamespace() }); } +void tryEnterPrivateMountNamespace() +{ + try { + saveMountNamespace(); + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + havePrivateMountNs = true; + } catch (Error & e) { + debug("failed to set up a private mount namespace: %s", e.message()); + } +} + +void remountReadOnlyWritable(const std::filesystem::path & path) +{ + struct statvfs stat; + if (statvfs(path.c_str(), &stat) != 0) + throw SysError("getting mount info for %s", PathFmt(path)); + + if (!(stat.f_flag & ST_RDONLY)) + return; + + if (!havePrivateMountNs) + throw Error( + "cannot remount %s writable: not in a private mount namespace, " + "so the remount would affect the host mount table. " + "This usually happens inside containers or user namespaces where unshare(CLONE_NEWNS) is not permitted", + PathFmt(path)); + + /* In a user namespace, mount flags like `nodev` and `nosuid` are + locked and dropping them causes `EPERM`, so here we translate each + `statvfs` flag to the corresponding `mount` flag individually. */ + unsigned long flags = MS_REMOUNT | MS_BIND; + if (stat.f_flag & ST_NODEV) + flags |= MS_NODEV; + if (stat.f_flag & ST_NOSUID) + flags |= MS_NOSUID; + if (stat.f_flag & ST_NOEXEC) + flags |= MS_NOEXEC; + if (stat.f_flag & ST_NOATIME) + flags |= MS_NOATIME; + if (stat.f_flag & ST_NODIRATIME) + flags |= MS_NODIRATIME; + if (stat.f_flag & ST_RELATIME) + flags |= MS_RELATIME; + if (stat.f_flag & ST_SYNCHRONOUS) + flags |= MS_SYNCHRONOUS; +#ifdef ST_NOSYMFOLLOW + if (stat.f_flag & ST_NOSYMFOLLOW) + flags |= MS_NOSYMFOLLOW; +#endif + if (mount(0, path.c_str(), "none", flags, 0) == -1) + throw SysError("remounting %s writable", PathFmt(path)); +} + void restoreMountNamespace() { + if (!havePrivateMountNs) + return; + try { auto savedCwd = std::filesystem::current_path(); - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + if (setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); if (fdSavedRoot) { @@ -120,6 +181,8 @@ void restoreMountNamespace() if (chdir(savedCwd.c_str()) == -1) throw SysError("restoring cwd"); + + havePrivateMountNs = false; } catch (Error & e) { debug(e.msg()); } diff --git a/src/nix/main.cc b/src/nix/main.cc index df54a14ccb66..963bc07ce274 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -398,15 +398,8 @@ void mainWrapped(int argc, char ** argv) flakeSettings.configureEvalSettings(evalSettings); #ifdef __linux__ - if (isRootUser()) { - try { - saveMountNamespace(); - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - } catch (Error & e) { - warn("failed to set up a private mount namespace: %s", e.msg()); - } - } + if (isRootUser()) + tryEnterPrivateMountNamespace(); #endif Finally f([] { logger->stop(); }); From 3ba59c1b138cba0262648ea20b287038ce2b28d8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 2 Jul 2026 23:20:25 +0300 Subject: [PATCH 538/555] libstore: Make FileTransfer throw Interrupted on interrupted transfers We are now issuing many more requests in queryMissing concurrently, without doing checkInterrupt() when enqueueing downloads (arguably we shouldn't). The exception ignoring behavior of querySubstitutablePathInfosAsync, where it swallows and prints exceptions derived from Error & doesn't help - we end up with a wall of text like: error: download of 'https://cache.nixos.org/b2s9lqzdi2pg80j0lawgw8ndian3k9mn.narinfo' was interrupted error: download of 'https://cache.nixos.org/5h2h4brfrgr89gvh1l93mh63mm0h5r6s.narinfo' was interrupted error: download of 'https://cache.nixos.org/azz5rixma8na4h9s75406w94yjsk2b4k.narinfo' was interrupted error: download of 'https://cache.nixos.org/xdlfnymy64jxfhnj19sjvdvmbbmyn468.narinfo' was interrupted error: download of 'https://cache.nixos.org/q8sra2q3n8gvalzbfq48jvv18xjsg2gb.narinfo' was interrupted error: download of 'https://cache.nixos.org/pzmm2bczb5gb1nvgv5xp6ims9iqcc4af.narinfo' was interrupted error: download of 'https://cache.nixos.org/s3slcivizcjkppyvf5drlbx0b8dxafg9.narinfo' was interrupted Rethrowing Interrupted instead of FileTransferError on proper interrupts seems correct to me. And since it doesn't inherit from Error (but rather BaseError) it escapes out of catch (Error &) blocks and properly unwinds the stack eagerly. Also add a new `Cancelled` exception type to reflect cases (not very useful for now) to indicate that a filetransfer was cancelled without a global interrupt. In the future we should add per-transfer cancellation mechanisms for this purpose. (cherry picked from commit 8c9426502817a493d23ce6e112965544fcb21aed) --- src/libstore/filetransfer.cc | 34 ++++++++++++------- .../include/nix/store/filetransfer.hh | 2 +- src/libutil/include/nix/util/signals.hh | 1 + src/libutil/unix/signals.cc | 2 ++ src/libutil/util.cc | 4 +++ src/libutil/windows/signals.cc | 2 ++ 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c58751e96bf3..6e9f2256eaa0 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -294,12 +294,7 @@ struct curlFileTransfer : public FileTransfer } try { if (!done && enqueued) - fail(FileTransferError( - Interrupted, - {}, - "%s of '%s' was interrupted", - Uncolored(request.noun()), - request.displayUri())); + failInterruptedOrCancelled(); } catch (...) { ignoreExceptionInDestructor(); } @@ -328,6 +323,18 @@ struct curlFileTransfer : public FileTransfer failEx(std::make_exception_ptr(std::forward(e))); } + void failInterruptedOrCancelled() + { + HintFmt fmt("%s of '%s' was interrupted", Uncolored(request.noun()), request.displayUri()); + + /* Technically, we don't really have per-transfer cancellation currently, + but it's nice to distinguish between the two in the future. */ + if (getInterrupted()) + fail(nix::Interrupted(std::move(fmt))); + else + fail(nix::Cancelled(std::move(fmt))); + } + LambdaSink finalSink; std::optional errorSink; @@ -872,13 +879,14 @@ struct curlFileTransfer : public FileTransfer std::optional response; if (errorSink) response = std::move(errorSink->s); - auto exc = code == CURLE_ABORTED_BY_CALLBACK && getInterrupted() ? FileTransferError( - Interrupted, - std::move(response), - "%s of '%s' was interrupted", - Uncolored(request.noun()), - request.displayUri()) - : httpStatus != 0 + + /* TODO: Also support per-transfer cancellations. */ + if (code == CURLE_ABORTED_BY_CALLBACK && getInterrupted()) { + failInterruptedOrCancelled(); + return; + } + + auto exc = httpStatus != 0 ? FileTransferError( err, std::move(response), diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 04980c9ac465..774a4e1403b3 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -470,7 +470,7 @@ public: void download(FileTransferRequest && request, Sink & sink, std::function resultCallback = {}); - enum Error { NotFound, Unauthorized, Forbidden, Misc, Transient, Interrupted }; + enum Error { NotFound, Unauthorized, Forbidden, Misc, Transient }; }; /** diff --git a/src/libutil/include/nix/util/signals.hh b/src/libutil/include/nix/util/signals.hh index 4f9d9516bbf0..5124ebaccafd 100644 --- a/src/libutil/include/nix/util/signals.hh +++ b/src/libutil/include/nix/util/signals.hh @@ -43,6 +43,7 @@ inline void checkInterrupt(); * @note Never will happen on Windows */ MakeError(Interrupted, BaseError); +MakeError(Cancelled, BaseError); struct InterruptCallback { diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index d1de7842aaaf..38030a77ddfd 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -11,6 +11,8 @@ namespace nix { void Interrupted::anchor() {} +void Cancelled::anchor() {} + std::atomic unix::_isInterrupted = false; thread_local std::function unix::interruptCheck; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 4f5f427734eb..e6b2fffa25fa 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -249,6 +249,10 @@ void ignoreExceptionExceptInterrupt(Verbosity lvl) throw; } catch (const Interrupted & e) { throw; + } catch (const Cancelled & e) { + /* Morally the same as Interrupted, just not triggered by a user but some other + cancellation. */ + throw; } catch (Error & e) { printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.info().msg); } catch (std::exception & e) { diff --git a/src/libutil/windows/signals.cc b/src/libutil/windows/signals.cc index ca4900313fc5..40fe5eeb63c4 100644 --- a/src/libutil/windows/signals.cc +++ b/src/libutil/windows/signals.cc @@ -4,4 +4,6 @@ namespace nix { void Interrupted::anchor() {} +void Cancelled::anchor() {} + } // namespace nix From 1b6f4eae679e423a2dad8133a055a4ed592c1a12 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 2 Jul 2026 23:20:36 +0300 Subject: [PATCH 539/555] libutil: Special-case forEachAsync for one awaitable, document current lack of cancellation We don't need to do the whole async_initiate dance for just one awaitable. (cherry picked from commit dc8fac0b68168a4c9eb9d491f7abe92205335703) --- src/libutil/include/nix/util/async.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libutil/include/nix/util/async.hh b/src/libutil/include/nix/util/async.hh index 3f6be9b48cc7..952cdb5610f8 100644 --- a/src/libutil/include/nix/util/async.hh +++ b/src/libutil/include/nix/util/async.hh @@ -95,10 +95,14 @@ asio::awaitable forEachAsync(Range && range, const F & f) auto pending = std::ranges::size(range); if (pending == 0) co_return; + else if (pending == 1) + co_return co_await f(*range.begin()); auto executor = co_await asio::this_coro::executor; std::exception_ptr err; + /* TODO: Handle cancellation on first error. Not very useful for now since we + do all-or-nothing cancellation typically. */ co_await asio::async_initiate( [&](auto handler) { auto h = std::make_shared(std::move(handler)); From 50a3b06cd5d28d9c6d2f47c74a21fd85e19e4c18 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 4 Jul 2026 18:39:19 +0300 Subject: [PATCH 540/555] release notes: 2.35.0 Co-authored-by: Lisanna Dettwyler (cherry picked from commit db08619e33d7f1cc8a0fe0004b084adbd9a3769d) --- doc/manual/rl-next/async-post-build-hook.md | 9 - .../aws-sts-webidentity-region-fallback.md | 11 - doc/manual/rl-next/beta-installer | 29 -- doc/manual/rl-next/build-trace-rework.md | 96 ---- doc/manual/rl-next/closure-gc.md | 8 - doc/manual/rl-next/delete-keep.md | 8 - .../rl-next/filetransfer-retry-backoff.md | 27 -- doc/manual/rl-next/fix-primop-eval-state.md | 10 - doc/manual/rl-next/flake-check-out-links.md | 7 - .../rl-next/flake-check-print-output-paths.md | 7 - doc/manual/rl-next/freebsd-sandboxing.md | 10 - doc/manual/rl-next/getflake-path.md | 6 - doc/manual/rl-next/git-url-scp.md | 30 -- .../github-fetcher-param-validation.md | 7 - .../rl-next/hash-self-reference-positions.md | 12 - doc/manual/rl-next/http3.md | 25 -- doc/manual/rl-next/mimalloc.md | 15 - .../s3-credential-chain-web-identity.md | 26 -- doc/manual/rl-next/seccomp-block-listxattr.md | 10 - .../rl-next/store-config-get-state-dir.md | 36 -- doc/manual/rl-next/zstd-multiframe.md | 18 - doc/manual/source/SUMMARY.md.in | 1 + doc/manual/source/release-notes/rl-2.35.md | 414 ++++++++++++++++++ 23 files changed, 415 insertions(+), 407 deletions(-) delete mode 100644 doc/manual/rl-next/async-post-build-hook.md delete mode 100644 doc/manual/rl-next/aws-sts-webidentity-region-fallback.md delete mode 100644 doc/manual/rl-next/beta-installer delete mode 100644 doc/manual/rl-next/build-trace-rework.md delete mode 100644 doc/manual/rl-next/closure-gc.md delete mode 100644 doc/manual/rl-next/delete-keep.md delete mode 100644 doc/manual/rl-next/filetransfer-retry-backoff.md delete mode 100644 doc/manual/rl-next/fix-primop-eval-state.md delete mode 100644 doc/manual/rl-next/flake-check-out-links.md delete mode 100644 doc/manual/rl-next/flake-check-print-output-paths.md delete mode 100644 doc/manual/rl-next/freebsd-sandboxing.md delete mode 100644 doc/manual/rl-next/getflake-path.md delete mode 100644 doc/manual/rl-next/git-url-scp.md delete mode 100644 doc/manual/rl-next/github-fetcher-param-validation.md delete mode 100644 doc/manual/rl-next/hash-self-reference-positions.md delete mode 100644 doc/manual/rl-next/http3.md delete mode 100644 doc/manual/rl-next/mimalloc.md delete mode 100644 doc/manual/rl-next/s3-credential-chain-web-identity.md delete mode 100644 doc/manual/rl-next/seccomp-block-listxattr.md delete mode 100644 doc/manual/rl-next/store-config-get-state-dir.md delete mode 100644 doc/manual/rl-next/zstd-multiframe.md create mode 100644 doc/manual/source/release-notes/rl-2.35.md diff --git a/doc/manual/rl-next/async-post-build-hook.md b/doc/manual/rl-next/async-post-build-hook.md deleted file mode 100644 index ea061f14ae9d..000000000000 --- a/doc/manual/rl-next/async-post-build-hook.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: Make post-build-hook asynchronous -prs: [15451] -issues: [15406] ---- - -This change makes the `post-build-hook` run asynchronously but still as part of the goal. -This retains the current behavior that a waiting goal will not start until the `post-build-hook` of the goal it is waiting on completes. -However, multiple `post-build-hook`s can now run concurrently just as multiple goals can run concurrently. diff --git a/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md b/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md deleted file mode 100644 index 5034c65438cb..000000000000 --- a/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -synopsis: S3 substituters fall back to the URL's region for STS WebIdentity auth -prs: [15594] ---- - -When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, -GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL -as a fallback for the STS endpoint region if neither `AWS_REGION` nor -`AWS_DEFAULT_REGION` is set. Previously, IRSA setups that exported -`AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail -with a misleading "IMDS provider" error. diff --git a/doc/manual/rl-next/beta-installer b/doc/manual/rl-next/beta-installer deleted file mode 100644 index b02564d95d2d..000000000000 --- a/doc/manual/rl-next/beta-installer +++ /dev/null @@ -1,29 +0,0 @@ ---- -synopsis: "Rust nix-installer in beta" -prs: [] ---- - -The Rust-based rewrite of the Nix installer is now in beta. -We'd love help testing it out! - -To test out the new installer, run: -``` -curl -sSfL https://artifacts.nixos.org/nix-installer | sh -s -- install -``` - -This installer can be run even when you have an existing, script-based Nix installation without any adjustments. - -This new installer also comes with the ability to uninstall your Nix installation; run: -``` -/nix/nix-installer uninstall -``` - -This will get rid of your entire Nix installation (even if you installed over an existing, script-based installation). - -This installer is a modified version of the [Determinate Nix Installer](https://github.com/DeterminateSystems/nix-installer) by Determinate Systems. -Thanks to Determinate Systems for all the investment they've put into the installer. - -Source for the installer is in https://github.com/NixOS/nix-installer. -Report any issues in that repo. - -For CI usage, a GitHub Action to install Nix using this installer is available at https://github.com/NixOS/nix-installer-action. diff --git a/doc/manual/rl-next/build-trace-rework.md b/doc/manual/rl-next/build-trace-rework.md deleted file mode 100644 index 61b6bc51b1c0..000000000000 --- a/doc/manual/rl-next/build-trace-rework.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -synopsis: "Content-addressed derivations: realisations keyed by store path instead of hash modulo" -issues: [11897] -prs: [12464] ---- - -The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. This affects the **binary cache protocol** and the **wire protocols**. - -### What changed - -#### Build trace format - -Previously, a build trace entry (realisation) was keyed by the **hash modulo** of the derivation. -A SHA-256 hash computed via the complex "derivation hash modulo" algorithm. -This required implementations to understand ATerm serialisation and the full derivation hashing scheme just to look up or store build results. - -Now, build trace entries are keyed by the **regular derivation store path** plus the output name. For example, instead of: - -``` -sha256:ba7816bf8f01...!out -``` - -The key is now: - -``` -/nix/store/abc...-foo.drv^out -``` - -This is simpler, more intuitive, and means that third-party tools implementing CA derivation support (e.g., Hydra) -no longer need to implement the derivation hash modulo algorithm. - -#### Build trace usage - -Previously the build trace contained entries for both unresolved and [resolved](@docroot@/store/resolution.md) derivations. -Now, it only contains entries for resolved derivations. -For now, unresolved derivations will be resolved from these underlying build trace entries. -This is slower, but avoids a bunch of correctness issues. - -### Binary cache protocol - -- The directory for build traces moved from `realisations/` to `build-trace-v2/`. -- File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. -- The JSON format of build trace entries is now split into `key` and `value` objects: - ```json - { - "key": { - "drvPath": "abc...-foo.drv", - "outputName": "out" - }, - "value": { - "outPath": "xyz...-foo", - "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] - } - } - ``` - Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. -- The deprecated `dependentRealisations` field has been removed. - -Existing binary caches will need to be re-populated with the new format for CA derivation build traces. -Old build traces at the previous URLs are simply abandoned. -Non-CA builds are unaffected. - -### Wire protocols - -- **Worker protocol**: - A new feature flag `realisation-with-path-not-hash` is negotiated during the handshake. - Clients and daemons that both support this feature use the new binary serialisation for `DrvOutput`, `UnkeyedRealisation`, and related types. - Fallback to older protocol versions gracefully degrades (realisations are unavailable). -- **Serve protocol**: - Bumped from 2.7 to 2.8 with native serialisers for the new types. - Fallback to older protocol versions gracefully degrades in the same way. - -Stable code paths do use the realization fields (`BuildResult::Success::builtOutputs`), but only the output name and outpath parts of that. -For older protocols, we can fake enough of the realisation format to provide those two parts forthat map, which keeps operations like `--print-output-paths` working. - -### Local Store SQLite schema - -The build trace entries no longer have any foreign key store objects in the store. -This is because we will need to remember the build trace entries for resolved derivations we may have deleted, otherwise we will effectively forget outputs resolved derivations we do have on disk. -GC for build trace will be implemented later --- there is no single correct choice (there is no closure property) so it will be a question of what policies users want. - -### Structured signatures - -[Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. -`nix path-info --json --json-format 3` opts into the new version for this command. -JSON parsing accepts both the old string format and new structured format for backwards compatibility. - -### Impact - -- **Non-CA derivation users**: No impact. This only affects the experimental `ca-derivations` feature. -- **Binary cache operators**: - Binary caches serving CA derivation build traces will need to be repopulated. - Existing NARs and narinfo files are unaffected. -- **Tool authors**: - Implementations interfacing with the CA derivations protocol are simplified. - The derivation hash modulo algorithm is no longer required to form build trace keys. diff --git a/doc/manual/rl-next/closure-gc.md b/doc/manual/rl-next/closure-gc.md deleted file mode 100644 index 945a78dc9f8c..000000000000 --- a/doc/manual/rl-next/closure-gc.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: "Added `--skip-alive` option to `nix store delete` for collecting garbage within a closure" -issues: 7239 -prs: [15236, 15727] ---- - -`nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. -The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. diff --git a/doc/manual/rl-next/delete-keep.md b/doc/manual/rl-next/delete-keep.md deleted file mode 100644 index 0332e0e3933e..000000000000 --- a/doc/manual/rl-next/delete-keep.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: "Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands" -prs: [15776] ---- - -Setting `keep-derivations = true` and trying to delete a derivation with realised outputs would previously fail. -Same with `keep-outputs = true` and trying to delete an output that still has derivers. -These options no longer affect the deletion commands, and are now documented as such. diff --git a/doc/manual/rl-next/filetransfer-retry-backoff.md b/doc/manual/rl-next/filetransfer-retry-backoff.md deleted file mode 100644 index 0e26ccd54a8c..000000000000 --- a/doc/manual/rl-next/filetransfer-retry-backoff.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -synopsis: Configurable file-transfer retry backoff with full jitter and Retry-After support -issues: [15419, 15023] -prs: [15449] ---- - -File transfer retries (downloads and uploads) now use AWS-style "full jitter" -exponential backoff, treat HTTP 503 as rate-limited (same longer delay as 429), -and honor the `Retry-After` response header. Retry timing is configurable via -new `nix.conf` settings: - -- `filetransfer-retry-delay` (default 100ms): base delay for transient errors -- `filetransfer-retry-delay-rate-limited` (default 5000ms): base delay for 429/503 -- `filetransfer-retry-max-delay` (default 60000ms): per-attempt delay ceiling -- `filetransfer-retry-jitter` (default true): enable full jitter - -The existing `download-attempts` setting has been renamed to -`filetransfer-retry-attempts` to reflect that it applies to uploads as well as -downloads. The old name remains as an alias for backwards compatibility. - -Per-substituter overrides are available as store URL parameters -(`retry-delay`, `retry-delay-rate-limited`, `retry-max-delay`, -`retry-attempts`), e.g. `s3://my-cache?retry-attempts=8`. - -This addresses thundering-herd scenarios where many CI jobs hit the same -S3 prefix and receive 503 SlowDown; previously the retry window for 503 -was only ~4 seconds. diff --git a/doc/manual/rl-next/fix-primop-eval-state.md b/doc/manual/rl-next/fix-primop-eval-state.md deleted file mode 100644 index 1a3bc287538f..000000000000 --- a/doc/manual/rl-next/fix-primop-eval-state.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: "C API: Fix `EvalState` pointer passed to primop callbacks" -prs: [15300, 15383] ---- - -The `EvalState *` passed to C API primop callbacks was incorrectly pointing to -the internal `nix::EvalState` rather than the C API wrapper struct. This caused -a segfault when the callback used the pointer with C API functions such as -`nix_alloc_value()`. The same issue affected `printValueAsJSON` and -`printValueAsXML` callbacks on external values. diff --git a/doc/manual/rl-next/flake-check-out-links.md b/doc/manual/rl-next/flake-check-out-links.md deleted file mode 100644 index eb369a629c3f..000000000000 --- a/doc/manual/rl-next/flake-check-out-links.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: nix flake check now supports --out-link -prs: [15476] -issues: [13470] ---- - -`nix flake check` now supports the flag `--out-link`, defaulting to not creating out links if the flag is not specified. diff --git a/doc/manual/rl-next/flake-check-print-output-paths.md b/doc/manual/rl-next/flake-check-print-output-paths.md deleted file mode 100644 index 5e0fd35b6737..000000000000 --- a/doc/manual/rl-next/flake-check-print-output-paths.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: nix flake check now supports --print-out-paths -prs: [15476] -issues: [13470] ---- - -`nix flake check` now supports the flag `--print-out-paths`. diff --git a/doc/manual/rl-next/freebsd-sandboxing.md b/doc/manual/rl-next/freebsd-sandboxing.md deleted file mode 100644 index 2c2d214494d9..000000000000 --- a/doc/manual/rl-next/freebsd-sandboxing.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: Enable FreeBSD sandboxing, add `x86_64-freebsd` to installer -prs: [15673, 13281, 9968] ---- - -A FreeBSD build has been added to the traditional installer script, with sandboxing enabled. -The beta installer is not yet supported. - -FreeBSD support is not as well-tested as Linux or macOS, but is fully capable of building packages -and performing other tasks expected of Nix on Linux. diff --git a/doc/manual/rl-next/getflake-path.md b/doc/manual/rl-next/getflake-path.md deleted file mode 100644 index 2360fe7693e0..000000000000 --- a/doc/manual/rl-next/getflake-path.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -synopsis: "`builtins.getFlake` now supports path values" -prs: [15290] ---- - -`builtins.getFlake` now accepts path values in addition to flakerefs, allowing you to write `builtins.getFlake ./subflake` instead of having to use ugly workarounds to construct a pure flakeref. diff --git a/doc/manual/rl-next/git-url-scp.md b/doc/manual/rl-next/git-url-scp.md deleted file mode 100644 index b1125f7b32a7..000000000000 --- a/doc/manual/rl-next/git-url-scp.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -synopsis: Support SCP-like URLs in fetchGit and type = "git" flake inputs -prs: [14863] -issues: [14852, 14867] ---- - -Nix now (once again) recognizes [SCP-like syntax for Git URLs](https://git-scm.com/docs/git-clone#_git_urls). This partially -restores compatibility with Nix 2.3 for `fetchGit`. The following syntax is once again supported: - -```nix -builtins.fetchGit "host:/absolute/path/to/repo" -``` - -Nix also passes through the tilde (for home directories) verbatim: - -```nix -builtins.fetchGit "host:~/relative/to/home" -``` - -IPv6 addresses also supported when bracketed: - -```nix -builtins.fetchGit "user@[::1]:~/relative/to/home" -``` - -`builtins.fetchTree` also supports this syntax now: - -```nix -builtins.fetchTree { type = "git"; url = "host:/path/to/repo"; } -``` diff --git a/doc/manual/rl-next/github-fetcher-param-validation.md b/doc/manual/rl-next/github-fetcher-param-validation.md deleted file mode 100644 index 2fb430ae4b24..000000000000 --- a/doc/manual/rl-next/github-fetcher-param-validation.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: GitHub fetcher now validates URL parameters -prs: [15331] -issues: [15304] ---- - -The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. diff --git a/doc/manual/rl-next/hash-self-reference-positions.md b/doc/manual/rl-next/hash-self-reference-positions.md deleted file mode 100644 index 0a7193b3a497..000000000000 --- a/doc/manual/rl-next/hash-self-reference-positions.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -synopsis: "Fix hash collision between store paths with self-references and their zeroed-out equivalents" -issues: [15837] -prs: [15931] ---- - -When computing the hash of a NAR with self-references, Nix zeroes out the self-references but also hashes their positions. -The latter was accidentally lost in Nix 2.17.0, which meant a NAR with self-references could hash to the same store path as an otherwise-identical NAR in which some of the self-references had been zeroed out. - -This release restores hashing the positions of self-references. -As a consequence, content-addressed store paths derived from self-referential NARs will differ from those produced by Nix 2.17 through 2.34. -This affects users of the experimental `ca-derivations` features, as well as users of `nix store make-content-addressed`. diff --git a/doc/manual/rl-next/http3.md b/doc/manual/rl-next/http3.md deleted file mode 100644 index 1f0a31227e6f..000000000000 --- a/doc/manual/rl-next/http3.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -synopsis: HTTP/3 (QUIC) support ---- - -Nix can now fetch from binary caches and other HTTP(S) sources over HTTP/3 -(QUIC), controlled by a new -[`http3`](@docroot@/command-ref/conf-file.md#conf-http3) setting (disabled by -default). When enabled, Nix requests HTTP/3 and transparently falls back to -HTTP/2 or HTTP/1.1 for servers that do not advertise QUIC. The setting only -takes effect when linked against a libcurl built with HTTP/3 support, otherwise -it is ignored and Nix keeps using HTTP/2 without warning or error. - -Enable it with: - -``` -nix.conf: http3 = true -CLI: --http3 -``` - -Or disable with: - -``` -nix.conf: http3 = false -CLI: --no-http3 -``` diff --git a/doc/manual/rl-next/mimalloc.md b/doc/manual/rl-next/mimalloc.md deleted file mode 100644 index dfcde0cab03a..000000000000 --- a/doc/manual/rl-next/mimalloc.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -synopsis: "Link mimalloc for faster evaluation" -prs: [15596] ---- - -The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) -by default on non-Windows platforms, replacing glibc's malloc for all -non-GC allocations. - -This yields a **5–12% wall-clock improvement** on evaluation workloads, -ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS -configurations. - -The allocator can be disabled at build time with `-Dmimalloc=disabled` -or by passing `withMimalloc = false` to the Nix package. diff --git a/doc/manual/rl-next/s3-credential-chain-web-identity.md b/doc/manual/rl-next/s3-credential-chain-web-identity.md deleted file mode 100644 index 4dfece0e3b7d..000000000000 --- a/doc/manual/rl-next/s3-credential-chain-web-identity.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -synopsis: "S3: restore STS WebIdentity and ECS container credential providers" -prs: [15507] ---- - -Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a -custom chain built on `aws-c-auth`. That chain omitted two providers, -breaking S3 binary cache access in container workloads: - -- **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, - `AWS_ROLE_SESSION_NAME`) — used by EKS IRSA, GitHub Actions OIDC, and - any `sts:AssumeRoleWithWebIdentity` federation. -- **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, - `AWS_CONTAINER_CREDENTIALS_FULL_URI`) — used by ECS tasks and EKS Pod - Identity. - -The typical symptom was a misleading IMDS error -(`Valid credentials could not be sourced by the IMDS provider`), because -IMDS is the last provider tried after the correct one was skipped. - -Both providers are now part of the chain, ordered to match the -pre-2.33 `DefaultAWSCredentialsProviderChain`: -`Environment → SSO → Profile → STS WebIdentity → (ECS | IMDS)`. -As in both the old and new AWS SDK default chains, ECS and IMDS are -mutually exclusive: when container credential environment variables are -set, IMDS is skipped. diff --git a/doc/manual/rl-next/seccomp-block-listxattr.md b/doc/manual/rl-next/seccomp-block-listxattr.md deleted file mode 100644 index 2455ed4f825f..000000000000 --- a/doc/manual/rl-next/seccomp-block-listxattr.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: "Linux sandbox: also block `listxattr` syscalls" -prs: [15743] ---- - -The Linux sandbox now also returns `ENOTSUP` for `listxattr`, -`llistxattr` and `flistxattr`, matching the existing treatment of -`getxattr`/`setxattr`/`removexattr`. This prevents host xattrs (e.g. -`security.selinux`) from leaking into builds and fixes tools such as -`mkfs.ubifs` that probe xattr support via `listxattr`. diff --git a/doc/manual/rl-next/store-config-get-state-dir.md b/doc/manual/rl-next/store-config-get-state-dir.md deleted file mode 100644 index a789bdf83511..000000000000 --- a/doc/manual/rl-next/store-config-get-state-dir.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -synopsis: "Improve daemon socket path logic for chroot stores" -prs: [15429] ---- - -The default daemon socket path now uses the per-store [`state`](@docroot@/store/types/local-store.md#store-local-store-state) directory whenever one is defined, rather than always using the global [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR). -This means [local chroot stores](@docroot@/store/types/local-store.md#chroot) each get their own socket path automatically. - -Example: - -```bash -nix-daemon --store /foo/bar -``` - -will now use a socket at: -``` -/foo/bar/nix/var/nix/daemon-socket/socket -``` -instead of -``` -$NIX_STATE_DIR/daemon-socket/socket -``` - -Users who wish to serve or connect to a chroot store at the old location will have to force the socket location: - -- When serving (running a daemon), use the new [`--socket-path`](@docroot@/command-ref/new-cli/nix3-daemon.md#opt-socket-path) flag: - - ```bash - nix daemon --socket-path "$NIX_STATE_DIR/daemon-socket/socket" - ``` - -- When connecting as a client put the path in the [store URL](@docroot@/store/types/local-daemon-store.md): - - ``` - unix://$NIX_STATE_DIR/daemon-socket/socket - ``` diff --git a/doc/manual/rl-next/zstd-multiframe.md b/doc/manual/rl-next/zstd-multiframe.md deleted file mode 100644 index 6f5d9875df74..000000000000 --- a/doc/manual/rl-next/zstd-multiframe.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -synopsis: zstd compression now emits multi-frame output and uses less memory -prs: [15550] ---- - -zstd-compressed NARs are now written as a sequence of independent 16 MiB -frames instead of a single large frame. This lays the groundwork for -parallel decompression in a future release without requiring caches to be -repopulated, and significantly lowers peak memory use during compression -(e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). - -The output remains standard zstd and is decoded unchanged by existing Nix -binaries and the `zstd` CLI; compression ratio is effectively unchanged. - -Per-frame compression now uses up to 4 worker threads. For zstd this is the -new default: the `parallel-compression` store setting defaults to `true` when -`compression=zstd` (it remains `false` for `xz`). Set -`?parallel-compression=false` to opt out. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index a12e84becc35..ae77759f5516 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -159,6 +159,7 @@ - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.35 (2026-06-22)](release-notes/rl-2.35.md) - [Release 2.34 (2026-02-27)](release-notes/rl-2.34.md) - [Release 2.33 (2025-12-09)](release-notes/rl-2.33.md) - [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md) diff --git a/doc/manual/source/release-notes/rl-2.35.md b/doc/manual/source/release-notes/rl-2.35.md new file mode 100644 index 000000000000..a712b6385627 --- /dev/null +++ b/doc/manual/source/release-notes/rl-2.35.md @@ -0,0 +1,414 @@ +# Release 2.35.0 (2026-06-22) + +## Highlights + +- Sources are copied to the store more lazily [#3121](https://github.com/NixOS/nix/issues/3121) [#15711](https://github.com/NixOS/nix/pull/15711) [#15920](https://github.com/NixOS/nix/pull/15920) + + Historically, flakes source trees have been eagerly fetched to and evaluated from the Nix store to ensure deterministic and hermetic evaluation, even if the resulting store object is not used as a derivation input. This made the implementation simpler, yet made flakes unusable in large repositories and performed unnecessary writes to the store on each change to the source tree. + + Since Nix 2.32, all I/O (excluding `path:` and `hg+:`-style inputs) for reading sources during evaluation has been funneled to their original filesystem location (or to the `~/.cache/nix/tarball-cache-v2` bare git repository for tarball-based inputs). However, the source tree was still fetched to the store -- primarily for computing the resulting content-addressed store path. In most cases, (such as importing the `nixpkgs` package set) this is not necessary. + + Touching (and hashing the NAR serialisation of) the whole source tree is unavoidable, since: + + - In case of flake inputs, `narHash` integrity must be checked eagerly. + - The `outPath` attribute of a flake must be known in advance, and for backwards compatibility must be a content-addressed store path string with [constant string context](@docroot@/language/string-context.md#string-context-constant) representing the flake source tree. + + Even within the constraints imposed by backwards compatibility requirements, there are several improvements that are achievable. To reduce the number of copies performed, Nix now hashes the input without copying first, assuming that the `.outPath` will not end up in a derivation attribute and thus would never have to be actually fetched to the store. This comes at the slight cost of doing more work in case the assumption is wrong, but results in less work in typical use cases. The evaluator continues to behave as if the copy was performed: + + - Flakes are still evaluated from the store, from the evaluator's point of view. + - `toString ./.` continues to produce a content-addressed store path string without context. + - Path resolution crossing trees located in the filesystem and in Nix's view of it (with "virtual" overlays on top) continues to work. For example, the flake source tree can contain a relative symlink pointing outside its corresponding store object (though such usage is discouraged and makes further improvements to laziness intractable). + - Reading files from the flake's `outPath` continues to work. For example, such code is well-formed and is not considered [IFD](@docroot@/language/import-from-derivation.md): + + ```nix + builtins.readFile ( /. + (builtins.unsafeDiscardStringContext self.outPath) + "/flake.nix" ) + ``` + + Similar treatment has been applied to `builtins.fetchTarball`, which no longer eagerly copies paths to the store. + `builtins.storePath` now also short-circuits on "lazy-ish" store paths and doesn't substitute unless necessary. + + This change is expected to significantly reduce disk usage required for typical evaluations and results in ~2x speedup for fetching and unpacking a nixpkgs tarball (either via `fetchTree`/flakes or via `fetchTarball`). + +- Support FreeBSD `libjail` based sandboxing, add `x86_64-freebsd` to installer [#9968](https://github.com/NixOS/nix/pull/9968) [#13281](https://github.com/NixOS/nix/pull/13281) [#15673](https://github.com/NixOS/nix/pull/15673) + + The FreeBSD build of Nix now supports build sandboxing via FreeBSD jails and is enabled by default. + A FreeBSD build has been added to the traditional installer script. The beta rust-based installer is not yet supported. + FreeBSD support is not as well-tested as Linux or macOS, but is fully capable of building packages and performing other tasks expected of Nix on Linux. + +## Improvements + +- HTTP/3 (QUIC) support [#15961](https://github.com/NixOS/nix/pull/15961) + + Nix can now fetch from binary caches and other HTTP(S) sources over HTTP/3 (QUIC), controlled by a new [`http3`](@docroot@/command-ref/conf-file.md#conf-http3) setting (disabled by default). + When enabled, Nix requests HTTP/3 and transparently falls back to HTTP/2 or HTTP/1.1 for servers that do not advertise QUIC. + The setting only takes effect when linked against a `libcurl` built with HTTP/3 support, otherwise it is ignored and Nix keeps using HTTP/2 without warning or error. + + Enable it with: + + ``` + nix.conf: http3 = true + CLI: --http3 + ``` + + Or disable with: + + ``` + nix.conf: http3 = false + CLI: --no-http3 + ``` + +- Link mimalloc for faster evaluation [#15596](https://github.com/NixOS/nix/pull/15596) + + The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) by default, replacing glibc's malloc for all non-GC allocations. + This yields a **5–12% wall-clock improvement** on evaluation workloads, ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS configurations. + The allocator can be disabled at build time with `-Dmimalloc=disabled`. + +- The `revCount` attribute of the Git fetchers is now lazily computed and passed-through as-is when explicitly specified [#15772](https://github.com/NixOS/nix/pull/15772) [#14596](https://github.com/NixOS/nix/pull/14596) + + `revCount` and `lastModified` attributes passed to the Git fetcher are no longer eagerly validated when explicitly specified. + + When not explicitly specified, `revCount` is now also a thunk value and not computed eagerly. This delays this (potentially) expensive computation until the value is actually required. + +- Configurable file-transfer retry backoff with full jitter and `Retry-After` support [#15023](https://github.com/NixOS/nix/issues/15023) [#15419](https://github.com/NixOS/nix/issues/15419) [#15449](https://github.com/NixOS/nix/pull/15449) + + File transfer retries (downloads and uploads) now use AWS-style "full jitter" exponential backoff, treat HTTP 503 as rate-limited (same longer delay as 429), + and honor the `Retry-After` response header. + + Retry timing is configurable via new `nix.conf` settings: + + - [`filetransfer-retry-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay): base delay for transient errors + - [`filetransfer-retry-delay-rate-limited`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay-rate-limited): base delay for 429/503 + - [`filetransfer-retry-max-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-max-delay): per-attempt delay ceiling + - [`filetransfer-retry-jitter`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-jitter): enable full jitter + + The existing `download-attempts` setting has been renamed to [`filetransfer-retry-attempts`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-attempts) to reflect that it applies to uploads as well as downloads. + The old name remains as an alias for backwards compatibility. + + Per-substituter overrides are available as store reference parameters ([`retry-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay), [`retry-delay-rate-limited`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay-rate-limited), [`retry-max-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-max-delay), [`retry-attempts`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-attempts)), e.g. `s3://my-cache?retry-attempts=8`. + +- Improve daemon socket path logic for chroot stores [#15190](https://github.com/NixOS/nix/pull/15190) + + The default daemon socket path now uses the per-store [`state`](@docroot@/store/types/local-store.md#store-local-store-state) directory whenever one is defined, rather than always using the global [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR). + This means [local chroot stores](@docroot@/store/types/local-store.md#chroot) each get their own socket path automatically. + + Example: + + ```bash + nix-daemon --store /foo/bar + ``` + + will now use a socket at: + ``` + /foo/bar/nix/var/nix/daemon-socket/socket + ``` + instead of + ``` + $NIX_STATE_DIR/daemon-socket/socket + ``` + + Users who wish to serve or connect to a chroot store at the old location will have to force the socket location: + + - When serving (running a daemon), use the new [`--socket-path`](@docroot@/command-ref/new-cli/nix3-daemon.md#opt-socket-path) flag: + + ```bash + nix daemon --socket-path "$NIX_STATE_DIR/daemon-socket/socket" + ``` + + - When connecting as a client, put the path in the [store URL](@docroot@/store/types/local-daemon-store.md): + + ``` + unix://$NIX_STATE_DIR/daemon-socket/socket + ``` + +- Linux sandbox: also block `listxattr` syscalls [#15743](https://github.com/NixOS/nix/pull/15743) + + The Linux sandbox now also returns `ENOTSUP` for `listxattr`, `llistxattr` and `flistxattr`, matching the existing treatment of `getxattr`/`setxattr`/`removexattr`. + This prevents host xattrs (e.g. `security.selinux`) from leaking into builds and fixes tools such as `mkfs.ubifs` that probe xattr support via `listxattr`. + +- Support SCP-like URLs in fetchGit and type = "git" flake inputs [#14852](https://github.com/NixOS/nix/issues/14852) [#14867](https://github.com/NixOS/nix/issues/14867) [#14863](https://github.com/NixOS/nix/pull/14863) + + Nix now (once again) recognizes [SCP-like syntax for Git URLs](https://git-scm.com/docs/git-clone#_git_urls). This partially + restores compatibility with Nix 2.3 for `fetchGit`. The following syntax is once again supported: + + ```nix + builtins.fetchGit "host:/absolute/path/to/repo" + ``` + + Nix also passes through the tilde (for home directories) verbatim: + + ```nix + builtins.fetchGit "host:~/relative/to/home" + ``` + + IPv6 addresses also supported when bracketed: + + ```nix + builtins.fetchGit "user@[::1]:~/relative/to/home" + ``` + + `builtins.fetchTree` also supports this syntax now: + + ```nix + builtins.fetchTree { type = "git"; url = "host:/path/to/repo"; } + ``` + +- `nix flake check` now supports `--print-out-paths` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) and `--out-link` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) defaulting to not creating out links if the flag is not specified. + +- Added `--skip-alive` (and `--skip-live` alias for compatibility with Lix users) option to `nix store delete` for collecting garbage within a closure [#7239](https://github.com/NixOS/nix/issues/7239) [#15236](https://github.com/NixOS/nix/pull/15236) [#15727](https://github.com/NixOS/nix/pull/15727) + + `nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. + The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. + +- `builtins.getFlake` now supports path values [#15290](https://github.com/NixOS/nix/pull/15290) + + `builtins.getFlake` now accepts path values in addition to flakerefs. This improves the usability of relative flakes, allowing you to write `builtins.getFlake ./subflake`. + This change does not allow specifying paths that are not already in the store (though they do not have valid store objects, i.e. this will not force a copy if the flake has only been hashed -- and not copied to the store). This may change in a future release. + +- `nix-profile.fish` and `nix-profile-daemon.fish` now use `$NIX_LINK` for computing the value of `NIX_PROFILE` instead of `$HOME/.nix-profile` [#14293](https://github.com/NixOS/nix/pull/14293) + +- `nix` binary now exports symbols from C bindings [#15696](https://github.com/NixOS/nix/pull/15696) + + This allows Nix plugins written against the C API to look up symbols dynamically without linking to corresponding `libnix*c.so` libraries. + +- The computed Git LFS endpoint URLs have been fixed to follow the spec [#15891](https://github.com/NixOS/nix/pull/15891) and memory usage of LFS fetches has been decreased [#15912](https://github.com/NixOS/nix/pull/15912) + +- We now verify that fetched Git LFS objects have the same OID as requested [#15845](https://github.com/NixOS/nix/pull/15845) + +- Primop documentation now includes time complexity information [#14554](https://github.com/NixOS/nix/pull/14554) + +- Improved documentation on store paths and derivation building [#14699](https://github.com/NixOS/nix/pull/14699) + +- The [build hook](@docroot@/command-ref/conf-file.md#conf-build-hook) is now killed with `SIGTERM` instead of `SIGKILL` [#15105](https://github.com/NixOS/nix/pull/15105) + +- Download/upload logs strip `userinfo` URL components [#15715](https://github.com/NixOS/nix/pull/15715) + +## Content-addressed derivations changes + + The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. + This changes the binary cache endpoints for realisations and the daemon/nix-serve protocol (gated behind a daemon protocol feature flag). + +- Realisations keyed by store path instead of hash modulo [#11897](https://github.com/NixOS/nix/issues/11897) [#12464](https://github.com/NixOS/nix/pull/12464) + + Previously, a build trace entry (realisation) was keyed by the hash modulo of the derivation. In simpler terms, derivations transitively depending on distinct fixed-output derivations with the same `outPath` would share a realisation. + + Now, build trace entries are keyed by the regular derivation store path (`.drvPath`) plus the output name. For example, instead of: + + ``` + sha256:ba7816bf8f01...!out + ``` + + The key is now: + + ``` + /nix/store/abc...-foo.drv^out + ``` + +- Removed support for "deep" realisations [#15289](https://github.com/NixOS/nix/pull/15289) + + Previously the build trace (set of "realisations") contained entries for both unresolved and [resolved](@docroot@/store/resolution.md) derivations. + Now, it contains entries exclusively for resolved derivations. + For now, unresolved derivations will be resolved from these underlying build trace entries. + This is slower, but has the benefit of making build trace entries stateless and self-describing --- making sharing realisations easier between stores. + + This change necessitates changes to the binary cache format: + + - The directory for build traces moved from `realisations/` to `build-trace-v2/`. + - File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. + - The JSON format of build trace entries is now split into `key` and `value` objects: + ```json + { + "key": { + "drvPath": "abc...-foo.drv", + "outputName": "out" + }, + "value": { + "outPath": "xyz...-foo", + "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] + } + } + ``` + Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. + - The deprecated `dependentRealisations` field has been removed. + + The build trace entries stored in the local SQLite database no longer have any foreign key references to store objects. + This is because the build trace entries for resolved derivations that may have been deleted need to be preserved, otherwise the outputs of other unresolved derivations will be effectively forgotten. + GC for the build trace is not yet implemented due to the lack of a clear default policy. + +- Structured signature for realisations and `path-info` [#15009](https://github.com/NixOS/nix/pull/15009) + + [Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. + `nix path-info --json --json-format 3` opts into the new version for this command. + JSON parsing accepts both the old string format and new structured format for backwards compatibility. + + This format is also used for the build trace entries in binary caches. + +- `nix realisation` command has been renamed to `nix store build-trace` [#16000](https://github.com/NixOS/nix/pull/16000) [#15948](https://github.com/NixOS/nix/pull/15948) + +## Build performance improvements + +- Make post-build-hook asynchronous [#15406](https://github.com/NixOS/nix/issues/15406) [#15451](https://github.com/NixOS/nix/pull/15451) + + The [`post-build-hook`](@docroot@/command-ref/conf-file.md#conf-post-build-hook) now runs asynchronously, without blocking the build event loop. + Dependent builds are not started until the hook finishes, but multiple hook instances are now launched concurrently -- up to the [`max-jobs`](@docroot@/command-ref/conf-file.md#conf-max-jobs) limit. + +- zstd compression now emits multi-frame output and uses less memory [#15550](https://github.com/NixOS/nix/pull/15550) + + zstd-compressed NARs are now written as a sequence of independent 16 MiB frames instead of a single large frame. + This lays the groundwork for parallel decompression in a future release without requiring caches to be repopulated, and significantly lowers peak memory use during compression + (e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). + + The output remains standard zstd and is decoded unchanged by existing Nix binaries and the `zstd` CLI; compression ratio is effectively unchanged. + + Per-frame compression now uses up to 4 worker threads. For zstd this is the new default: the [`parallel-compression`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-parallel-compression) store setting defaults to `true` when `compression=zstd` (it remains `false` for other compression algorithms like `xz`). + Set `?parallel-compression=false` to opt out. + +- More parallelism for binary cache uploads [#15957](https://github.com/NixOS/nix/pull/15957) + + Uploads of NARs now start without waiting for all references to be uploaded. + Also, NARs are now uploaded in order of descending (decompressed) NAR size. + The closure invariant is still maintained by copying `.narinfo` in a topologically sorted order. + +- The derivation build scheduler memory usage reduction and performance improvements [#15611](https://github.com/NixOS/nix/pull/15611) [#15695](https://github.com/NixOS/nix/pull/15695) + + Memory usage of the derivation build scheduler has been improved to allow more state sharing. + Inefficiencies leading to quadratic complexity of scheduling build/substitution jobs have been addressed. + Scheduling resources are allocated more sparingly and freed earlier to reduce peak consumption. + + These improvements amount to ~2-8x less `nix-daemon` memory usage for typical workloads and more in larger derivation graphs, not accounting for short-lived allocations used during substitution. + + Notably, the current architecture of the build scheduler gets proportionally slower on Linux with larger heaps as derivation "builder" processes are `fork`-ed directly from the Nix process, which blocks the builder event loop for the duration of the `fork`. Thus, smaller heap of `nix-daemon` translates into faster build startups. + +- Concurrent path substitutions and eval-time fetches of the same inputs now run only once [#15555](https://github.com/NixOS/nix/pull/15555) [#15644](https://github.com/NixOS/nix/pull/15644) + + This avoids redundant work in case multiple Nix processes try to substitute/download the same resource concurrently. + +- `.narinfo` lookups in binary caches are more concurrent + + Querying the existence and path metadata in binary caches is now more asynchronous. Operations like `nix path-info` on large closures are faster and more efficient. + The build scheduler event loop now doesn't block on `.narinfo` queries, which improves performance with passthru binary caches. + +## Bug fixes + +- Fix hash collision between store paths with self-references and their zeroed-out equivalents [#15837](https://github.com/NixOS/nix/issues/15837) [#15931](https://github.com/NixOS/nix/pull/15931) + + When computing the hash of a NAR with self-references, Nix zeroes out the self-references but also hashes their positions. + The latter was accidentally lost in Nix 2.17.0, which meant a NAR with self-references could hash to the same store path as an otherwise-identical NAR in which some of the self-references had been zeroed out. + + This release restores hashing the positions of self-references. + As a consequence, content-addressed store paths derived from self-referential NARs will differ from those produced by Nix 2.17 through 2.34. + This affects users of the experimental `ca-derivations` features, as well as users of `nix store make-content-addressed`. + +- C API: Fix `EvalState` pointer passed to primop callbacks [#15300](https://github.com/NixOS/nix/pull/15300) [#15383](https://github.com/NixOS/nix/pull/15383) + + The `EvalState *` passed to C API primop callbacks was incorrectly pointing to the internal `nix::EvalState` rather than the C API wrapper struct. + This caused a segfault when the callback used the pointer with C API functions such as `nix_alloc_value()`. + The same issue affected `printValueAsJSON` and `printValueAsXML` callbacks on external values. + +- GitHub fetcher now validates URL parameters [#15304](https://github.com/NixOS/nix/issues/15304) [#15331](https://github.com/NixOS/nix/pull/15331) + + The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. + +- Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands [#15776](https://github.com/NixOS/nix/pull/15776) + + Setting [`keep-derivations`](@docroot@/command-ref/conf-file.md#conf-keep-derivations) to `true` and trying to delete a derivation with realised outputs would previously fail. + Same with [`keep-outputs`](@docroot@/command-ref/conf-file.md#conf-keep-outputs) and trying to delete an output that still has derivers. + These options no longer affect the deletion commands, and are now documented as such. + +- S3 substituters fall back to the URL's region for STS WebIdentity auth [#15594](https://github.com/NixOS/nix/pull/15594) + + When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL as a fallback for the STS endpoint region if neither `AWS_REGION` nor `AWS_DEFAULT_REGION` is set. + Previously, IRSA setups that exported `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail with a misleading "IMDS provider" error. + +- S3: restore STS WebIdentity and ECS container credential providers [#15507](https://github.com/NixOS/nix/pull/15507) + + Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a custom chain built on `aws-c-auth`. + That chain omitted two providers, breaking S3 binary cache access in container workloads: + + - **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, `AWS_ROLE_SESSION_NAME`) -- used by EKS IRSA, GitHub Actions OIDC, and any `sts:AssumeRoleWithWebIdentity` federation. + - **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, `AWS_CONTAINER_CREDENTIALS_FULL_URI`) -- used by ECS tasks and EKS Pod Identity. + + The typical symptom was a misleading IMDS error (`Valid credentials could not be sourced by the IMDS provider`), because IMDS is the last provider tried after the correct one was skipped. + + Both providers are now part of the chain, ordered to match the pre-2.33 behaviour. + As in both the old and new AWS SDK default chains, ECS and IMDS are mutually exclusive: when container credential environment variables are set, IMDS is skipped. + +- HTTP 401 and 407 responses from binary caches are no longer treated as missing files [#15877](https://github.com/NixOS/nix/pull/15877) + + Nix no longer treats `Unauthorized` and `Proxy Authentication Required` HTTP codes as an indication of a missing file. This used to be the case because AWS S3 returns 403 `Forbidden` for missing objects in unlistable buckets. 401/407 were accidentally included and this workaround is now tightly scoped to 403 responses. + +- Fixed `nixbld` gid in `/etc/group` in the Linux build sandbox when user namespaces are not supported [#15131](https://github.com/NixOS/nix/pull/15131) + +- Store garbage collection is now more robust [#15992](https://github.com/NixOS/nix/pull/15992) [#15720](https://github.com/NixOS/nix/pull/15720) [#15616](https://github.com/NixOS/nix/pull/15616) + +- Fixed deadlock for hash-mismatching fixed-output derivations [#15874](https://github.com/NixOS/nix/pull/15874) + +- `nix-copy-closure` no longer ignores `--include-outputs` flag [#15896](https://github.com/NixOS/nix/pull/15896) + +- Fixes to `recursive-nix` experimental feature + + Prior to this release, internal datastructures used to implement this feature were not used in a thread-safe manner. + Threads handling daemon connections are now reaped promptly, fixing resource leaks. + +## Contributors + +This release was made possible by the following 59 contributors: + +- Michael Wang [**(@zwang20)**](https://github.com/zwang20) +- Amaan Qureshi [**(@amaanq)**](https://github.com/amaanq) +- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium) +- Reuben Gardos Reid [**(@ReubenJ)**](https://github.com/ReubenJ) +- StepBroBD [**(@stepbrobd)**](https://github.com/stepbrobd) +- dram [**(@dramforever)**](https://github.com/dramforever) +- Tom [**(@thunze)**](https://github.com/thunze) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- steveoliphant [**(@steveoliphant)**](https://github.com/steveoliphant) +- espes [**(@espes)**](https://github.com/espes) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Artemis Tosini [**(@artemist)**](https://github.com/artemist) +- sander [**(@sandydoo)**](https://github.com/sandydoo) +- Erik Jensen [**(@rkjnsn)**](https://github.com/rkjnsn) +- Cameron Will [**(@cwill747)**](https://github.com/cwill747) +- Maciej Krüger [**(@mkg20001)**](https://github.com/mkg20001) +- Dror Speiser [**(@drorspei)**](https://github.com/drorspei) +- Eveeifyeve [**(@Eveeifyeve)**](https://github.com/Eveeifyeve) +- Audrey Dutcher [**(@rhelmot)**](https://github.com/rhelmot) +- Lisanna Dettwyler [**(@lisanna-dettwyler)**](https://github.com/lisanna-dettwyler) +- TyIsI [**(@TyIsI)**](https://github.com/TyIsI) +- Adam Kliś [**(@BonusPlay)**](https://github.com/BonusPlay) +- Domen Kožar [**(@domenkozar)**](https://github.com/domenkozar) +- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii) +- ryota2357 [**(@ryota2357)**](https://github.com/ryota2357) +- LIN, Jian [**(@jian-lin)**](https://github.com/jian-lin) +- znmz [**(@znmz)**](https://github.com/znmz) +- Felix Stupp [**(@Zocker1999NET)**](https://github.com/Zocker1999NET) +- Johannes Kirschbauer [**(@hsjobeki)**](https://github.com/hsjobeki) +- Antonio Nuno Monteiro [**(@anmonteiro)**](https://github.com/anmonteiro) +- tomberek [**(@tomberek)**](https://github.com/tomberek) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- adisbladis [**(@adisbladis)**](https://github.com/adisbladis) +- Luna Nova [**(@LunNova)**](https://github.com/LunNova) +- Riccardo Mazzarini [**(@noib3)**](https://github.com/noib3) +- Bouke van der Bijl [**(@bouk)**](https://github.com/bouk) +- Dario [**(@dve00)**](https://github.com/dve00) +- Michael Hoang [**(@Enzime)**](https://github.com/Enzime) +- Paul Sbarra [**(@tones111)**](https://github.com/tones111) +- edef [**(@edef1c)**](https://github.com/edef1c) +- Adam Dinwoodie [**(@me-and)**](https://github.com/me-and) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Jeremy Fleischman [**(@jfly)**](https://github.com/jfly) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Alex Ionescu [**(@aionescu)**](https://github.com/aionescu) +- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy) +- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault) +- Pierre Penninckx [**(@ibizaman)**](https://github.com/ibizaman) +- Leonard Sheng Sheng Lee [**(@sheeeng)**](https://github.com/sheeeng) +- rszyma [**(@rszyma)**](https://github.com/rszyma) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Lennart Kolmodin [**(@kolmodin)**](https://github.com/kolmodin) +- zowoq [**(@zowoq)**](https://github.com/zowoq) +- Peter Collingbourne [**(@pcc)**](https://github.com/pcc) +- Simon Žlender [**(@szlend)**](https://github.com/szlend) +- Lily Foster [**(@lilyinstarlight)**](https://github.com/lilyinstarlight) +- randomizedcoder [**(@randomizedcoder)**](https://github.com/randomizedcoder) +- Krish Jaiswal From 7b99db2a09d65da80a23f9f40f45dd122703151a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 8 Jul 2026 23:56:21 +0300 Subject: [PATCH 541/555] Once again follow final symlinks in GitRepoImpl::getAccessor This used to be the case before 02e4f4ad75fda707a594ffd8ad511691fc1a311f, and this restores that compatibility. Ideally we'd check that access to that path is allowed, but libfetchers does this now pretty inconsistently. Also test a lot more things. (cherry picked from commit 3aff4dc5edf30998d64eec024de186ac2d6fb5ea) --- src/libfetchers/git-utils.cc | 16 +++++++++------- tests/functional/git/meson.build | 5 ++++- tests/functional/git/symlinked-repo.sh | 17 +++++++++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) create mode 100644 tests/functional/git/symlinked-repo.sh diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index e03503280831..cd04bfee774e 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1449,13 +1449,15 @@ ref GitRepoImpl::getAccessor( const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) { auto self = ref(shared_from_this()); - ref fileAccessor = AllowListSourceAccessor::create( - makeFSSourceAccessor(path), - /*allowedPrefixes=*/wd.files, - // Always allow access to the root, but not its children. - /*allowedPaths=*/{CanonPath::root}, - std::move(makeNotAllowedError)) - .cast(); + ref fileAccessor = + AllowListSourceAccessor::create( + // Follow the final symlink to the repo. Older nix versions used to do this (maybe somewhat accidentally). + makeFSSourceAccessor(path, /*trackLastModified=*/false, FinalSymlink::Follow), + /*allowedPrefixes=*/wd.files, + // Always allow access to the root, but not its children. + /*allowedPaths=*/{CanonPath::root}, + std::move(makeNotAllowedError)) + .cast(); if (options.exportIgnore) fileAccessor = make_ref(self, fileAccessor, std::nullopt); return fileAccessor; diff --git a/tests/functional/git/meson.build b/tests/functional/git/meson.build index af6882698b2c..73c4bc782871 100644 --- a/tests/functional/git/meson.build +++ b/tests/functional/git/meson.build @@ -1,6 +1,9 @@ suites += { 'name' : 'git', 'deps' : [], - 'tests' : [ 'packed-refs-no-cache.sh' ], + 'tests' : [ + 'packed-refs-no-cache.sh', + 'symlinked-repo.sh', + ], 'workdir' : meson.current_source_dir(), } diff --git a/tests/functional/git/symlinked-repo.sh b/tests/functional/git/symlinked-repo.sh new file mode 100644 index 000000000000..e30b0b3c0d8f --- /dev/null +++ b/tests/functional/git/symlinked-repo.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +source ../common.sh + +requireGit + +repo=$TEST_ROOT/repo +link=$TEST_ROOT/link + +createGitRepo "$repo" +ln -s "$repo" "$link" + +# Dereferencing final symlink component should work and follow it to the directory. +# Also test various cases of symlink trickery in case that ever changes. +for path in {repo,link}{,/,/.} {repo,link}/.././repo{,/,/.}; do + [ "$(nix-instantiate --eval --expr "builtins.readFileType (builtins.fetchTree \"git+file://$TEST_ROOT/$path\").outPath")" == '"directory"' ] +done From 808f22dd25946c016cceeb9210ea78516f5debd8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 11 Jul 2026 00:26:17 +0300 Subject: [PATCH 542/555] libutil: Don't trample parent's havePrivateMountNs in restoreMountNamespace This is only a band-aid, see the comment for the explanation why the current code sucks in an indescribable way. A much better solution would be to get rid of vfork() entirely (or pay the incredible cost of making it safe for each libc we target) by replacing that with a zygote process. That's a huge change though, so let's cross our fingers and pray that this sucky code won't lead to a security issue. (cherry picked from commit 9640ab1fa4a88def1c5be31e45cd819caa24a638) --- src/libutil/linux/linux-namespaces.cc | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index f0416307d126..5ad9bfc9decc 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -161,12 +161,25 @@ void remountReadOnlyWritable(const std::filesystem::path & path) throw SysError("remounting %s writable", PathFmt(path)); } +/* This code runs in a (possibly) vfork-ed child, so technically everything you see below is beyond + broken because vfork()-ed child: + + * Must not trample parent's memory in any way shape or form. That includes (but not limited to) + * Throwing any exceptions (because that would unwind into the parent stack frame and do who knows what). + * Modify any state - obviously that includes global state. + * Not allocate any memory, since that can also lead to a deadlock if some thread in the (now stopped) parent + holds a lock while we are running. That's because *all* of the parent tasks are suspended for the duration + of the vfork. + + As it stands now, this code should be considered incredibly fragile and slated for a complete rework. + */ void restoreMountNamespace() { if (!havePrivateMountNs) return; try { + /* FIXME: Allocation in a possibly vforked child. */ auto savedCwd = std::filesystem::current_path(); if (setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) @@ -182,7 +195,8 @@ void restoreMountNamespace() if (chdir(savedCwd.c_str()) == -1) throw SysError("restoring cwd"); - havePrivateMountNs = false; + /* Do not reset havePrivateMountNs! This code can run in a vfork-ed child and we absolutely + must not trample any of the parent's state. */ } catch (Error & e) { debug(e.msg()); } From 8ab6055b9980befdca4855d3e39d0f472a535d58 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 13 Jul 2026 20:23:01 +0300 Subject: [PATCH 543/555] Make official release --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index ac2bd551121c..67532424ee67 100644 --- a/flake.nix +++ b/flake.nix @@ -31,7 +31,7 @@ let inherit (nixpkgs) lib; - officialRelease = false; + officialRelease = true; linux32BitSystems = [ "i686-linux" ]; linux64BitSystems = [ From 022e8aa6cb5bbd9f86dae9db04b556d6539bf337 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 12 Jul 2026 15:29:17 +0300 Subject: [PATCH 544/555] packaging: Fix sqlite on mingw tcl is broken on mingw in nixpkgs and so is sqlite. This makes it build now at least. (cherry picked from commit 526e916abd12032141f7bd45d67d2d9f252b1be3) --- packaging/dependencies.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 17071fb0c443..5777dd564962 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -62,6 +62,17 @@ scope: { ); }; + sqlite = + if !stdenv.hostPlatform.isWindows then + pkgs.sqlite + else + pkgs.sqlite.overrideAttrs (prevAttrs: { + nativeBuildInputs = lib.filter (x: !(x.pname == "tcl")) prevAttrs.nativeBuildInputs or [ ]; + configureFlags = (lib.filter (x: !(lib.hasPrefix "--with-tcl" x)) prevAttrs.configureFlags) ++ [ + "--disable-tcl" + ]; + }); + libgit2 = if lib.versionAtLeast pkgs.libgit2.version "1.9.4" then pkgs.libgit2 From 370bd13240d7688198b10cf3a30839fc4f079fbc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Jul 2026 01:46:38 +0300 Subject: [PATCH 545/555] Bump pinned nix in install-nix-action composite action (cherry picked from commit f553163d4cbad6d437b8ee12d660cb5a96df1e99) --- .github/actions/install-nix-action/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml index f889944b2d03..2fb433b741c3 100644 --- a/.github/actions/install-nix-action/action.yaml +++ b/.github/actions/install-nix-action/action.yaml @@ -9,7 +9,7 @@ inputs: install_url: description: "URL of the Nix installer" required: false - default: "https://releases.nixos.org/nix/nix-2.32.1/install" + default: "https://releases.nixos.org/nix/nix-2.34.8/install" github_token: description: "Github token" required: true From 6c985da02e17633b75f54de97ccd4e7e59a17f53 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Jul 2026 01:51:24 +0300 Subject: [PATCH 546/555] Use stable nix for PR CI --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f294211ea74..75637c096bd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: dogfood: description: 'Use dogfood Nix build' required: false - default: true + default: false type: boolean concurrency: @@ -29,7 +29,7 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/install-nix-action with: - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false extra_nix_config: experimental-features = nix-command flakes github_token: ${{ secrets.GITHUB_TOKEN }} @@ -43,7 +43,7 @@ jobs: - uses: actions/checkout@v6 - uses: ./.github/actions/install-nix-action with: - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false extra_nix_config: experimental-features = nix-command flakes github_token: ${{ secrets.GITHUB_TOKEN }} - run: ./ci/gha/tests/pre-commit-checks @@ -93,7 +93,7 @@ jobs: - uses: ./.github/actions/install-nix-action with: github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: @@ -150,7 +150,7 @@ jobs: - uses: ./.github/actions/install-nix-action with: github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false - name: Run Windows unit tests run: | nix build --file ci/gha/tests/windows.nix unitTests.nix-util-tests -L @@ -223,7 +223,7 @@ jobs: - uses: ./.github/actions/install-nix-action with: github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false extra_nix_config: "sandbox = true" - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Run clang-tidy @@ -278,7 +278,7 @@ jobs: - uses: ./.github/actions/install-nix-action with: github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false extra_nix_config: | experimental-features = flakes nix-command ca-derivations impure-derivations max-jobs = 1 From 3d4e186068fafcaf984c469642b548ba2f5c8271 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Jul 2026 00:36:16 +0300 Subject: [PATCH 547/555] Don't openStore in nix-env --version This has caused dealocks after the fixes to acquire an exclusive lock for performing store DB migrations, because upgrade-nix.cc runs a nix-env --version to check that the command works at all. Since the old code would try to open a store, we'd deadlock. Fixes #16136 (cherry picked from commit 5063db70674d5db052af50d59135aa9ae03c16ba) --- src/libmain/include/nix/main/shared.hh | 2 +- src/nix/nix-env/nix-env.cc | 28 ++++++++++++++------------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/libmain/include/nix/main/shared.hh b/src/libmain/include/nix/main/shared.hh index f9e771205ba2..7b0e7a54d4e0 100644 --- a/src/libmain/include/nix/main/shared.hh +++ b/src/libmain/include/nix/main/shared.hh @@ -26,7 +26,7 @@ void parseCmdLine( const Strings & args, fun parseArg); -void printVersion(const std::string & programName); +[[noreturn]] void printVersion(const std::string & programName); /** * Ugh. No better place to put this. diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index abe04d96341c..7fc518d21534 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -1391,7 +1391,7 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr } } -static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) +[[noreturn]] static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) { printVersion("nix-env"); } @@ -1508,23 +1508,25 @@ static int main_nix_env(int argc, char ** argv) if (!op) throw UsageError("no operation specified"); - auto store = openStore(); + if (op != opVersion) { + auto store = openStore(); - globals.state = - std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); - globals.state->repair = myArgs.repair; + globals.state = + std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); + globals.state->repair = myArgs.repair; - globals.instSource.nixExprPath = std::make_shared( - file != "" ? lookupFileArg(*globals.state, file) - : globals.state->rootPath(CanonPath(nixExprPath.string()))); + globals.instSource.nixExprPath = std::make_shared( + file != "" ? lookupFileArg(*globals.state, file) + : globals.state->rootPath(CanonPath(nixExprPath.string()))); - globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); + globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); - if (globals.profile == "") - globals.profile = getEnv("NIX_PROFILE").value_or(""); + if (globals.profile == "") + globals.profile = getEnv("NIX_PROFILE").value_or(""); - if (globals.profile == "") - globals.profile = getDefaultProfile(settings.getProfileDirsOptions()).string(); + if (globals.profile == "") + globals.profile = getDefaultProfile(settings.getProfileDirsOptions()).string(); + } op(globals, std::move(opFlags), std::move(opArgs)); From 8a0b159b5a92800bec61f5ca552794f69fd9ac1b Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Mon, 13 Jul 2026 18:18:10 -0400 Subject: [PATCH 548/555] Revert temproots path name change This is causing breakages when downgrading from nix 2.35.0, since it expects the filename to have a parseable integer. Signed-off-by: Lisanna Dettwyler (cherry picked from commit c1cc521d2ffbc0b9e0a2cf3e3aa22c9d04f037ef) --- src/libstore/gc.cc | 4 ++-- src/libstore/local-store.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index e399dd949a61..8e997c8e2471 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -61,8 +61,8 @@ void LocalStore::createTempRootsFile() while (1) { if (pathExists(fnTempRoots)) - /* The file is stale since each LocalStore instance - uses a unique filename (pid + counter). */ + /* It *must* be stale, since there can be no two + processes with the same pid. */ tryUnlink(fnTempRoots); *fdTempRoots = openLockFile(fnTempRoots, true); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5a1c161c0739..909f22369dc0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -129,7 +129,7 @@ LocalStore::LocalStore(ref config) , reservedPath(dbDir / "reserved") , schemaPath(dbDir / "schema") , tempRootsDir(config->stateDir.get() / "temproots") - , fnTempRoots(makeTempPath(tempRootsDir, "temproots")) + , fnTempRoots(tempRootsDir / std::to_string(getpid())) { auto state(_state->lock()); state->stmts = std::make_unique(); From 85855aacbf5659dd85a8a290c2532d5dc6196555 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Jul 2026 02:07:05 +0300 Subject: [PATCH 549/555] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index aa5388f63762..d07233cc933c 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.35.0 +2.35.1 From 65646c2b0b6eea5fcbc31f305e857b6a3a34aa43 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jul 2026 16:48:19 +0200 Subject: [PATCH 550/555] Fix building without wasmtime --- src/libstore/unix/build/wasi-derivation-builder.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/wasi-derivation-builder.cc b/src/libstore/unix/build/wasi-derivation-builder.cc index 9334fb0a70ad..6fb7c41c229a 100644 --- a/src/libstore/unix/build/wasi-derivation-builder.cc +++ b/src/libstore/unix/build/wasi-derivation-builder.cc @@ -1,6 +1,10 @@ -#include "derivation-builder-impl.hh" +#include "store-config-private.hh" -#include +#if NIX_USE_WASMTIME + +# include "derivation-builder-impl.hh" + +# include namespace nix { @@ -90,3 +94,5 @@ DerivationBuilderUnique makeWasiDerivationBuilder( } } // namespace nix + +#endif // NIX_USE_WASMTIME From 7d803ff947dd9b10712d76ce63b3eb18ea8b3906 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jul 2026 17:07:48 +0200 Subject: [PATCH 551/555] Disable mimalloc in the static build on aarch64-darwin This fails with a duplicate symbol '_reallocarray' error between mimalloc and lowdown. --- src/nix/package.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nix/package.nix b/src/nix/package.nix index 3752f370e83d..be9a253e6f3a 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -30,7 +30,12 @@ # workloads (~10-15% on large evaluations). # mimalloc is disabled on FreeBSD due to a crash in nixpkgs 25.11. # Once the nixpkgs flake is updated, mimalloc can be enabled again. - withMimalloc ? !stdenv.hostPlatform.isWindows && !stdenv.hostPlatform.isFreeBSD, + # It's also disabled on static aarch64-darwin because of a duplicate + # `reallocarray` symbol in libmimalloc.a and lowdown's compats.o. + withMimalloc ? + !stdenv.hostPlatform.isWindows + && !stdenv.hostPlatform.isFreeBSD + && !(stdenv.hostPlatform.isStatic && stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64), # Whether to embed the public C API into the `nix` executable so plugins can # resolve those symbols without linking Nix libraries directly. From ea2a649a58595751841436b75de8e4a00763a5f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jul 2026 17:33:26 +0200 Subject: [PATCH 552/555] Shut up compiler warning --- src/libutil-tests/compression.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libutil-tests/compression.cc b/src/libutil-tests/compression.cc index 7978a008fe65..60d2cd29ed35 100644 --- a/src/libutil-tests/compression.cc +++ b/src/libutil-tests/compression.cc @@ -29,8 +29,9 @@ TEST_P(CompressionDecompressionTest, roundtrip) TEST_P(CompressionDecompressionTest, empty) { auto compressed = compress(GetParam(), ""); - if (GetParam() != CompressionAlgo::none) + if (GetParam() != CompressionAlgo::none) { ASSERT_FALSE(compressed.empty()); + } auto o = decompress(GetParam(), compressed); ASSERT_EQ(o, ""); } From dbbc5e1f47a6430c11cd03c5d744f91a4c8580a8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jul 2026 19:36:50 +0200 Subject: [PATCH 553/555] Fix use of readOnlyMode --- src/libexpr/paths.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 84caa1ca5af0..6d644fe961e8 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -29,7 +29,7 @@ StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites) fetchSettings, *store, SourcePath{ref(mount)}, - settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + settings.isReadOnly() ? FetchMode::DryRun : FetchMode::Copy, path.name()); assert(storePath.name() == path.name()); if (rewrites) From 26c2df11d2ca56cfb54f41c4d6eedfcc1b377b51 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jul 2026 19:58:35 +0200 Subject: [PATCH 554/555] Fix linkcheck --- .../protocols/json/schema/derivation-resolved-v4.yaml | 6 ------ doc/manual/source/protocols/json/schema/signature-v2.yaml | 6 ------ 2 files changed, 12 deletions(-) diff --git a/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml index 781f07fb374f..705156eb8451 100644 --- a/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml @@ -12,12 +12,6 @@ description: | [primary (not necessarily resolved) JSON schema](@docroot@/protocols/json/derivation/index.md), but actually it is the first version for this format. - > **Warning** - > - > This JSON format is currently - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) - > and subject to change. - type: object required: - name diff --git a/doc/manual/source/protocols/json/schema/signature-v2.yaml b/doc/manual/source/protocols/json/schema/signature-v2.yaml index e2b25dbc6fc0..d94798902f47 100644 --- a/doc/manual/source/protocols/json/schema/signature-v2.yaml +++ b/doc/manual/source/protocols/json/schema/signature-v2.yaml @@ -6,12 +6,6 @@ description: | This schema describes the JSON representation of signatures as used in various Nix JSON APIs. - > **Warning** - > - > This JSON format is currently - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) - > and subject to change. - ## Version History - Version 1: Colon-separated string in the format `:` From 8051880e0a6842c10afd4710688af49d38028e9a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jul 2026 14:18:10 +0200 Subject: [PATCH 555/555] Remove redundant unity settings --- src/libexpr/meson.build | 2 -- src/libfetchers/meson.build | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 484ee8993f11..995202dbcb30 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -7,8 +7,6 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'unity=on', - 'unity_size=1024', ], meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 1a38b1905adf..134fd496a7a1 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -7,8 +7,6 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'unity=on', - 'unity_size=1024', ], meson_version : '>= 1.8', license : 'LGPL-2.1-or-later',