diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d7c9f71cda86..445563f38d22 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -29,6 +29,7 @@ #include "nix/util/processes.hh" #include "nix/store/async-path-writer.hh" #include "nix/expr/parallel-eval.hh" +#include "nix/util/forwarding-source-accessor.hh" #include "parser-tab.hh" @@ -532,6 +533,26 @@ static std::string sanitizeZoneNameForStore(std::string_view zonePath) return result; } +// Helper to sanitize a subpath for use in store path names. +// Same rules as sanitizeZoneNameForStore. +static std::string sanitizeSubpath(std::string_view subpath) +{ + std::string result; + result.reserve(subpath.size()); + for (char c : subpath) { + if (c == '/') { + result += '-'; + } else if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || c == '+' || c == '-' || + c == '.' || c == '_' || c == '?' || c == '=') { + result += c; + } else { + result += '_'; + } + } + return result; +} + Hash EvalState::getWorldTreeSha(std::string_view worldPath) const { auto path = normalizeZonePath(worldPath); @@ -885,6 +906,41 @@ StorePath EvalState::mountZoneByTreeSha(const Hash & treeSha, std::string_view z return storePath; } +/** + * Wraps a SourceAccessor so that all operations are rooted at `prefix`. + * E.g., readFile("/foo") becomes next->readFile(prefix / "/foo"). + */ +struct PrefixSourceAccessor : ForwardingSourceAccessor +{ + CanonPath prefix; + + PrefixSourceAccessor(ref next, CanonPath prefix) + : ForwardingSourceAccessor(next), prefix(std::move(prefix)) + { + } + + std::string readFile(const CanonPath & path) override + { return next->readFile(prefix / path); } + + void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override + { next->readFile(prefix / path, sink, sizeCallback); } + + std::optional maybeLstat(const CanonPath & path) override + { return next->maybeLstat(prefix / path); } + + DirEntries readDirectory(const CanonPath & path) override + { return next->readDirectory(prefix / path); } + + std::string readLink(const CanonPath & path) override + { return next->readLink(prefix / path); } + + std::string showPath(const CanonPath & path) override + { return next->showPath(prefix / path); } + + std::optional getPhysicalPath(const CanonPath & path) override + { return next->getPhysicalPath(prefix / path); } +}; + /** * Overlays dirty files from disk on top of a clean git tree accessor. */ @@ -1004,6 +1060,166 @@ StorePath EvalState::getZoneFromCheckout(std::string_view zonePath, const boost: return storePath; } +StorePath EvalState::getZoneSubpathStorePath(std::string_view zonePath, std::string_view subpath) +{ + auto subpathCanon = CanonPath("/" + std::string(subpath)); + + // Check dirty status + const ZoneDirtyInfo * dirtyInfo = nullptr; + if (isTectonixSourceAvailable()) { + auto & dirtyZones = getTectonixDirtyZones(); + auto it = dirtyZones.find(std::string(zonePath)); + if (it != dirtyZones.end() && it->second.dirty) + dirtyInfo = &it->second; + } + + if (dirtyInfo) { + debug("getZoneSubpathStorePath: %s/%s is dirty, using checkout", zonePath, subpath); + return getZoneSubpathFromCheckout(zonePath, subpath, &dirtyInfo->dirtyFiles); + } + + // Clean zone: get tree SHA and accessor to validate subpath exists + auto treeSha = getWorldTreeSha(zonePath); + auto repo = getWorldRepo(); + auto commitHash = Hash::parseNonSRIUnprefixed(requireTectonixGitSha(), HashAlgorithm::SHA1); + auto opts = makeZoneAccessorOptions(repo, commitHash, normalizeZonePath(zonePath)); + auto accessor = repo->getAccessor(treeSha, opts, "zone"); + + // Eagerly validate that the subpath exists in the zone + if (!accessor->pathExists(subpathCanon)) + throw Error("subpath '%s' does not exist in zone '%s'", subpath, zonePath); + + if (!settings.lazyTrees) { + debug("getZoneSubpathStorePath: %s/%s clean, eager copy from git (tree %s)", zonePath, subpath, treeSha.gitRev()); + + std::string name = "zone-" + sanitizeZoneNameForStore(zonePath) + "-" + sanitizeSubpath(subpath); + auto storePath = fetchToStore( + fetchSettings, *store, + SourcePath(accessor, subpathCanon), + FetchMode::Copy, name); + + allowPath(storePath); + return storePath; + } + + debug("getZoneSubpathStorePath: %s/%s clean, lazy mount (tree %s)", zonePath, subpath, treeSha.gitRev()); + return mountZoneSubpathByTreeSha(treeSha, zonePath, subpath); +} + +StorePath EvalState::mountZoneSubpathByTreeSha(const Hash & treeSha, std::string_view zonePath, std::string_view subpath) +{ + auto cacheKey = std::make_pair(treeSha, std::string(subpath)); + + // Read lock fast path + { + auto cache = tectonixZoneSubpathCache_.readLock(); + auto it = cache->find(cacheKey); + if (it != cache->end()) return it->second; + } + + // Write lock check for races + { + auto cache = tectonixZoneSubpathCache_.lock(); + auto it = cache->find(cacheKey); + if (it != cache->end()) return it->second; + } + + // Expensive work without holding lock + auto repo = getWorldRepo(); + auto commitHash = Hash::parseNonSRIUnprefixed(requireTectonixGitSha(), HashAlgorithm::SHA1); + auto opts = makeZoneAccessorOptions(repo, commitHash, std::string(zonePath)); + auto accessor = repo->getAccessor(treeSha, opts, "zone"); + + auto subpathCanon = CanonPath("/" + std::string(subpath)); + auto prefixedAccessor = make_ref(accessor, subpathCanon); + + std::string name = "zone-" + sanitizeZoneNameForStore(zonePath) + "-" + sanitizeSubpath(subpath); + auto storePath = StorePath::random(name); + + // Re-acquire lock and check before mounting + auto cache = tectonixZoneSubpathCache_.lock(); + auto it = cache->find(cacheKey); + if (it != cache->end()) return it->second; + + storeFS->mount(CanonPath(store->printStorePath(storePath)), prefixedAccessor); + allowPath(storePath); + cache->emplace(cacheKey, storePath); + + debug("mounted zone subpath %s/%s (tree %s) at %s", + zonePath, subpath, treeSha.gitRev(), store->printStorePath(storePath)); + + return storePath; +} + +StorePath EvalState::getZoneSubpathFromCheckout(std::string_view zonePath, std::string_view subpath, const boost::unordered_flat_set * dirtyFiles) +{ + auto zone = normalizeZonePath(zonePath); + std::string name = "zone-" + sanitizeZoneNameForStore(zonePath) + "-" + sanitizeSubpath(subpath); + auto checkoutPath = settings.tectonixCheckoutPath.get(); + auto fullPath = std::filesystem::path(checkoutPath) / zone; + auto subpathCanon = CanonPath("/" + std::string(subpath)); + + auto makeDirtyAccessor = [&]() -> ref { + auto repo = getWorldRepo(); + auto commitHash = Hash::parseNonSRIUnprefixed(requireTectonixGitSha(), HashAlgorithm::SHA1); + auto zoneOpts = makeZoneAccessorOptions(repo, commitHash, zone); + auto baseAccessor = repo->getAccessor(getWorldTreeSha(zone), zoneOpts, "zone"); + boost::unordered_flat_set zoneDirtyFiles; + if (dirtyFiles) { + auto zonePrefix = zone + "/"; + for (auto & f : *dirtyFiles) + if (f.starts_with(zonePrefix)) + zoneDirtyFiles.insert(f.substr(zonePrefix.size())); + } + return make_ref( + baseAccessor, makeFSSourceAccessor(fullPath), std::move(zoneDirtyFiles)); + }; + + if (!settings.lazyTrees) { + auto accessor = makeDirtyAccessor(); + + // Eagerly validate subpath exists + if (!accessor->pathExists(subpathCanon)) + throw Error("subpath '%s' does not exist in zone '%s'", subpath, zonePath); + + auto storePath = fetchToStore( + fetchSettings, *store, + SourcePath(accessor, subpathCanon), + FetchMode::Copy, name); + allowPath(storePath); + return storePath; + } + + auto cacheKey = std::make_pair(std::string(zonePath), std::string(subpath)); + + { + auto cache = tectonixCheckoutZoneSubpathCache_.readLock(); + auto it = cache->find(cacheKey); + if (it != cache->end()) return it->second; + } + + auto cache = tectonixCheckoutZoneSubpathCache_.lock(); + auto it = cache->find(cacheKey); + if (it != cache->end()) return it->second; + + if (!std::filesystem::exists(fullPath)) + throw Error("zone '%s' not found in checkout at '%s'", zonePath, fullPath.string()); + + auto dirtyAccessor = makeDirtyAccessor(); + + // Eagerly validate subpath exists + if (!dirtyAccessor->pathExists(subpathCanon)) + throw Error("subpath '%s' does not exist in zone '%s'", subpath, zonePath); + + auto prefixedAccessor = make_ref(dirtyAccessor, subpathCanon); + + auto storePath = StorePath::random(name); + storeFS->mount(CanonPath(store->printStorePath(storePath)), prefixedAccessor); + allowPath(storePath); + cache->emplace(cacheKey, storePath); + return storePath; +} + inline static bool isJustSchemePrefix(std::string_view prefix) { return !prefix.empty() && prefix[prefix.size() - 1] == ':' diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 53d4b4982edc..80cad57c8d42 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -565,6 +565,18 @@ private: */ mutable SharedSync> tectonixCheckoutZoneCache_; + /** + * Cache (tree SHA, subpath) → virtual store path for lazy zone subpath mounts. + * Thread-safe for eval-cores > 1. + */ + mutable SharedSync, StorePath>> tectonixZoneSubpathCache_; + + /** + * Cache (zone path, subpath) → virtual store path for lazy checkout zone subpath mounts. + * Thread-safe for eval-cores > 1. + */ + mutable SharedSync, StorePath>> tectonixCheckoutZoneSubpathCache_; + /** * Mount a zone by tree SHA, returning a (potentially virtual) store path. * Caches by tree SHA for deduplication across world revisions. @@ -577,6 +589,17 @@ private: */ StorePath getZoneFromCheckout(std::string_view zonePath, const boost::unordered_flat_set * dirtyFiles = nullptr); + /** + * Mount a zone subpath by tree SHA, returning a virtual store path + * containing only the subpath contents. + */ + StorePath mountZoneSubpathByTreeSha(const Hash & treeSha, std::string_view zonePath, std::string_view subpath); + + /** + * Get zone subpath store path from checkout (for dirty zones). + */ + StorePath getZoneSubpathFromCheckout(std::string_view zonePath, std::string_view subpath, const boost::unordered_flat_set * dirtyFiles = nullptr); + public: /** @@ -656,6 +679,12 @@ public: */ StorePath getZoneStorePath(std::string_view zonePath); + /** + * Get a store path containing only a subpath of a zone. + * Like getZoneStorePath but scoped to a specific file or subdirectory. + */ + StorePath getZoneSubpathStorePath(std::string_view zonePath, std::string_view subpath); + /** * Return a `SourcePath` that refers to `path` in the root * filesystem. diff --git a/src/libexpr/primops/tectonix.cc b/src/libexpr/primops/tectonix.cc index a06ad1a8f82a..7a90115f314d 100644 --- a/src/libexpr/primops/tectonix.cc +++ b/src/libexpr/primops/tectonix.cc @@ -209,6 +209,74 @@ static RegisterPrimOp primop_unsafeTectonixInternalZoneSrc({ .fun = prim_unsafeTectonixInternalZoneSrc, }); +// ============================================================================ +// builtins.unsafeTectonixInternalZoneSrcSubpath zonePath subpath +// Returns a store path containing only the specified subpath of a zone +// ============================================================================ + +// Validate a subpath: must be non-empty, relative, no ".." components +static void validateSubpath(EvalState & state, const PosIdx pos, std::string_view subpath) +{ + if (subpath.empty()) + state.error("subpath must not be empty") + .atPos(pos).debugThrow(); + + if (subpath.starts_with("/")) + state.error("subpath '%s' must be relative (no leading '/')", subpath) + .atPos(pos).debugThrow(); + + // Check for ".." components + auto p = std::string(subpath); + for (size_t i = 0; i < p.size(); ) { + auto next = p.find('/', i); + auto component = p.substr(i, next == std::string::npos ? next : next - i); + if (component == ".." || component == ".") + state.error("subpath '%s' contains invalid component '%s'", subpath, component) + .atPos(pos).debugThrow(); + if (next == std::string::npos) break; + i = next + 1; + } +} + +static void prim_unsafeTectonixInternalZoneSrcSubpath(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + auto zonePath = state.forceStringNoCtx(*args[0], pos, + "while evaluating the 'zonePath' argument to builtins.unsafeTectonixInternalZoneSrcSubpath"); + + auto subpath = state.forceStringNoCtx(*args[1], pos, + "while evaluating the 'subpath' argument to builtins.unsafeTectonixInternalZoneSrcSubpath"); + + validateZonePath(state, pos, zonePath); + validateSubpath(state, pos, subpath); + + auto storePath = state.getZoneSubpathStorePath(zonePath, subpath); + state.allowAndSetStorePathString(storePath, v); +} + +static RegisterPrimOp primop_unsafeTectonixInternalZoneSrcSubpath({ + .name = "__unsafeTectonixInternalZoneSrcSubpath", + .args = {"zonePath", "subpath"}, + .doc = R"( + Get a subpath of a zone's source as a store path. + + Unlike `unsafeTectonixInternalZoneSrc` which returns the entire zone, + this returns a store path containing only the specified file or subdirectory. + This is more efficient when only a subset of the zone content is needed. + + The subpath must be relative (no leading `/`), non-empty, and must not + contain `.` or `..` components. + + With `lazy-trees = true`, returns a virtual store path that is only + materialized when used as a derivation input (devirtualized). + + Example: `builtins.unsafeTectonixInternalZoneSrcSubpath "//areas/tools/tec" "src/lib"` + + Uses `--tectonix-git-dir` (defaults to `~/world/git`) and requires + `--tectonix-git-sha` to be set. + )", + .fun = prim_unsafeTectonixInternalZoneSrcSubpath, +}); + // ============================================================================ // builtins.unsafeTectonixInternalSparseCheckoutRoots // Returns list of zone IDs in sparse checkout diff --git a/tests/functional/tectonix/basic.sh b/tests/functional/tectonix/basic.sh index 03c51c7bb9bc..8cae65c6664a 100644 --- a/tests/functional/tectonix/basic.sh +++ b/tests/functional/tectonix/basic.sh @@ -70,4 +70,53 @@ if [[ "$zone_is_dirty" == "true" ]]; then fail "Zone should not be dirty in clean repo" fi +# ================================================================== +# Zone source subpath tests +# ================================================================== +echo "Testing zone source subpath access..." + +# Test: Subpath for a single file +subpath_src=$(tectonix_eval "$TEST_WORLD/.git" "$HEAD_SHA" \ + 'builtins.unsafeTectonixInternalZoneSrcSubpath "//areas/tools/dev" "zone.nix"') +echo "Subpath source (file): $subpath_src" + +# Verify it's a store path +if [[ ! "$subpath_src" =~ ^${NIX_STORE_DIR:-/nix/store}/ ]]; then + fail "Subpath source should be a store path, got: $subpath_src" +fi + +# Verify subpath result differs from full zone source +if [[ "$subpath_src" == "$zone_src" ]]; then + fail "Subpath source should differ from full zone source" +fi + +# Test: Subpath for a subdirectory +subpath_dir_src=$(tectonix_eval "$TEST_WORLD/.git" "$HEAD_SHA" \ + 'builtins.unsafeTectonixInternalZoneSrcSubpath "//areas/tools/dev" "src"') +echo "Subpath source (dir): $subpath_dir_src" + +if [[ ! "$subpath_dir_src" =~ ^${NIX_STORE_DIR:-/nix/store}/ ]]; then + fail "Subpath dir source should be a store path, got: $subpath_dir_src" +fi + +# Test: Invalid subpath - empty string +echo "Testing invalid subpath: empty string..." +expect_failure tectonix_eval "$TEST_WORLD/.git" "$HEAD_SHA" \ + 'builtins.unsafeTectonixInternalZoneSrcSubpath "//areas/tools/dev" ""' + +# Test: Invalid subpath - absolute path +echo "Testing invalid subpath: absolute path..." +expect_failure tectonix_eval "$TEST_WORLD/.git" "$HEAD_SHA" \ + 'builtins.unsafeTectonixInternalZoneSrcSubpath "//areas/tools/dev" "/zone.nix"' + +# Test: Invalid subpath - parent traversal +echo "Testing invalid subpath: parent traversal..." +expect_failure tectonix_eval "$TEST_WORLD/.git" "$HEAD_SHA" \ + 'builtins.unsafeTectonixInternalZoneSrcSubpath "//areas/tools/dev" "../tools/tec/zone.nix"' + +# Test: Invalid subpath - nonexistent path +echo "Testing invalid subpath: nonexistent..." +expect_failure tectonix_eval "$TEST_WORLD/.git" "$HEAD_SHA" \ + 'builtins.unsafeTectonixInternalZoneSrcSubpath "//areas/tools/dev" "nonexistent.nix"' + echo "Basic tests passed!" diff --git a/tests/functional/tectonix/common.sh b/tests/functional/tectonix/common.sh index 63946cdae7d9..ccf50db7dab7 100644 --- a/tests/functional/tectonix/common.sh +++ b/tests/functional/tectonix/common.sh @@ -39,6 +39,8 @@ MANIFEST_EOF # Create zone files echo '{ }' > areas/tools/dev/zone.nix echo 'Dev zone README' > areas/tools/dev/README.md + mkdir -p areas/tools/dev/src + echo '{ main = true; }' > areas/tools/dev/src/main.nix echo '{ }' > areas/tools/tec/zone.nix echo '{ }' > areas/platform/core/zone.nix echo 'Test World' > README.md