Tecnix eval caching

misc/launchd/org.nixos.nix-daemon.plist.in

@@ -6,6 +6,8 @@
     <string>org.nixos.nix-daemon</string>
     <key>KeepAlive</key>
     <true/>
+    <key>ProcessType</key>
+    <string>Interactive</string>
     <key>RunAtLoad</key>
     <true/>
     <key>ProgramArguments</key>

src/libexpr/include/nix/expr/eval.hh

@@ -24,6 +24,7 @@
 #include <boost/unordered/unordered_flat_map.hpp>
 #include <boost/unordered/unordered_flat_set.hpp>
 #include <boost/unordered/concurrent_flat_map_fwd.hpp>
+#include <boost/unordered/concurrent_flat_set.hpp>
 
 #include <nlohmann/json_fwd.hpp>
 
@@ -311,6 +312,24 @@ struct StaticEvalSymbols
     }
 };
 
+/**
+ * Tracks file/directory accesses during tecnix target resolution for cache invalidation.
+ * One per target. Paths are repo-relative (e.g. "areas/core/shopify/default.nix").
+ */
+struct TrackingContext {
+    boost::concurrent_flat_set<std::string> accessedPaths;
+
+    void recordAccess(const std::string & path) {
+        accessedPaths.insert(path);
+    }
+};
+
+/**
+ * Thread-local pointer to the active tracking context.
+ * Set during tecnix target resolution, nullptr otherwise.
+ */
+extern thread_local TrackingContext * currentTrackingContext;
+
 class EvalMemory
 {
 #if NIX_USE_BOEHMGC
@@ -527,23 +546,31 @@ private:
     mutable std::once_flag worldGitAccessorFlag;
     mutable std::optional<ref<SourceAccessor>> worldGitAccessor;
 
-    /** Cache: world path → tree SHA (lazy computed, cached at each path level) */
-    const ref<boost::concurrent_flat_map<std::string, Hash>> worldTreeShaCache;
+    /**
+     * Repo-wide source accessor with dirty overlay. Lazily created.
+     * All file reads during tecnix evaluation go through this single accessor,
+     * so tracked paths are naturally repo-relative.
+     */
+    mutable std::once_flag repoAccessorFlag;
+    mutable std::optional<ref<SourceAccessor>> repoAccessor;
 
-    /** Lazy-initialized set of zone IDs in sparse checkout (thread-safe via once_flag) */
-    mutable std::once_flag tectonixSparseCheckoutRootsFlag;
-    mutable std::set<std::string> tectonixSparseCheckoutRoots;
+    /**
+     * Dirty files and their parent directories from the repo checkout.
+     * Used to check whether a commit-keyed cache entry might be stale
+     * due to uncommitted changes overlapping with tracked paths.
+     */
+    mutable boost::unordered_flat_set<std::string> repoDirtyFiles;
+    mutable boost::unordered_flat_set<std::string> repoDirtyDirs;
 
-    /** Per-zone dirty status: whether the zone is dirty, and if so, which
-     *  repo-relative file paths are dirty (from git status). */
-    struct ZoneDirtyInfo {
-        bool dirty = false;
-        boost::unordered_flat_set<std::string> dirtyFiles; // repo-relative paths
-    };
+    /**
+     * Virtual store path where the repo-wide accessor is lazily mounted.
+     * All repo subtree store paths are subpaths of this mount.
+     */
+    mutable std::once_flag repoMountFlag;
+    mutable std::optional<StorePath> repoMountStorePath;
 
-    /** Lazy-initialized map of zone path → dirty info (thread-safe via once_flag) */
-    mutable std::once_flag tectonixDirtyZonesFlag;
-    mutable std::map<std::string, ZoneDirtyInfo> tectonixDirtyZones;
+    /** Cache: world path → tree SHA (lazy computed, cached at each path level) */
+    const ref<boost::concurrent_flat_map<std::string, Hash>> worldTreeShaCache;
 
     /** Cached manifest content (thread-safe via once_flag) */
     mutable std::once_flag tectonixManifestFlag;
@@ -553,30 +580,6 @@ private:
     mutable std::once_flag tectonixManifestJsonFlag;
     mutable std::unique_ptr<nlohmann::json> tectonixManifestJson;
 
-    /**
-     * Cache tree SHA → virtual store path for lazy zone mounts.
-     * Thread-safe for eval-cores > 1.
-     */
-    mutable SharedSync<std::map<Hash, StorePath>> tectonixZoneCache_;
-
-    /**
-     * Cache zone path → virtual store path for lazy checkout zone mounts.
-     * Thread-safe for eval-cores > 1.
-     */
-    mutable SharedSync<std::map<std::string, StorePath>> tectonixCheckoutZoneCache_;
-
-    /**
-     * Mount a zone by tree SHA, returning a (potentially virtual) store path.
-     * Caches by tree SHA for deduplication across world revisions.
-     */
-    StorePath mountZoneByTreeSha(const Hash & treeSha, std::string_view zonePath);
-
-    /**
-     * Get zone store path from checkout (for dirty zones).
-     * With lazy-trees enabled, mounts lazily and caches by zone path.
-     */
-    StorePath getZoneFromCheckout(std::string_view zonePath, const boost::unordered_flat_set<std::string> * dirtyFiles = nullptr);
-
     /**
      * Return the configured tectonix git SHA, or throw if unset.
      */
@@ -622,8 +625,8 @@ public:
      * exportIgnore policy for tectonix accessors:
      * - World accessor (getWorldGitAccessor): exportIgnore=false
      *   Used for path validation and tree SHA computation; needs to see all files
-     * - Zone accessors (mountZoneByTreeSha, getZoneStorePath): exportIgnore=true
-     *   Used for actual zone content; honors .gitattributes for filtered output
+     * - Repo accessor (getRepoAccessor): exportIgnore=true
+     *   Used for repo content; honors .gitattributes for filtered output
      * - Raw tree accessor (__unsafeTectonixInternalTree): exportIgnore=false
      *   Low-level access by SHA; provides unfiltered content
      */
@@ -635,26 +638,38 @@ public:
     /** Check if we're in source-available mode */
     bool isTectonixSourceAvailable() const;
 
-    /** Get set of zone IDs in sparse checkout (source-available mode only) */
-    const std::set<std::string> & getTectonixSparseCheckoutRoots() const;
-
-    /** Get map of zone path → dirty status (only for sparse-checked-out zones) */
-    const std::map<std::string, ZoneDirtyInfo> & getTectonixDirtyZones() const;
-
     /** Get cached manifest content (thread-safe, lazy-loaded) */
     const std::string & getManifestContent() const;
 
     /** Get cached parsed manifest JSON (thread-safe, lazy-loaded) */
     const nlohmann::json & getManifestJson() const;
 
     /**
-     * Get a zone's store path, handling dirty detection and lazy mounting.
-     *
-     * For clean zones with lazy-trees enabled: mounts accessor lazily
-     * For dirty zones: currently eager-copies from checkout (extension point)
-     * For lazy-trees disabled: eager-copies from git
+     * Get the repo-wide source accessor with dirty overlay.
+     * All file reads go through this single accessor, producing
+     * repo-relative paths for tracking.
+     */
+    ref<SourceAccessor> getRepoAccessor();
+
+    /**
+     * Check whether any dirty (uncommitted) files overlap with the given
+     * tracked paths. A dirty file overlaps if it equals a tracked path or
+     * lives under a tracked directory. A tracked file overlaps if it lives
+     * under a dirty directory.
+     */
+    bool dirtyFilesOverlap(const std::vector<std::string> & trackedPaths) const;
+
+    /**
+     * Lazily mount the repo-wide accessor and return the virtual store path.
+     * All repo reads go through this mount so file accesses are tracked.
+     */
+    StorePath mountRepoAccessor();
+
+    /**
+     * Get a filesystem path for a repo-relative subtree.
+     * Returns a subpath within the mounted repo accessor.
      */
-    StorePath getZoneStorePath(std::string_view zonePath);
+    std::string getRepoSubtreePath(std::string_view repoRelPath);
 
     /**
      * Return a `SourcePath` that refers to `path` in the root

src/libexpr/include/nix/expr/primops.hh

@@ -38,4 +38,10 @@ void prim_exec(EvalState & state, const PosIdx pos, Value ** args, Value & v);
 
 void makePositionThunks(EvalState & state, const PosIdx pos, Value & line, Value & column);
 
+/**
+ * Reconstruct a derivation Value from a .drv store path.
+ */
+void derivationToValue(
+    EvalState & state, const PosIdx pos, const SourcePath & path, const StorePath & storePath, Value & v);
+
 } // namespace nix

src/libexpr/parallel-eval.cc

@@ -115,6 +115,9 @@ std::vector<std::future<void>> Executor::spawn(std::vector<std::pair<work_t, uin
     if (items.empty())
         return {};
 
+    // Capture parent's tracking context so spawned tasks inherit it.
+    auto parentTrackingCtx = currentTrackingContext;
+
     std::vector<std::future<void>> futures;
 
     {
@@ -125,7 +128,16 @@ std::vector<std::future<void>> Executor::spawn(std::vector<std::pair<work_t, uin
             thread_local std::random_device rd;
             thread_local std::uniform_int_distribution<uint64_t> dist(0, 1ULL << 48);
             auto key = (uint64_t(item.second) << 48) | dist(rd);
-            state->queue.emplace(key, Item{.promise = std::move(promise), .work = std::move(item.first)});
+
+            // Wrap work to propagate tracking context into worker thread.
+            auto wrappedWork = [parentTrackingCtx, work = std::move(item.first)]() {
+                auto prev = currentTrackingContext;
+                currentTrackingContext = parentTrackingCtx;
+                work();
+                currentTrackingContext = prev;
+            };
+
+            state->queue.emplace(key, Item{.promise = std::move(promise), .work = std::move(wrappedWork)});
         }
     }