Mirror of upstream PR #82129

crates/next-api/src/project.rs

@@ -323,14 +323,15 @@ impl ProjectContainer {
                 .await?;
         } else {
             project_fs.invalidate_with_reason(|path| invalidation::Initialize {
-                path: RcStr::from(path),
+                // this path is just used for display purposes
+                path: RcStr::from(path.to_string_lossy()),
             });
         }
         let output_fs = output_fs_operation(project)
             .read_strongly_consistent()
             .await?;
         output_fs.invalidate_with_reason(|path| invalidation::Initialize {
-            path: RcStr::from(path),
+            path: RcStr::from(path.to_string_lossy()),
         });
         Ok(())
     }
@@ -421,13 +422,14 @@ impl ProjectContainer {
                     .await?;
             } else {
                 project_fs.invalidate_with_reason(|path| invalidation::Initialize {
-                    path: RcStr::from(path),
+                    // this path is just used for display purposes
+                    path: RcStr::from(path.to_string_lossy()),
                 });
             }
         }
         if !ReadRef::ptr_eq(&prev_output_fs, &output_fs) {
             prev_output_fs.invalidate_with_reason(|path| invalidation::Initialize {
-                path: RcStr::from(path),
+                path: RcStr::from(path.to_string_lossy()),
             });
         }
 

turbopack/crates/turbo-tasks-fs/src/invalidator_map.rs

@@ -1,4 +1,8 @@
-use std::sync::{LockResult, Mutex, MutexGuard};
+use std::{
+    collections::BTreeMap,
+    path::PathBuf,
+    sync::{LockResult, Mutex, MutexGuard},
+};
 
 use concurrent_queue::ConcurrentQueue;
 use rustc_hash::FxHashMap;
@@ -13,18 +17,18 @@ pub enum WriteContent {
     Link(ReadRef<LinkContent>),
 }
 
-type InnerMap = FxHashMap<String, FxHashMap<Invalidator, Option<WriteContent>>>;
+pub type LockedInvalidatorMap = BTreeMap<PathBuf, FxHashMap<Invalidator, Option<WriteContent>>>;
 
 pub struct InvalidatorMap {
-    queue: ConcurrentQueue<(String, Invalidator, Option<WriteContent>)>,
-    map: Mutex<InnerMap>,
+    queue: ConcurrentQueue<(PathBuf, Invalidator, Option<WriteContent>)>,
+    map: Mutex<LockedInvalidatorMap>,
 }
 
 impl Default for InvalidatorMap {
     fn default() -> Self {
         Self {
             queue: ConcurrentQueue::unbounded(),
-            map: Default::default(),
+            map: Mutex::<LockedInvalidatorMap>::default(),
         }
     }
 }
@@ -34,7 +38,7 @@ impl InvalidatorMap {
         Self::default()
     }
 
-    pub fn lock(&self) -> LockResult<MutexGuard<'_, InnerMap>> {
+    pub fn lock(&self) -> LockResult<MutexGuard<'_, LockedInvalidatorMap>> {
         let mut guard = self.map.lock()?;
         while let Ok((key, value, write_content)) = self.queue.pop() {
             guard.entry(key).or_default().insert(value, write_content);
@@ -44,7 +48,7 @@ impl InvalidatorMap {
 
     pub fn insert(
         &self,
-        key: String,
+        key: PathBuf,
         invalidator: Invalidator,
         write_content: Option<WriteContent>,
     ) {
@@ -66,7 +70,15 @@ impl Serialize for InvalidatorMap {
     where
         S: serde::Serializer,
     {
-        serializer.serialize_newtype_struct("InvalidatorMap", &*self.lock().unwrap())
+        // TODO: This stores absolute `PathBuf`s, which are machine-specific. This should
+        // normalize/denormalize paths relative to the disk filesystem root.
+        //
+        // Potential optimization: We invalidate all fs reads immediately upon resuming from a
+        // persisted cache, but we don't invalidate the fs writes. Those read invalidations trigger
+        // re-inserts into the `InvalidatorMap`. If we knew that certain invalidators were only
+        // needed for reads, we could potentially avoid serializing those paths entirely.
+        let inner: &LockedInvalidatorMap = &self.lock().unwrap();
+        serializer.serialize_newtype_struct("InvalidatorMap", inner)
     }
 }
 

turbopack/crates/turbo-tasks-fs/src/lib.rs

@@ -1,4 +1,5 @@
 #![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this
+#![feature(btree_cursors)] // needed for the `InvalidatorMap` and watcher, reduces time complexity
 #![feature(trivial_bounds)]
 #![feature(min_specialization)]
 #![feature(iter_advance_by)]
@@ -16,13 +17,15 @@ pub mod invalidation;
 mod invalidator_map;
 pub mod json;
 mod mutex_map;
+mod path_map;
 mod read_glob;
 mod retry;
 pub mod rope;
 pub mod source_context;
 pub mod util;
 pub(crate) mod virtual_fs;
 mod watcher;
+
 use std::{
     borrow::Cow,
     cmp::{Ordering, min},
@@ -40,14 +43,10 @@ use anyhow::{Context, Result, anyhow, bail};
 use auto_hash_map::{AutoMap, AutoSet};
 use bitflags::bitflags;
 use dunce::simplified;
-use glob::Glob;
 use indexmap::IndexSet;
-use invalidator_map::InvalidatorMap;
 use jsonc_parser::{ParseOptions, parse_to_serde_value};
 use mime::Mime;
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
-pub use read_glob::ReadGlobResult;
-use read_glob::{read_glob, track_glob};
 use rustc_hash::FxHashSet;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -60,17 +59,19 @@ use turbo_tasks::{
     mark_session_dependent, mark_stateful, trace::TraceRawVcs,
 };
 use turbo_tasks_hash::{DeterministicHash, DeterministicHasher, hash_xxh3_hash64};
-use util::{extract_disk_access, join_path, normalize_path, sys_to_unix, unix_to_sys};
-pub use virtual_fs::VirtualFileSystem;
-use watcher::DiskWatcher;
 
 use self::{invalidation::Write, json::UnparsableJson, mutex_map::MutexMap};
 use crate::{
     attach::AttachedFileSystem,
-    invalidator_map::WriteContent,
+    glob::Glob,
+    invalidator_map::{InvalidatorMap, WriteContent},
+    read_glob::{read_glob, track_glob},
     retry::retry_blocking,
     rope::{Rope, RopeReader},
+    util::{extract_disk_access, join_path, normalize_path, sys_to_unix, unix_to_sys},
+    watcher::DiskWatcher,
 };
+pub use crate::{read_glob::ReadGlobResult, virtual_fs::VirtualFileSystem};
 
 /// A (somewhat arbitrary) filename limit that we should try to keep output file names below.
 ///
@@ -254,10 +255,11 @@ impl DiskFileSystemInner {
     fn register_read_invalidator(&self, path: &Path) -> Result<()> {
         let invalidator = turbo_tasks::get_invalidator();
         self.invalidator_map
-            .insert(path_to_key(path), invalidator, None);
-        #[cfg(not(any(target_os = "macos", target_os = "windows")))]
-        if let Some(dir) = path.parent() {
-            self.watcher.ensure_watching(dir, self.root_path())?;
+            .insert(path.to_owned(), invalidator, None);
+        if let Some(non_recursive) = &self.watcher.non_recursive_state
+            && let Some(dir) = path.parent()
+        {
+            non_recursive.ensure_watching(&self.watcher, dir, self.root_path())?;
         }
         Ok(())
     }
@@ -272,7 +274,7 @@ impl DiskFileSystemInner {
         write_content: WriteContent,
     ) -> Result<Vec<(Invalidator, Option<WriteContent>)>> {
         let mut invalidator_map = self.invalidator_map.lock().unwrap();
-        let invalidators = invalidator_map.entry(path_to_key(path)).or_default();
+        let invalidators = invalidator_map.entry(path.to_owned()).or_default();
         let old_invalidators = invalidators
             .extract_if(|i, old_write_content| {
                 i == &invalidator
@@ -284,9 +286,10 @@ impl DiskFileSystemInner {
             .collect::<Vec<_>>();
         invalidators.insert(invalidator, Some(write_content));
         drop(invalidator_map);
-        #[cfg(not(any(target_os = "macos", target_os = "windows")))]
-        if let Some(dir) = path.parent() {
-            self.watcher.ensure_watching(dir, self.root_path())?;
+        if let Some(non_recursive) = &self.watcher.non_recursive_state
+            && let Some(dir) = path.parent()
+        {
+            non_recursive.ensure_watching(&self.watcher, dir, self.root_path())?;
         }
         Ok(old_invalidators)
     }
@@ -296,9 +299,10 @@ impl DiskFileSystemInner {
     fn register_dir_invalidator(&self, path: &Path) -> Result<()> {
         let invalidator = turbo_tasks::get_invalidator();
         self.dir_invalidator_map
-            .insert(path_to_key(path), invalidator, None);
-        #[cfg(not(any(target_os = "macos", target_os = "windows")))]
-        self.watcher.ensure_watching(path, self.root_path())?;
+            .insert(path.to_owned(), invalidator, None);
+        if let Some(non_recursive) = &self.watcher.non_recursive_state {
+            non_recursive.ensure_watching(&self.watcher, path, self.root_path())?;
+        }
         Ok(())
     }
 
@@ -330,7 +334,7 @@ impl DiskFileSystemInner {
     /// Calls the given
     fn invalidate_with_reason<R: InvalidationReason + Clone>(
         &self,
-        reason: impl Fn(String) -> R + Sync,
+        reason: impl Fn(&Path) -> R + Sync,
     ) {
         let _span = tracing::info_span!("invalidate filesystem", name = &*self.root).entered();
         let span = tracing::Span::current();
@@ -342,7 +346,7 @@ impl DiskFileSystemInner {
             .chain(dir_invalidator_map.into_par_iter())
             .flat_map(|(path, invalidators)| {
                 let _span = span.clone().entered();
-                let reason_for_path = reason(path);
+                let reason_for_path = reason(&path);
                 invalidators
                     .into_par_iter()
                     .map(move |i| (reason_for_path.clone(), i))
@@ -446,7 +450,7 @@ impl DiskFileSystem {
 
     pub fn invalidate_with_reason<R: InvalidationReason + Clone>(
         &self,
-        reason: impl Fn(String) -> R + Sync,
+        reason: impl Fn(&Path) -> R + Sync,
     ) {
         self.inner.invalidate_with_reason(reason);
     }
@@ -501,10 +505,6 @@ fn format_absolute_fs_path(path: &Path, name: &str, root_path: &Path) -> Option<
     }
 }
 
-pub fn path_to_key(path: impl AsRef<Path>) -> String {
-    path.as_ref().to_string_lossy().to_string()
-}
-
 #[turbo_tasks::value_impl]
 impl DiskFileSystem {
     /// Create a new instance of `DiskFileSystem`.
@@ -753,11 +753,12 @@ impl FileSystem for DiskFileSystem {
                 .await?;
             if compare == FileComparison::Equal {
                 if !old_invalidators.is_empty() {
-                    let key = path_to_key(&full_path);
                     for (invalidator, write_content) in old_invalidators {
-                        inner
-                            .invalidator_map
-                            .insert(key.clone(), invalidator, write_content);
+                        inner.invalidator_map.insert(
+                            full_path.clone().into_owned(),
+                            invalidator,
+                            write_content,
+                        );
                     }
                 }
                 return Ok(());
@@ -896,11 +897,12 @@ impl FileSystem for DiskFileSystem {
             };
             if is_equal {
                 if !old_invalidators.is_empty() {
-                    let key = path_to_key(&full_path);
                     for (invalidator, write_content) in old_invalidators {
-                        inner
-                            .invalidator_map
-                            .insert(key.clone(), invalidator, write_content);
+                        inner.invalidator_map.insert(
+                            full_path.clone().into_owned(),
+                            invalidator,
+                            write_content,
+                        );
                     }
                 }
                 return Ok(());

turbopack/crates/turbo-tasks-fs/src/path_map.rs

@@ -0,0 +1,80 @@
+use std::{
+    collections::{BTreeMap, btree_map::CursorMut},
+    ops::Bound,
+    path::{Path, PathBuf},
+};
+
+/// A thin wrapper around [`BTreeMap<PathBuf, V>`] that provides efficient extraction of child
+/// paths.
+///
+/// In the future, this may use a more efficient representation, like a radix tree or trie.
+pub trait OrderedPathMapExt<V> {
+    fn extract_path_with_children<'a>(&'a mut self, path: &'a Path) -> ExtractWithChildren<'a, V>;
+}
+
+impl<V> OrderedPathMapExt<V> for BTreeMap<PathBuf, V> {
+    fn extract_path_with_children<'a>(&'a mut self, path: &'a Path) -> ExtractWithChildren<'a, V> {
+        ExtractWithChildren {
+            cursor: self.lower_bound_mut(Bound::Included(path)),
+            parent_path: path,
+        }
+    }
+}
+
+pub struct ExtractWithChildren<'a, V> {
+    cursor: CursorMut<'a, PathBuf, V>,
+    parent_path: &'a Path,
+}
+
+impl<V> Iterator for ExtractWithChildren<'_, V> {
+    type Item = (PathBuf, V);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // this simple implementation works because `Path` implements `Ord` (and `starts_with`)
+        // using path component comparision, rather than raw byte comparisions. The parent path is
+        // always guaranteed to be placed immediately before its children (pre-order traversal).
+        if self
+            .cursor
+            .peek_next()
+            .is_none_or(|(k, _v)| !k.starts_with(self.parent_path))
+        {
+            return None;
+        }
+        self.cursor.remove_next()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_extract_with_children() {
+        let mut map = BTreeMap::default();
+        map.insert(PathBuf::from("a"), 1);
+        map.insert(PathBuf::from("a/b"), 2);
+        map.insert(PathBuf::from("a/b/c"), 3);
+        map.insert(PathBuf::from("a/b/d"), 4);
+        map.insert(PathBuf::from("a/c"), 5);
+        map.insert(PathBuf::from("x/y/z"), 6);
+        map.insert(PathBuf::from("z/a/b"), 7);
+
+        let parent_path = PathBuf::from("a/b");
+        let extracted: Vec<_> = map.extract_path_with_children(&parent_path).collect();
+
+        let expected_extracted = vec![
+            (PathBuf::from("a/b"), 2),
+            (PathBuf::from("a/b/c"), 3),
+            (PathBuf::from("a/b/d"), 4),
+        ];
+        assert_eq!(extracted, expected_extracted);
+
+        let mut expected_remaining = BTreeMap::new();
+        expected_remaining.insert(PathBuf::from("a"), 1);
+        expected_remaining.insert(PathBuf::from("a/c"), 5);
+        expected_remaining.insert(PathBuf::from("x/y/z"), 6);
+        expected_remaining.insert(PathBuf::from("z/a/b"), 7);
+
+        assert_eq!(map, expected_remaining);
+    }
+}