perf: replace fuzzy-matcher with nucleo-matcher for improved performance

pylight/Cargo.toml

@@ -30,7 +30,7 @@ ruff_python_parser = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1"
 ruff_python_ast = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" }
 ruff_text_size = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" }
 ruff_source_file = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" }
-fuzzy-matcher = "0.3"
+nucleo-matcher = "0.3.1"
 bincode = "1.3"
 flate2 = "1.0"
 fluent-uri = "0.3"

pylight/README.md

@@ -27,6 +27,19 @@ pylight/
 └── benches/            # Performance benchmarks
 ```
 
+## Build & Test
+
+```bash
+# Build in release mode
+cargo build --release
+
+# Run tests
+cargo test
+
+# Run benchmarks
+cargo bench
+```
+
 ## Usage
 
 ### As an LSP Server
@@ -43,40 +56,28 @@ pylight
 pylight --standalone --directory /path/to/project --query "test"
 ```
 
-## Building
+## Local testing tool
 
-```bash
-# Build in release mode
-cargo build --release
+There is tool to help with development that opens a web page where you can try the symbol search outside of VSCode
 
-# Run tests
-cargo test
+Run this and open the webpage, enter the path to python code:
 
-# Run benchmarks
-cargo bench
+```
+cargo run --release --bin pylight_devtools
 ```
 
 ## Integration with VSCode
 
-The `pylight` LSP server is designed to work with the `pydance` VSCode extension. 
+The `pylight` LSP server is designed to work with the `pydance` VSCode extension.
 The extension will automatically start the language server when opening Python files.
 
-## Performance
-
-- Simple function parsing: ~7.7µs
-- Complex file parsing: ~72µs  
-- Scales linearly with file size
-- Efficient parallel processing for large codebases
-
 ## Development
 
-This project uses test-driven development:
-
-1. Write integration tests first (`tests/integration/`)
+1. Write integration tests (`tests/integration/`)
 2. Write unit tests for components (`src/*/tests.rs`)
 3. Implement functionality to pass tests
 4. Benchmark critical paths (`benches/`)
 
 ## License
 
-MIT
+MIT

pylight/src/bin/pylight_devtools.rs

@@ -29,6 +29,7 @@ fn default_parser() -> String {
 }
 
 #[derive(Serialize, Deserialize)]
+#[allow(dead_code)]
 struct SearchRequest {
     query: String,
 }
@@ -112,14 +113,18 @@ fn main() {
                 );
 
                 let result = spawn_pylight(&index_req.path, &index_req.parser, pylight.clone());
-                let response = if result.is_ok() {
-                    info!("Successfully spawned pylight for {}", index_req.path);
-                    Response::from_string(json!({"status": "success"}).to_string())
-                } else {
-                    let err = result.unwrap_err();
-                    error!("Failed to spawn pylight: {}", err);
-                    Response::from_string(json!({"status": "error", "message": err}).to_string())
+                let response = match result {
+                    Ok(()) => {
+                        info!("Successfully spawned pylight for {}", index_req.path);
+                        Response::from_string(json!({"status": "success"}).to_string())
+                    }
+                    Err(err) => {
+                        error!("Failed to spawn pylight: {}", err);
+                        Response::from_string(
+                            json!({"status": "error", "message": err}).to_string(),
+                        )
                         .with_status_code(500)
+                    }
                 };
                 request
                     .respond(

pylight/src/search.rs

@@ -1,12 +1,12 @@
 //! Symbol search functionality
 
 use crate::symbols::Symbol;
-use fuzzy_matcher::skim::SkimMatcherV2;
-use fuzzy_matcher::FuzzyMatcher;
+use nucleo_matcher::pattern::{Atom, CaseMatching, Normalization};
+use nucleo_matcher::{Config, Matcher, Utf32Str};
 use std::sync::Arc;
 
 pub struct SearchEngine {
-    matcher: SkimMatcherV2,
+    matcher: Matcher,
 }
 
 #[derive(Debug)]
@@ -17,8 +17,10 @@ pub struct SearchResult {
 
 impl SearchEngine {
     pub fn new() -> Self {
+        let mut config = Config::DEFAULT;
+        config.normalize = true;
         Self {
-            matcher: SkimMatcherV2::default().smart_case().use_cache(true),
+            matcher: Matcher::new(config),
         }
     }
 
@@ -37,15 +39,23 @@ impl SearchEngine {
 
         let start_time = std::time::Instant::now();
 
+        // Create the search pattern with smart case matching
+        let pattern = Atom::parse(query, CaseMatching::Smart, Normalization::Smart);
+
+        // Create a matcher instance for scoring
+        let mut matcher = self.matcher.clone();
+
         // First pass: collect fuzzy match results
         let mut results: Vec<SearchResult> = symbols
             .iter()
             .filter_map(|symbol| {
-                self.matcher
-                    .fuzzy_match(&symbol.name, query)
+                let mut buf = Vec::new();
+                let haystack = Utf32Str::new(&symbol.name, &mut buf);
+                pattern
+                    .score(haystack, &mut matcher)
                     .map(|score| SearchResult {
                         symbol: Arc::clone(symbol),
-                        score,
+                        score: score as i64,
                     })
             })
             .collect();