Add worker-backed source scanning

docs/config.md

@@ -10,6 +10,7 @@ Create `.sniffler/config.json` in the project root:
 
 ```json
 {
+  "workers": "auto",
   "source": {
     "roots": ["apps", "packages"],
     "ignore": ["**/*.test.*", "**/*.spec.*", "**/__tests__/**"]
@@ -87,6 +88,7 @@ Every property is optional. Missing properties are filled from the defaults belo
 
 ```ts
 type SnifflerConfig = {
+  workers?: "auto" | number;
   source?: {
     roots?: string[];
     extensions?: string[];
@@ -127,6 +129,34 @@ type SnifflerConfig = {
 
 The config file may also include a string `$schema` property.
 
+### `workers`
+
+Controls source scanning parallelism for cache misses.
+
+Default:
+
+```json
+"auto"
+```
+
+Accepted values:
+
+```json
+"auto"
+```
+
+or any non-negative integer:
+
+```json
+0
+```
+
+```json
+4
+```
+
+`0` keeps scanning serial. Any value above `0` enables worker-based scanning when Sniffler chooses the worker path. Worker scanning currently requires the Node file system implementation.
+
 `platform` is intentionally not a config property. It is a runtime option passed to `sniffler impact` or `sniffler run`, so the same project config can be reused for `android`, `ios`, web, or default resolution runs.
 
 ## Properties

src/config/config-schema.ts

@@ -1,4 +1,5 @@
 export type SnifflerConfig = {
+  workers?: "auto" | number;
   source?: {
     roots?: ReadonlyArray<string>;
     extensions?: ReadonlyArray<string>;
@@ -45,6 +46,7 @@ export type SnifflerConfigFile = SnifflerConfig & {
 export const defaultConfigPath = ".sniffler/config.json";
 
 export const defaultConfig = {
+  workers: "auto",
   source: {
     roots: ["."],
     extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"],
@@ -82,3 +84,4 @@ export const defaultConfig = {
 export type SnifflerWorkspaceStrategy = "package-json" | "pnpm-workspace";
 export type SnifflerOutputFormat = "text" | "json";
 export type SnifflerCacheStaleStrategy = "content" | "metadata";
+export type SnifflerWorkersSetting = "auto" | number;

src/config/load-config.ts

@@ -6,6 +6,7 @@ import {
   type SnifflerCacheStaleStrategy,
   type SnifflerConfigFile,
   type SnifflerOutputFormat,
+  type SnifflerWorkersSetting,
   type SnifflerWorkspaceStrategy
 } from "./config-schema.js";
 
@@ -91,6 +92,14 @@ const isCacheStaleStrategy = (value: unknown): value is SnifflerCacheStaleStrate
   return value === "content" || value === "metadata";
 };
 
+const isWorkersSetting = (value: unknown): value is SnifflerWorkersSetting => {
+  if (value === "auto") {
+    return true;
+  }
+
+  return typeof value === "number" && Number.isInteger(value) && value >= 0;
+};
+
 const validateConfig = (value: unknown, path: string): SnifflerConfigFile => {
   if (!isRecord(value)) {
     throw createLoadError(
@@ -158,6 +167,14 @@ const validateConfig = (value: unknown, path: string): SnifflerConfigFile => {
     }
   }
 
+  if ("workers" in value && value.workers !== undefined && !isWorkersSetting(value.workers)) {
+    throw createLoadError(
+      "SNIFFLER_INVALID_CONFIG",
+      path,
+      `Invalid config in ${path}: workers must be "auto" or a non-negative integer.`
+    );
+  }
+
   if ("graph" in value && value.graph !== undefined) {
     if (!isRecord(value.graph)) {
       throw createLoadError(
@@ -357,6 +374,7 @@ const validateConfig = (value: unknown, path: string): SnifflerConfigFile => {
 
 const normalizeConfig = (config: SnifflerConfigFile): SnifflerConfig => {
   return {
+    workers: config.workers ?? defaultConfig.workers,
     source: {
       roots: config.source?.roots ?? defaultConfig.source?.roots,
       extensions: config.source?.extensions ?? defaultConfig.source?.extensions,

src/filesystem/filesystem.ts

@@ -19,6 +19,7 @@ export type SnifflerInvalidJsonError = Error & {
 };
 
 export type FileSystem = {
+  supportsWorkerScanning: boolean;
   readFile: (path: string) => Promise<string>;
   readFileBuffer?: (path: string) => Promise<Uint8Array>;
   readJson: <T>(path: string) => Promise<T>;

src/filesystem/memory-filesystem.ts

@@ -96,6 +96,7 @@ export const createMemoryFileSystem = (entries: Record<string, string> = {}): Me
   };
 
   return {
+    supportsWorkerScanning: false,
     seed,
     readFile: readFileInternal,
     readFileBuffer: async (path: string) => {

src/filesystem/node-filesystem.ts

@@ -41,6 +41,7 @@ export const createNodeFileSystem = (): FileSystem => {
   };
 
   return {
+    supportsWorkerScanning: true,
     readFile: readFileInternal,
     readFileBuffer: async (path: string) => {
       return readFile(path);

src/filesystem/path-utils.js

@@ -0,0 +1,53 @@
+import { posix as pathPosix } from "node:path";
+export const normalizePath = (path) => {
+    return pathPosix.normalize(path.replaceAll("\\", "/"));
+};
+export const parentDirectories = (path) => {
+    const normalized = normalizePath(path);
+    const parts = normalized.split("/").filter((part) => part.length > 0);
+    const parents = [];
+    for (let index = 1; index < parts.length; index += 1) {
+        parents.push(parts.slice(0, index).join("/"));
+    }
+    return parents;
+};
+export const isPathWithinDirectory = (path, directory) => {
+    const normalizedPath = normalizePath(path);
+    const normalizedDirectory = normalizePath(directory);
+    return (normalizedPath === normalizedDirectory ||
+        normalizedPath.startsWith(`${normalizedDirectory}/`));
+};
+const escapeRegExp = (value) => {
+    return value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
+};
+export const createGlobMatcher = (pattern) => {
+    const normalizedPattern = normalizePath(pattern);
+    let regex = "^";
+    for (let index = 0; index < normalizedPattern.length; index += 1) {
+        const char = normalizedPattern[index];
+        if (char === "*") {
+            if (normalizedPattern[index + 1] === "*") {
+                if (normalizedPattern[index + 2] === "/") {
+                    regex += "(?:.*/)?";
+                    index += 2;
+                }
+                else {
+                    regex += ".*";
+                    index += 1;
+                }
+            }
+            else {
+                regex += "[^/]*";
+            }
+            continue;
+        }
+        if (char === "?") {
+            regex += "[^/]";
+            continue;
+        }
+        regex += escapeRegExp(char);
+    }
+    regex += "$";
+    const matcher = new RegExp(regex);
+    return (path) => matcher.test(normalizePath(path));
+};

src/graph/build-graph.ts

@@ -63,6 +63,7 @@ export const buildGraph = async (
   input: BuildGraphInput = {}
 ): Promise<DependencyGraph> => {
   const fallbackFileSystem: FileSystem = {
+    supportsWorkerScanning: false,
     readFile: async (path: string) => {
       throw new Error(`File not found: ${path}`);
     },

src/impact/cache-lifecycle.ts

@@ -1,16 +1,15 @@
-import { createHash } from "node:crypto";
 import { join } from "node:path";
 import type { GraphCache, ResolvedEdge, CacheEntry } from "../cache/cache-types.js";
 import { createGraphCacheStore, type GraphCacheStore } from "../cache/cache-store.js";
 import { getCacheConfigHash, SCANNER_VERSION } from "../cache/cache-key.js";
 import { loadCache } from "../cache/load-cache.js";
 import { saveCache } from "../cache/save-cache.js";
-import { readSourceFileMetadata, type StaleChecker } from "../cache/stale-checker.js";
+import type { StaleChecker } from "../cache/stale-checker.js";
 import type { SnifflerConfig } from "../config/config-schema.js";
 import { normalizePath } from "../filesystem/path-utils.js";
 import type { FileSystem } from "../filesystem/filesystem.js";
 import type { GraphNode, DependencyGraph } from "../graph/build-graph.js";
-import { scanFileText } from "../scanner/scan-file.js";
+import { resolveSourceScanner } from "../scanner/source-scanner.js";
 import type { Diagnostics } from "../diagnostics/diagnostics.js";
 
 export type ImpactCacheScanState = {
@@ -31,10 +30,7 @@ export type ImpactCacheLifecycleInput = {
   staleChecker: StaleChecker;
   cacheStoreFactory?: (input: { cache: GraphCache | null; staleChecker: StaleChecker }) => GraphCacheStore;
   platform?: string;
-};
-
-const hashText = (text: string): string => {
-  return createHash("sha256").update(Buffer.from(text, "utf8")).digest("hex");
+  sourceFiles: ReadonlyArray<string>;
 };
 
 const loadCacheForImpact = async (input: ImpactCacheLifecycleInput): Promise<{
@@ -70,6 +66,8 @@ const createGraphCache = async (input: {
   cacheStoreFactory?: (input: { cache: GraphCache | null; staleChecker: StaleChecker }) => GraphCacheStore;
   diagnostics: Diagnostics;
   sourceFiles: ReadonlyArray<string>;
+  cwd: string;
+  workers?: "auto" | number;
 }): Promise<ImpactCacheScanState> => {
   const warnings: string[] = [];
   const graphNodes: GraphNode[] = [];
@@ -82,6 +80,11 @@ const createGraphCache = async (input: {
     input.cacheStoreFactory?.({ cache: input.cache, staleChecker: input.staleChecker }) ??
     createGraphCacheStore(input.cache, input.staleChecker);
   const contentHashes = new Map<string, string>();
+  const sourceFileStates: Array<{
+    path: string;
+    cacheEntry: CacheEntry | null;
+  }> = [];
+  const missPaths: string[] = [];
   let cacheNeedsRefresh = input.cache === null || !canReuseCachedResolvedEdges;
   let cacheScanHits = 0;
   let cacheScanMisses = 0;
@@ -90,28 +93,61 @@ const createGraphCache = async (input: {
   await input.diagnostics.time("impact.sources.scan", async () => {
     for (const path of input.sourceFiles) {
       const cacheEntry = await cacheStore.getEntry(path);
-      const canReuseCachedEntry = cacheEntry !== null;
-      let scan: CacheEntry["scan"];
-      let contentHash: string;
+      sourceFileStates.push({
+        path,
+        cacheEntry
+      });
 
       if (cacheEntry !== null) {
-        scan = cacheEntry.scan;
-        contentHash = cacheEntry.contentHash;
-      } else {
-        const text = await input.fs.readFile(path);
-        scan = scanFileText({ filePath: path, text });
-        contentHash = hashText(text);
-      }
-
-      const metadata = cacheEntry === null ? await readSourceFileMetadata(input.fs, path) : cacheEntry.metadata;
-      contentHashes.set(path, contentHash);
-
-      if (canReuseCachedEntry) {
         cacheScanHits += 1;
       } else {
         cacheScanMisses += 1;
         cacheNeedsRefresh = true;
+        missPaths.push(path);
       }
+    }
+
+    const sourceScanner = resolveSourceScanner({
+      fs: input.fs,
+      cwd: input.cwd,
+      workers: input.workers,
+      missCount: missPaths.length
+    });
+    input.diagnostics.record("sourceScannerMode", sourceScanner.mode);
+    input.diagnostics.record("sourceScannerWorkers", sourceScanner.workers);
+    input.diagnostics.record("sourceScannerJobs", missPaths.length);
+    input.diagnostics.record("sourceScannerWorkerFailures", 0);
+
+    let missResults: ReadonlyArray<{
+      path: string;
+      scan: CacheEntry["scan"];
+      contentHash: string;
+      metadata?: CacheEntry["metadata"];
+    }> = [];
+
+    try {
+      missResults = missPaths.length === 0 ? [] : await sourceScanner.scan(missPaths);
+    } catch (error) {
+      if (sourceScanner.mode === "worker") {
+        input.diagnostics.increment("sourceScannerWorkerFailures");
+      }
+
+      throw error;
+    }
+
+    const missResultsByPath = new Map(missResults.map((entry) => [entry.path, entry] as const));
+
+    for (const { path, cacheEntry } of sourceFileStates) {
+      const canReuseCachedEntry = cacheEntry !== null;
+      const scan = cacheEntry?.scan ?? missResultsByPath.get(path)?.scan;
+      const contentHash = cacheEntry?.contentHash ?? missResultsByPath.get(path)?.contentHash;
+      const metadata = cacheEntry?.metadata ?? missResultsByPath.get(path)?.metadata;
+
+      if (scan === undefined || contentHash === undefined) {
+        throw new Error(`Missing scan result for ${path}`);
+      }
+
+      contentHashes.set(path, contentHash);
 
       if (canReuseCachedResolvedEdges && canReuseCachedEntry) {
         cachedResolvedEdgeFiles += 1;
@@ -167,9 +203,7 @@ const createGraphCache = async (input: {
   };
 };
 
-export const prepareImpactCacheState = async (input: ImpactCacheLifecycleInput & {
-  sourceFiles: ReadonlyArray<string>;
-}): Promise<ImpactCacheScanState> => {
+export const prepareImpactCacheState = async (input: ImpactCacheLifecycleInput): Promise<ImpactCacheScanState> => {
   const loaded = await loadCacheForImpact(input);
 
   return await createGraphCache({
@@ -180,7 +214,9 @@ export const prepareImpactCacheState = async (input: ImpactCacheLifecycleInput &
     staleChecker: input.staleChecker,
     cacheStoreFactory: input.cacheStoreFactory,
     diagnostics: input.diagnostics,
-    sourceFiles: input.sourceFiles
+    sourceFiles: input.sourceFiles,
+    cwd: input.cwd,
+    workers: input.config.workers
   });
 };
 

src/impact/impact-command.ts

@@ -104,6 +104,7 @@ export const selectImpact = async (
     return await resolveChangedFiles(input, deps, cwd);
   });
   const runAllReasons = resolveRunAllReasons(changedFiles, config.tests?.runAllWhenChanged ?? []);
+
   if (runAllReasons.length > 0) {
     const runAllSelection = await diagnostics.time("impact.testMap.load", async () => {
       const testMap = await loadTestMap(fs, testMapPath);