feat(search,preflight): result_grouping parameter — overlay-first ranking when knowledge_base_url set (#150)

CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- **`result_grouping` parameter for search and preflight** — when `knowledge_base_url` is set, overlay (knowledge-base) docs are ranked above baseline docs by default (`"overlay_first"`). Callers can explicitly choose `"merged"` (pure BM25 score order, the previous default), `"overlay_first"` (overlay before baseline, preserving score order within each partition), or `"grouped"` (separate `overlay_hits`/`baseline_hits` arrays in search, `start_here_overlay`/`start_here_baseline` in preflight). Conditional default: `knowledge_base_url` unset → `"merged"` (no behavior change); `knowledge_base_url` set → `"overlay_first"`. Telemetry records the caller-specified value in blob9 (`result_grouping`). Closes #150.
+- **`result_grouping` mirrored in Node CLI (`src/core/actions.js`)** — same conditional default and partition logic for parity with the Cloudflare Worker. CLI uses `origin: "local" | "baseline"` (its existing field) where the worker uses `source: "canon" | "baseline"`. Without this, CLI users would see baseline-contaminated rankings even though the worker is fixed.
+
+### Fixed
+
+- **Candidate-pool widening for grouping** — when `result_grouping !== "merged"`, both worker `runSearch` and CLI `search` now retrieve 50 BM25 candidates instead of 5, partition, then truncate to the response cap of 5. The original implementation truncated to 5 *before* partitioning, which made overlay docs ranked at BM25 position 6+ invisible to the partition logic. Two regression tests added (`partition surfaces overlay even when overlay is mostly low-score`, `widened pool: 50 candidates partition correctly without losing overlay`).
+
 ## [0.26.0] - 2026-04-26
 
 ### Added

src/core/actions.js

@@ -88,6 +88,22 @@ function getBM25Index(docs, baselineSha) {
   return cachedBM25;
 }
 
+// ──────────────────────────────────────────────────────────────────────────────
+// Result grouping — stable partition by origin (local vs baseline). The Node CLI
+// mirror of the worker's partitionBySource (which keys on `source`). See
+// klappy/oddkit issue #150.
+// ──────────────────────────────────────────────────────────────────────────────
+
+function partitionByOrigin(arr) {
+  const overlay = [];
+  const baselineHits = [];
+  for (const h of arr) {
+    if ((h.origin || "local") === "local") overlay.push(h);
+    else baselineHits.push(h);
+  }
+  return { overlay, baseline: baselineHits };
+}
+
 // ──────────────────────────────────────────────────────────────────────────────
 // Response text builders
 // ──────────────────────────────────────────────────────────────────────────────
@@ -160,14 +176,20 @@ export function buildEncodeResponse(taskResult) {
  * @param {string} [params.baseline] - Baseline override (canon_url takes precedence)
  * @param {string} [params.repoRoot] - Repository root (defaults to cwd)
  * @param {Object} [params.state] - Optional state for threading (MCP orchestrator)
+ * @param {"merged"|"overlay_first"|"grouped"} [params.result_grouping] - Search ranking policy (#150)
  * @returns {Object} { action, result, assistant_text, debug, state? }
  */
 export async function handleAction(params) {
-  const { action, input, context, mode, canon_url, state, include_metadata, section, reference, compare } = params;
+  const { action, input, context, mode, canon_url, state, include_metadata, section, reference, compare, result_grouping } = params;
   const repoRoot = params.repoRoot || process.cwd();
   const baseline = canon_url || params.baseline;
   const startMs = Date.now();
 
+  // Issue #150: when an overlay (custom canon_url/baseline) is set, default to
+  // overlay_first so local docs rank above baseline. With no overlay,
+  // every doc has origin: "local" and the partition is a no-op anyway.
+  const resolvedGrouping = result_grouping ?? (baseline ? "overlay_first" : "merged");
+
   // Helper: enrich debug output with baseline SHA for observability
   function makeDebug(extra = {}) {
     return {
@@ -286,26 +308,49 @@ export async function handleAction(params) {
         }
 
         const bm25 = getBM25Index(index.documents, baselineSha);
-        const results = searchBM25(bm25, input, 5);
+
+        // Issue #150 fix-forward: when grouping is active, retrieve a wider
+        // candidate pool from BM25 so overlay (local) docs ranked beyond
+        // position 5 in raw BM25 are not truncated before partitioning.
+        const FINAL_LIMIT = 5;
+        const candidateLimit = resolvedGrouping !== "merged" ? 50 : FINAL_LIMIT;
+        const results = searchBM25(bm25, input, candidateLimit);
 
         const docMap = new Map(index.documents.map((d) => [d.path, d]));
-        const hits = results
+        const rawHits = results
           .map((r) => {
             const doc = docMap.get(r.id);
             if (!doc) return null;
             return { ...doc, score: r.score };
           })
           .filter(Boolean);
 
+        // Apply result_grouping partition. Single forward pass — no re-sort.
+        // After partitioning the wider candidate pool, truncate to FINAL_LIMIT.
+        let hits = rawHits;
+        let isGrouped = false;
+        if (resolvedGrouping === "overlay_first" || resolvedGrouping === "grouped") {
+          const { overlay, baseline: baselineHits } = partitionByOrigin(rawHits);
+          hits = [...overlay, ...baselineHits].slice(0, FINAL_LIMIT);
+          isGrouped = resolvedGrouping === "grouped";
+        } else {
+          hits = rawHits.slice(0, FINAL_LIMIT);
+        }
+
         const updatedState = state ? addCanonRefs(initState(state), hits.map((h) => h.path)) : undefined;
 
         if (hits.length === 0) {
+          const noMatchResult = { status: "NO_MATCH", docs_considered: index.documents.length, hits: [] };
+          if (isGrouped) {
+            noMatchResult.overlay_hits = [];
+            noMatchResult.baseline_hits = [];
+          }
           return {
             action: "search",
-            result: { status: "NO_MATCH", docs_considered: index.documents.length, hits: [] },
+            result: noMatchResult,
             state: updatedState,
             assistant_text: `Searched ${index.documents.length} documents but found no matches for "${input}". Try rephrasing or use action "catalog".`,
-            debug: makeDebug({ search_index_size: bm25.N }),
+            debug: makeDebug({ search_index_size: bm25.N, result_grouping: resolvedGrouping }),
           };
         }
 
@@ -322,43 +367,58 @@ export async function handleAction(params) {
           ...hits.map((r) => `- \`${r.path}\` — ${r.title || "(untitled)"} (score: ${r.score.toFixed(2)})`),
         ];
 
-        return {
-          action: "search",
-          result: {
-            status: "FOUND",
-            hits: hits.map((h) => {
-              const hit = {
-                uri: h.uri,
-                path: h.path,
-                title: h.title,
-                tags: h.tags,
-                score: h.score,
-                snippet: (h.contentPreview || "").slice(0, 200),
-                source: h.origin || "local",
-              };
-              if (include_metadata) {
-                if (h.frontmatter) {
-                  hit.metadata = h.frontmatter;
-                } else if (h.absolutePath && existsSync(h.absolutePath)) {
-                  // Fallback: index predates frontmatter storage — parse from file
-                  try {
-                    const { data } = matter(readFileSync(h.absolutePath, "utf-8"));
-                    if (data && Object.keys(data).length > 0) {
-                      hit.metadata = data;
-                    }
-                  } catch {
-                    // File not readable — omit metadata for this hit
-                  }
+        const hitObjects = hits.map((h) => {
+          const hit = {
+            uri: h.uri,
+            path: h.path,
+            title: h.title,
+            tags: h.tags,
+            score: h.score,
+            snippet: (h.contentPreview || "").slice(0, 200),
+            source: h.origin || "local",
+          };
+          if (include_metadata) {
+            if (h.frontmatter) {
+              hit.metadata = h.frontmatter;
+            } else if (h.absolutePath && existsSync(h.absolutePath)) {
+              // Fallback: index predates frontmatter storage — parse from file
+              try {
+                const { data } = matter(readFileSync(h.absolutePath, "utf-8"));
+                if (data && Object.keys(data).length > 0) {
+                  hit.metadata = data;
                 }
+              } catch {
+                // File not readable — omit metadata for this hit
               }
-              return hit;
-            }),
-            evidence,
-            docs_considered: index.documents.length,
-          },
+            }
+          }
+          return hit;
+        });
+
+        const foundResult = {
+          status: "FOUND",
+          hits: hitObjects,
+          evidence,
+          docs_considered: index.documents.length,
+        };
+
+        if (isGrouped) {
+          const overlayHits = [];
+          const baselineHits = [];
+          for (const h of hitObjects) {
+            if (h.source === "local") overlayHits.push(h);
+            else baselineHits.push(h);
+          }
+          foundResult.overlay_hits = overlayHits;
+          foundResult.baseline_hits = baselineHits;
+        }
+
+        return {
+          action: "search",
+          result: foundResult,
           state: updatedState,
           assistant_text: assistantLines.join("\n").trim(),
-          debug: makeDebug({ search_index_size: bm25.N }),
+          debug: makeDebug({ search_index_size: bm25.N, result_grouping: resolvedGrouping }),
         };
       }
 
@@ -424,6 +484,7 @@ export async function handleAction(params) {
           repoRoot,
           baseline,
           action: "preflight",
+          result_grouping: resolvedGrouping,
         });
         return {
           action: "preflight",

src/mcp/orchestrate.js

@@ -291,6 +291,7 @@ export function detectAction(message) {
  * @param {string} [options.baseline_root] - For instruction_sync: filesystem mode baseline path
  * @param {Object} [options.registry_payload] - For instruction_sync: payload mode registry object
  * @param {Object} [options.state_payload] - For instruction_sync: payload mode state object
+ * @param {"merged"|"overlay_first"|"grouped"} [options.result_grouping] - Ranking policy (#150)
  * @returns {Object} { action, assistant_text, result, debug, suggest_orient }
  */
 export async function runOrchestrate(options) {
@@ -303,6 +304,7 @@ export async function runOrchestrate(options) {
     baseline_root,
     registry_payload,
     state_payload,
+    result_grouping,
   } = options;
 
   // Runtime validation (schema is permissive, runtime enforces)
@@ -409,6 +411,7 @@ export async function runOrchestrate(options) {
           repo: repoRoot || process.cwd(),
           baseline,
           message,
+          result_grouping,
         });
         result.result = taskResult;
         result.assistant_text = buildPreflightAssistantText(taskResult);

src/tasks/preflight.js

@@ -158,10 +158,17 @@ function findPitfallDocs(docs, keywords) {
  * @param {string} options.repo - Repository root path
  * @param {string} options.baseline - Baseline override
  * @param {string} options.message - The preflight message (what the agent is about to do)
+ * @param {"merged"|"overlay_first"|"grouped"} [options.result_grouping] - Ranking policy (#150)
  * @returns {Promise<Object>}
  */
 export async function runPreflight(options) {
-  const { repo: repoRoot, baseline: baselineOverride, message } = options;
+  const { repo: repoRoot, baseline: baselineOverride, message, result_grouping } = options;
+  // Mirror the worker's conditional default (#150 validator F-1): when a
+  // baseline override is set, default to overlay_first; otherwise merged.
+  // src/core/actions.js pre-resolves this before delegating, so the public
+  // CLI/MCP paths are unaffected. This handles direct importers of
+  // runPreflight that bypass that pre-resolution.
+  const resolvedGrouping = result_grouping ?? (baselineOverride ? "overlay_first" : "merged");
 
   // Reuse catalog to get start_here, next_up, canon_by_tag, playbooks
   const catalogResult = await runCatalog({
@@ -191,10 +198,29 @@ export async function runPreflight(options) {
   const dod = findDodDoc(docs);
   const pitfalls = findPitfallDocs(docs, keywords);
 
+  // Apply result_grouping to start_here. Mirror of the worker's runPreflight:
+  // overlay_first/grouped reorder the list so overlay (local) entries come
+  // first; "grouped" additionally exposes start_here_overlay/start_here_baseline.
+  // Origin is read from the index (start_here entries don't carry origin themselves).
+  let startHere = catalogResult.start_here;
+  let overlayEntries = null;
+  let baselineEntries = null;
+  if (resolvedGrouping === "overlay_first" || resolvedGrouping === "grouped") {
+    const originByPath = new Map(docs.map((d) => [d.path, d.origin || "local"]));
+    overlayEntries = [];
+    baselineEntries = [];
+    for (const entry of startHere) {
+      const origin = originByPath.get(entry.path) || "local";
+      if (origin === "local") overlayEntries.push(entry);
+      else baselineEntries.push(entry);
+    }
+    startHere = [...overlayEntries, ...baselineEntries];
+  }
+
   const result = {
     status: "SUPPORTED",
     advisory: false,
-    start_here: catalogResult.start_here,
+    start_here: startHere,
     next_up: catalogResult.next_up,
     canon_by_tag: catalogResult.canon_by_tag,
     playbooks: catalogResult.playbooks,
@@ -212,9 +238,16 @@ export async function runPreflight(options) {
       timestamp: new Date().toISOString(),
       repo_root: repoRoot,
       keywords_extracted: keywords,
+      result_grouping: resolvedGrouping,
     },
   };
 
+  // For "grouped" mode, also expose start_here split by origin (parity with worker).
+  if (resolvedGrouping === "grouped") {
+    result.start_here_overlay = overlayEntries.slice(0, 3).map((e) => e.path);
+    result.start_here_baseline = baselineEntries.slice(0, 3).map((e) => e.path);
+  }
+
   writeLast(result);
   return result;
 }

workers/src/index.ts

@@ -220,6 +220,7 @@ Use when:
         "canon-tier-2", "canon-tier-1", "published-essay",
       ]).optional().describe("Optional mode hint. Epistemic modes (exploration/planning/execution) or writing-lifecycle modes (voice-dump/drafting/peer-review-ready/canon-tier-2/canon-tier-1/published-essay). Sourced from odd/challenge/stakes-calibration."),
       knowledge_base_url: z.string().optional().describe("Optional GitHub repo URL for your knowledge base. When set, strict mode is automatic: missing files fall through to the bundled governance tier rather than silently substituting from the default knowledge base."),
+      result_grouping: z.enum(["merged", "overlay_first", "grouped"]).optional().describe("For action='search' or 'preflight': controls how overlay (knowledge_base) and baseline results are ordered. 'merged' = pure score order (default when knowledge_base_url unset). 'overlay_first' = overlay docs ranked above baseline (default when knowledge_base_url set). 'grouped' = separate overlay_hits/baseline_hits arrays in response."),
       include_metadata: z.boolean().optional().describe("When true, search/get responses include a metadata object with full parsed frontmatter. Default: false."),
       section: z.string().optional().describe("For action='get': extract only the named ## section from the document. Returns section content or available sections if not found."),
       sort_by: z.enum(["date", "path"]).optional().describe("For action='catalog': sort articles. 'date' returns newest first (requires frontmatter). 'path' returns all docs alphabetically, including undated."),
@@ -241,6 +242,7 @@ Use when:
         context: args.context,
         mode: args.mode,
         knowledge_base_url: args.knowledge_base_url,
+        result_grouping: args.result_grouping,
         include_metadata: args.include_metadata,
         section: args.section,
         sort_by: args.sort_by,
@@ -321,6 +323,7 @@ Use when:
       schema: {
         input: z.string().describe("Natural language query or tags to search for."),
         knowledge_base_url: z.string().optional().describe("Optional: GitHub repo URL for your knowledge base. When set, strict mode is automatic: missing files fall through to the bundled governance tier."),
+        result_grouping: z.enum(["merged", "overlay_first", "grouped"]).optional().describe("Controls how overlay (knowledge_base) and baseline results are ordered. 'merged' = pure score order (default when knowledge_base_url unset). 'overlay_first' = overlay docs ranked above baseline (default when knowledge_base_url set). 'grouped' = separate overlay_hits/baseline_hits arrays in response."),
         include_metadata: z.boolean().optional().describe("When true, each hit includes a metadata object with full parsed frontmatter. Default: false."),
       },
       annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
@@ -386,6 +389,7 @@ Use when:
       schema: {
         input: z.string().describe("Description of what you're about to implement."),
         knowledge_base_url: z.string().optional().describe("Optional: GitHub repo URL for your knowledge base. When set, strict mode is automatic: missing files fall through to the bundled governance tier."),
+        result_grouping: z.enum(["merged", "overlay_first", "grouped"]).optional().describe("Controls how overlay (knowledge_base) and baseline start_here results are ordered. 'merged' = pure score order (default when knowledge_base_url unset). 'overlay_first' = overlay docs ranked above baseline (default when knowledge_base_url set). 'grouped' = separate start_here_overlay/start_here_baseline arrays."),
       },
       annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
     },
@@ -432,6 +436,7 @@ Use when:
           context: args.context as string | undefined,
           mode: args.mode as string | undefined,
           knowledge_base_url: args.knowledge_base_url as string | undefined,
+          result_grouping: args.result_grouping as "merged" | "overlay_first" | "grouped" | undefined,
           include_metadata: args.include_metadata as boolean | undefined,
           section: args.section as string | undefined,
           sort_by: args.sort_by as string | undefined,