Skip to content

Commit 5c51baa

Browse files
HumanBean17claude
andcommitted
serialize lance optimize to fix reprocess commit-conflict race
cocoindex 1.0.7 schedules table.optimize() (a LanceDB Rewrite transaction) as a background asyncio task that races concurrent table.delete() (Delete) transactions, which LanceDB rejects (upstream lancedb#1504), flooding reprocess stderr with "Retryable commit conflict ... preempted by concurrent transaction Delete" and leaving tables un-optimized. - Disable the in-flow background optimize by setting num_transactions_before_optimize=10**12 on all three mount_table_target calls in java_index_flow_lancedb.py (optimize is pure maintenance; upsert/ delete correctness via merge_insert does not depend on it). - Add java_codebase_rag/lance_optimize.py with a serialized optimize_lance_tables() helper that runs table.optimize() once per table after the flow returns (no concurrent writers), with retry + exponential backoff on the residual commit-conflict. LANCE_TABLE_NAMES becomes the single source of truth, imported by the flow. - Wire the post-flow optimize into both cocoindex chokepoints: pipeline.run_cocoindex_update (used by init/increment/reprocess --vectors-only) and server.run_refresh_pipeline (default reprocess). An optimize failure is surfaced via stderr / the new RefreshIndexOutput optimize_error field / message; success is never flipped. No schema/ontology bump; no re-index-required callout. Co-Authored-By: Claude <noreply@anthropic.com>
1 parent 59de211 commit 5c51baa

7 files changed

Lines changed: 423 additions & 4 deletions

File tree

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
"""Serialized post-flow LanceDB optimize with commit-conflict retry.
2+
3+
cocoindex 1.0.7 schedules ``table.optimize()`` (a LanceDB **Rewrite**/compaction
4+
transaction) as a *background* ``asyncio`` task that races concurrent
5+
``table.delete()`` (**Delete**) transactions emitted by later mutation batches.
6+
LanceDB does not allow a Rewrite to commit concurrently with a Delete
7+
(upstream lancedb#1504 — "We do not support concurrent deletes right now"),
8+
which surfaces as a flood of::
9+
10+
RuntimeError: lance error: Retryable commit conflict for version N: \
11+
This Rewrite transaction was preempted by concurrent transaction Delete ...
12+
13+
To eliminate the race, the flow (``java_index_flow_lancedb.py``) disables the
14+
in-flight background optimize entirely by raising
15+
``num_transactions_before_optimize`` to a value that is effectively never
16+
reached. This module then performs a *single*, serialized optimize after the
17+
flow returns (exit 0 → no concurrent writers), retrying the rare residual
18+
commit conflict that two internal compaction passes can still produce.
19+
"""
20+
from __future__ import annotations
21+
22+
import asyncio
23+
import sys
24+
from pathlib import Path
25+
26+
# Single source of truth for the three Lance table names created by the flow.
27+
# Keep in sync with ``search_lancedb.TABLES`` (the values there mirror these).
28+
LANCE_TABLE_NAMES: tuple[str, ...] = (
29+
"javacodeindex_java_code",
30+
"sqlschemaindex_sql_schema",
31+
"yamlconfigindex_yaml_config",
32+
)
33+
34+
# Commit conflicts are transient; a handful of exponential-backoff retries is
35+
# enough because, post-flow, there are no concurrent writers — only successive
36+
# optimize/compaction passes within this single serialized call can still
37+
# transiently preempt one another.
38+
_MAX_ATTEMPTS = 6
39+
_BASE_BACKOFF_S = 0.1
40+
41+
# Substrings identifying the retryable Lance commit-conflict error. LanceDB
42+
# wraps the underlying lance error text into the raised ``RuntimeError`` str,
43+
# so a substring match is the robust detector (no dedicated exception type).
44+
_RETRYABLE_MARKERS = (
45+
"Retryable commit conflict",
46+
"preempted by concurrent transaction",
47+
)
48+
49+
50+
def _is_retryable(exc: BaseException) -> bool:
51+
text = str(exc)
52+
return any(marker in text for marker in _RETRYABLE_MARKERS)
53+
54+
55+
async def _list_table_names(db: object) -> set[str]:
56+
"""Existing table names across LanceDB API variants (``list_tables`` ≥ ``table_names``)."""
57+
if hasattr(db, "list_tables"):
58+
response = await db.list_tables()
59+
return set(getattr(response, "tables", response))
60+
return set(await db.table_names())
61+
62+
63+
async def optimize_lance_tables(index_dir: Path, *, quiet: bool = False) -> dict[str, str]:
64+
"""Optimize all known Lance tables under *index_dir*, serially, with retry.
65+
66+
Runs ``table.optimize()`` for each name in :data:`LANCE_TABLE_NAMES` that
67+
exists in the DB. Retryable commit conflicts are retried with exponential
68+
backoff; any other exception (or an exhausted retry budget) is captured
69+
per-table in the returned dict and logged to **stderr** — never stdout,
70+
since this is callable from stdio-MCP / JSON-stdout contexts.
71+
72+
Args:
73+
index_dir: directory holding the Lance tables (the flow's LanceDB URI).
74+
quiet: when True, suppress the per-table success/skip info lines on
75+
stderr (errors are always logged).
76+
77+
Returns:
78+
Mapping of table name → status. Values are ``"ok"``, ``"skipped"``
79+
(table absent — e.g. a repo with no SQL/YAML), or ``"error: <text>"``.
80+
"""
81+
# Lazy import: the flow imports this module for LANCE_TABLE_NAMES and must
82+
# not pay the lancedb import cost at flow-definition time.
83+
import lancedb
84+
85+
results: dict[str, str] = {}
86+
db = await lancedb.connect_async(str(index_dir))
87+
try:
88+
try:
89+
existing = await _list_table_names(db)
90+
except Exception as exc:
91+
print(
92+
f"java-codebase-rag: optimize: failed to list tables in "
93+
f"{index_dir}: {exc}",
94+
file=sys.stderr,
95+
)
96+
return {name: f"error: list failed: {exc}" for name in LANCE_TABLE_NAMES}
97+
98+
for name in LANCE_TABLE_NAMES:
99+
if name not in existing:
100+
results[name] = "skipped"
101+
if not quiet:
102+
print(
103+
f"java-codebase-rag: optimize: {name} absent, skipped",
104+
file=sys.stderr,
105+
)
106+
continue
107+
try:
108+
table = await db.open_table(name)
109+
except Exception as exc:
110+
results[name] = f"error: open failed: {exc}"
111+
print(
112+
f"java-codebase-rag: optimize: {name} open failed: {exc}",
113+
file=sys.stderr,
114+
)
115+
continue
116+
117+
last_exc: BaseException | None = None
118+
for attempt in range(_MAX_ATTEMPTS):
119+
try:
120+
await table.optimize()
121+
last_exc = None
122+
break
123+
except Exception as exc:
124+
last_exc = exc
125+
if _is_retryable(exc) and attempt < _MAX_ATTEMPTS - 1:
126+
await asyncio.sleep(_BASE_BACKOFF_S * (2**attempt))
127+
continue
128+
# Non-retryable, or retries exhausted: stop the loop and
129+
# surface below — do not swallow silently.
130+
break
131+
132+
if last_exc is None:
133+
results[name] = "ok"
134+
if not quiet:
135+
print(
136+
f"java-codebase-rag: optimize: {name} ok",
137+
file=sys.stderr,
138+
)
139+
else:
140+
results[name] = f"error: {last_exc}"
141+
print(
142+
f"java-codebase-rag: optimize: {name} failed: {last_exc}",
143+
file=sys.stderr,
144+
)
145+
finally:
146+
# ``AsyncConnection.close`` is a *sync* method in lancedb 0.30.x.
147+
db.close()
148+
return results

java_codebase_rag/pipeline.py

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""Subprocess helpers for cocoindex + graph builder (no heavy ML imports at import time)."""
22
from __future__ import annotations
33

4+
import asyncio
45
import os
56
import shutil
67
import subprocess
@@ -111,6 +112,57 @@ def run_cocoindex_update(
111112
quiet: bool,
112113
verbose: bool = True,
113114
lance_project_root: Path | None = None,
115+
) -> subprocess.CompletedProcess[str]:
116+
result = _run_cocoindex_update_impl(
117+
env,
118+
full_reprocess=full_reprocess,
119+
quiet=quiet,
120+
verbose=verbose,
121+
lance_project_root=lance_project_root,
122+
)
123+
# After cocoindex returns exit 0 there are no concurrent writers, so this
124+
# is the safe window to compact the Lance tables. The flow disabled its
125+
# in-flight background optimize (see java_index_flow_lancedb.py), making
126+
# this serialized pass the sole optimizer. Optimize failure does not flip
127+
# the cocoindex CompletedProcess (a successful index is still usable, just
128+
# not compacted); the outcome is logged to stderr only.
129+
if result.returncode == 0:
130+
_maybe_run_serialized_optimize(env, quiet=quiet)
131+
return result
132+
133+
134+
def _maybe_run_serialized_optimize(env: dict[str, str], *, quiet: bool) -> None:
135+
"""Resolve the index dir from *env* and run the serialized Lance optimize.
136+
137+
The flow's lifespan reads ``JAVA_CODEBASE_RAG_INDEX_DIR`` (set by the CLI /
138+
config.subprocess_env), so it is guaranteed present when cocoindex ran.
139+
If it is somehow absent we skip optimize with a stderr warning rather than
140+
crash — a successful index is still searchable un-compacted.
141+
"""
142+
idx_raw = env.get("JAVA_CODEBASE_RAG_INDEX_DIR", "").strip()
143+
if not idx_raw:
144+
print(
145+
"java-codebase-rag: optimize skipped — JAVA_CODEBASE_RAG_INDEX_DIR "
146+
"not set in subprocess env",
147+
file=sys.stderr,
148+
)
149+
return
150+
try:
151+
from java_codebase_rag.lance_optimize import optimize_lance_tables
152+
153+
asyncio.run(optimize_lance_tables(Path(idx_raw), quiet=quiet))
154+
except Exception as exc:
155+
# Never crash the CLI on an optimize failure — surface on stderr only.
156+
print(f"java-codebase-rag: optimize failed: {exc}", file=sys.stderr)
157+
158+
159+
def _run_cocoindex_update_impl(
160+
env: dict[str, str],
161+
*,
162+
full_reprocess: bool,
163+
quiet: bool,
164+
verbose: bool = True,
165+
lance_project_root: Path | None = None,
114166
) -> subprocess.CompletedProcess[str]:
115167
exe = cocoindex_bin()
116168
if not exe.is_file():

java_index_flow_lancedb.py

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
from cocoindex.resources.file import PatternFilePathMatcher
3737

3838
from java_codebase_rag.config import resolved_sbert_model_for_process_env
39+
from java_codebase_rag.lance_optimize import LANCE_TABLE_NAMES
3940
from java_index_v1_common import (
4041
JAVA_CHUNK,
4142
SBERT_MODEL,
@@ -68,6 +69,20 @@
6869

6970
splitter = RecursiveSplitter()
7071

72+
# cocoindex 1.0.7 schedules ``table.optimize()`` (a LanceDB Rewrite/compaction
73+
# transaction) as a *background* asyncio task after every
74+
# ``num_transactions_before_optimize`` mutation batches (default 50). That
75+
# background Rewrite races the concurrent ``table.delete()`` (Delete)
76+
# transactions emitted by later batches, and LanceDB does not allow a Rewrite
77+
# to commit concurrently with a Delete (upstream lancedb#1504), which floods
78+
# stderr with "Retryable commit conflict ... preempted by concurrent
79+
# transaction Delete". Setting this effectively to infinity disables the
80+
# in-flight background optimize; the serialized post-flow optimize in
81+
# ``lance_optimize.optimize_lance_tables`` then compacts the table with no
82+
# concurrent writers. ``optimize()`` is pure maintenance (compact/prune/index);
83+
# upsert/delete correctness via merge_insert does not depend on it.
84+
_NUM_TXN_BEFORE_OPTIMIZE = 10**12
85+
7186

7287
@dataclass
7388
class JavaLanceChunk:
@@ -317,8 +332,9 @@ async def app_main() -> None:
317332
)
318333
java_table = await lancedb.mount_table_target(
319334
LANCE_DB,
320-
"javacodeindex_java_code",
335+
LANCE_TABLE_NAMES[0],
321336
java_schema,
337+
num_transactions_before_optimize=_NUM_TXN_BEFORE_OPTIMIZE,
322338
)
323339

324340
sql_schema = await lancedb.TableSchema.from_class(
@@ -327,8 +343,9 @@ async def app_main() -> None:
327343
)
328344
sql_table = await lancedb.mount_table_target(
329345
LANCE_DB,
330-
"sqlschemaindex_sql_schema",
346+
LANCE_TABLE_NAMES[1],
331347
sql_schema,
348+
num_transactions_before_optimize=_NUM_TXN_BEFORE_OPTIMIZE,
332349
)
333350

334351
yaml_schema = await lancedb.TableSchema.from_class(
@@ -337,8 +354,9 @@ async def app_main() -> None:
337354
)
338355
yaml_table = await lancedb.mount_table_target(
339356
LANCE_DB,
340-
"yamlconfigindex_yaml_config",
357+
LANCE_TABLE_NAMES[2],
341358
yaml_schema,
359+
num_transactions_before_optimize=_NUM_TXN_BEFORE_OPTIMIZE,
342360
)
343361

344362
project_root = coco.use_context(PROJECT_ROOT)

server.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ class RefreshIndexOutput(BaseModel):
8787
graph_stdout: str = ""
8888
graph_stderr: str = ""
8989
phases_run: list[Literal["vectors", "graph"]] = Field(default_factory=list)
90+
optimize_error: str | None = None
9091

9192

9293
class IndexInfoOutput(BaseModel):
@@ -329,9 +330,29 @@ async def run_refresh_pipeline(*, quiet: bool = False, verbose: bool = True) ->
329330
graph_code: int | None = None
330331
graph_out = ""
331332
graph_err = ""
333+
optimize_error: str | None = None
332334
if ok:
333335
if not quiet:
334336
print(file=sys.stderr, flush=True)
337+
# Serialized post-flow Lance optimize: the flow disabled its background
338+
# optimize, so with cocoindex returned exit 0 there are no concurrent
339+
# writers — this is the safe window to compact. An optimize failure is
340+
# surfaced via optimize_error / stderr and must NOT flip the success of
341+
# a vectors phase that succeeded; the index is still searchable.
342+
try:
343+
from java_codebase_rag.lance_optimize import optimize_lance_tables
344+
345+
idx_raw = os.environ.get("JAVA_CODEBASE_RAG_INDEX_DIR", "").strip()
346+
if idx_raw and not idx_raw.startswith(("s3://", "gs://", "az://")):
347+
idx_dir = Path(idx_raw).expanduser().resolve()
348+
elif idx_raw:
349+
idx_dir = Path(idx_raw)
350+
else:
351+
idx_dir = (root / ".java-codebase-rag").resolve()
352+
await optimize_lance_tables(idx_dir, quiet=quiet)
353+
except Exception as exc:
354+
optimize_error = f"lance optimize failed: {exc}"
355+
print(f"java-codebase-rag: {optimize_error}", file=sys.stderr)
335356
builder = Path(__file__).resolve().parent / "build_ast_graph.py"
336357
if builder.is_file():
337358
try:
@@ -368,6 +389,10 @@ async def run_refresh_pipeline(*, quiet: bool = False, verbose: bool = True) ->
368389
message = f"cocoindex exit {proc.returncode}"
369390
elif graph_code is not None and graph_code != 0:
370391
message = f"graph builder exit {graph_code}"
392+
# Surface a post-flow optimize failure in the message too (success is not
393+
# flipped — the vectors phase succeeded and the index is still usable).
394+
if optimize_error is not None:
395+
message = optimize_error if message is None else f"{message}; {optimize_error}"
371396
return RefreshIndexOutput(
372397
success=ok and (graph_code is None or graph_code == 0),
373398
exit_code=proc.returncode,
@@ -378,6 +403,7 @@ async def run_refresh_pipeline(*, quiet: bool = False, verbose: bool = True) ->
378403
graph_stdout=graph_out[-4000:] if len(graph_out) > 4000 else graph_out,
379404
graph_stderr=graph_err[-4000:] if len(graph_err) > 4000 else graph_err,
380405
phases_run=phases_run,
406+
optimize_error=optimize_error,
381407
)
382408

383409

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"exit_code": 0, "graph_exit_code": 0, "graph_stderr": "", "graph_stdout": "", "message": null, "phases_run": ["vectors", "graph"], "stderr": "", "stdout": "", "success": true}
1+
{"exit_code": 0, "graph_exit_code": 0, "graph_stderr": "", "graph_stdout": "", "message": null, "optimize_error": null, "phases_run": ["vectors", "graph"], "stderr": "", "stdout": "", "success": true}

0 commit comments

Comments
 (0)