From 51da1ee4c3fb5762318cc30616e1aa22a7a0ca94 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 2 Mar 2026 23:06:15 -0500 Subject: [PATCH 1/3] Move `fix-run-external-deps` spec to `specs/done/` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implemented in 052e04748 ("Fix `dvx run` circular dependency error + add `git_deps` support"). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- specs/done/fix-run-external-deps.md | 130 ++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 specs/done/fix-run-external-deps.md diff --git a/specs/done/fix-run-external-deps.md b/specs/done/fix-run-external-deps.md new file mode 100644 index 000000000..f5664e373 --- /dev/null +++ b/specs/done/fix-run-external-deps.md @@ -0,0 +1,130 @@ +# Fix `dvx run` circular dependency error for external deps + git dep support + +## Problem + +`dvx run` reports "Circular dependency detected" when `.dvc` files have deps on files that aren't DVX-tracked (e.g. git-tracked notebooks, PDFs). + +## Root Cause + +In `src/dvx/run/executor.py:run()` (lines 511-516), when building the artifact graph from `.dvc` files, dependencies are only added to the graph if they have a corresponding `.dvc` file: + +```python +if artifact.computation: + for dep in artifact.computation.deps: + dep_path = dep.path if isinstance(dep, Artifact) else str(dep) + dvc_file = Path(str(dep_path) + ".dvc") + if dvc_file.exists() and dep_path not in artifacts: + pending.append(dvc_file) +``` + +External dependencies (git-tracked files like `.pdf`, `.ipynb`) are silently dropped from the graph. Then in `_group_into_levels()`, when checking if all deps are "done", these missing dep paths are never in the `done` set, so the artifact is permanently "not ready". Since nothing is ever ready, it raises "Circular dependency detected". + +## Immediate Fix (unblock `dvx run`) + +In `run()`, when a dep has no `.dvc` file, add it as a leaf `Artifact` (no computation): + +```python +if artifact.computation: + for dep in artifact.computation.deps: + dep_path = dep.path if isinstance(dep, Artifact) else str(dep) + if dep_path not in artifacts: + dvc_file = Path(str(dep_path) + ".dvc") + if dvc_file.exists(): + pending.append(dvc_file) + else: + # External dependency (no .dvc file) - add as leaf + artifacts[dep_path] = Artifact(path=dep_path) +``` + +This way `_group_into_levels` sees leaf artifacts (computation=None) for external deps and immediately adds them to `done` at line 78. + +## Design: `git_deps` for git-tracked dependencies + +Currently deps use content MD5 hashes: +```yaml +deps: + data/2025-PATH-Monthly-Ridership-Report.pdf: 22c8cb757209390928b525c6fc5b37f5 + monthly.ipynb: 48c56466c13cd724fe771ab7631fff57 +``` + +But for git-tracked files, using git blob SHAs would be better: +- Freshness checks via `git ls-tree HEAD` (already cached in `_get_blob_cache()` in `dvc_files.py`) +- No need to read file content / compute MD5 for freshness +- Clear semantic distinction: DVX deps (have `.dvc` files, MD5) vs git deps (git-tracked, SHA-1) + +### Proposed format + +Add a separate `git_deps` key in the computation block: + +```yaml +meta: + computation: + cmd: papermill monthly.ipynb out/monthly-2025.ipynb -p year 2025 + deps: + data/2025.pqt: 278da34972b360e3a54aade6cdaf0384 + git_deps: + data/2025-PATH-Monthly-Ridership-Report.pdf: + monthly.ipynb: +``` + +- `deps`: DVX-tracked artifacts (have `.dvc` files). Hash is content MD5. +- `git_deps`: Git-tracked files. Hash is git blob SHA-1. + +### Implementation + +**`DVCFileInfo`** (`dvc_files.py`): add `git_deps: dict[str, str]` field, populate from `computation.get("git_deps") or {}`. + +**`Computation`** (`artifact.py`): add `git_deps: list[Artifact | str | Path]` field. + +**`Artifact.from_dvc`** (`artifact.py`): populate `git_deps` from `info.git_deps`. + +**`is_output_fresh`** (`dvc_files.py`): for `git_deps`, compare recorded blob SHA against `get_git_blob_sha(dep_path, "HEAD")` (uses the already-cached `_get_blob_cache`). + +**`run()`** (`executor.py`): when building graph, git_deps are always leaf nodes (no `.dvc` file). Add them as leaf Artifacts. + +**`_group_into_levels`**: no change needed — git deps become leaf artifacts, immediately "done". + +**`write_dvc_file`** (`dvc_files.py`): accept `git_deps` param, write into `meta.computation.git_deps`. + +**`Computation.get_dep_hashes`**: for git_deps, use `get_git_blob_sha()` instead of `compute_md5()`. + +### Script to populate git_deps + +The `add-dvc-computations.py` script (or a new one) should: +1. For each `.dvc` file, determine which deps are git-tracked vs DVX-tracked +2. For git-tracked deps, compute `get_git_blob_sha(path, "HEAD")` and put in `git_deps` +3. For DVX-tracked deps, keep in `deps` with content MD5 + +## Reproduction + +In the `hccs/path` repo: +```bash +dvx run -n -v data/2025.pqt.dvc +# Error: Circular dependency detected +``` + +The `data/2025.pqt.dvc` file has: +```yaml +meta: + computation: + cmd: papermill monthly.ipynb out/monthly-2025.ipynb -p year 2025 + deps: + data/2025-PATH-Monthly-Ridership-Report.pdf: 22c8cb757209390928b525c6fc5b37f5 + monthly.ipynb: 48c56466c13cd724fe771ab7631fff57 +``` + +Neither dep has a `.dvc` file (both are git-tracked). + +## Testing + +After the fix: +```bash +dvx run -n -v data/2025.pqt.dvc +# Should show: "would run" or "skip (up-to-date)" instead of circular dep error +``` + +Also test with multi-target: +```bash +dvx run -n -v data/*.dvc www/public/*.dvc +# Should show execution plan with levels +``` From 971a643edb2d7ef199050096a608b3e314751e0f Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 2 Mar 2026 23:50:15 -0500 Subject: [PATCH 2/3] Store HTTP `Last-Modified` as `mtime` in `.dvc` deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin `dvc-data` to runsascoded/dvc-data@`9b27dc6` which captures HTTP `Last-Modified` headers into `Meta.mtime` via `from_info()`. Add `_compat.py` to patch DVC at runtime: - Allow `mtime` in DVC's `.dvc` file schema validation - Extend `Meta.to_dict()` to serialize `mtime` for HTTP sources (distinguished from local fs mtime by absence of `inode`) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- pyproject.toml | 5 ++++- src/dvx/__init__.py | 2 ++ src/dvx/_compat.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 src/dvx/_compat.py diff --git a/pyproject.toml b/pyproject.toml index 45d444a7e..adef01498 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "click>=8.0", "dffs>=0.1.1", "dvc>=3.50", - "dvc-data>=3.18.3", + "dvc-data @ git+https://github.com/runsascoded/dvc-data.git@9b27dc6e51842015ae5b697e2d8079efa9a1c53d", ] [project.optional-dependencies] @@ -67,6 +67,9 @@ dvx = "dvx.cli:main" Homepage = "https://github.com/runsascoded/dvx" Repository = "https://github.com/runsascoded/dvx" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.version] source = "vcs" diff --git a/src/dvx/__init__.py b/src/dvx/__init__.py index 2bb374392..73803c249 100644 --- a/src/dvx/__init__.py +++ b/src/dvx/__init__.py @@ -15,6 +15,8 @@ repo.push() """ +import dvx._compat # noqa: F401 # patch DVC schemas before any validation + from dvx.repo import Repo __all__ = ["Repo"] diff --git a/src/dvx/_compat.py b/src/dvx/_compat.py new file mode 100644 index 000000000..0e9a241b5 --- /dev/null +++ b/src/dvx/_compat.py @@ -0,0 +1,39 @@ +"""Patch DVC to support HTTP `Last-Modified` as `mtime` in `.dvc` files. + +dvc-data's `Meta.from_info()` captures HTTP `Last-Modified` into `mtime`, but: +1. DVC's voluptuous schema doesn't allow `mtime` in deps/outs +2. `Meta.to_dict()` doesn't serialize `mtime` (doing so breaks local files) + +This module patches both: adds `mtime` to the schema, and extends `to_dict()` +to include `mtime` when `checksum` is set (HTTP sources use `checksum` for +ETag/Content-MD5, while local/S3/GCS use `etag`/`md5`). +""" + + +def _patch(): + from dvc.dependency import SCHEMA as DEP_SCHEMA + from dvc.output import ARTIFACT_SCHEMA, META_SCHEMA + from dvc_data.hashfile.meta import Meta + + # 1. Allow `mtime` in DVC's .dvc file schema validation + for schema in (META_SCHEMA, ARTIFACT_SCHEMA, DEP_SCHEMA): + if "mtime" not in schema: + schema["mtime"] = float + + # 2. Extend Meta.to_dict() to serialize mtime for HTTP sources. + # Local filesystems always have inode set; HTTP sources never do. + # Use this to distinguish HTTP mtime (from Last-Modified) from + # local filesystem mtime (which DVC uses internally but shouldn't + # persist to .dvc files). + _orig_to_dict = Meta.to_dict + + def _to_dict_with_mtime(self): + ret = _orig_to_dict(self) + if self.mtime is not None and self.inode is None: + ret["mtime"] = self.mtime + return ret + + Meta.to_dict = _to_dict_with_mtime + + +_patch() From d14bfa738e4d0826ae38aa69bc923ab93626f73b Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 3 Mar 2026 00:04:47 -0500 Subject: [PATCH 3/3] Update `dvc-data` pin to `9d3ef02` (ruff fix + CI split) Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index adef01498..34f751211 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "click>=8.0", "dffs>=0.1.1", "dvc>=3.50", - "dvc-data @ git+https://github.com/runsascoded/dvc-data.git@9b27dc6e51842015ae5b697e2d8079efa9a1c53d", + "dvc-data @ git+https://github.com/runsascoded/dvc-data.git@9d3ef02c2c1e6752f897d2885f467d12fe934646", ] [project.optional-dependencies]