diff --git a/install.ps1 b/install.ps1 index bcd5d44..111f6e1 100644 --- a/install.ps1 +++ b/install.ps1 @@ -10,12 +10,15 @@ # 2. Downloads volca--windows-amd64.zip from the GH Release. # 3. Downloads volca-data-.tar.gz (reference data bundle). # 4. Verifies both against SHA256SUMS. -# 5. Extracts to: +# 5. Extracts to (matches platformdirs.user_data_dir("volca", appauthor=False)): # $env:LOCALAPPDATA\volca\\volca.exe # $env:LOCALAPPDATA\volca\data\\{flows.csv,...} # and points $env:LOCALAPPDATA\volca\data\current at . +# Same root as install.sh and pyvolca.download(). # 6. Installs a thin shim at $env:LOCALAPPDATA\volca\bin\volca.cmd that # sets VOLCA_DATA_DIR and execs the real binary. +# +# Override the install root with $env:VOLCA_HOME = 'C:\full\path'. # ============================================================================= [CmdletBinding()] @@ -26,7 +29,7 @@ param( $ErrorActionPreference = 'Stop' $Repo = 'ccomb/volca' -$Prefix = if ($env:VOLCA_PREFIX) { $env:VOLCA_PREFIX } else { Join-Path $env:LOCALAPPDATA 'volca' } +$Prefix = if ($env:VOLCA_HOME) { $env:VOLCA_HOME } else { Join-Path $env:LOCALAPPDATA 'volca' } $BinDir = Join-Path $Prefix 'bin' $Shim = Join-Path $BinDir 'volca.cmd' diff --git a/install.sh b/install.sh index 4065541..883d25f 100755 --- a/install.sh +++ b/install.sh @@ -11,14 +11,17 @@ # 2. Downloads volca--.tar.gz from the GH Release # 3. Downloads volca-data-.tar.gz (the reference data bundle) # 4. Verifies both against SHA256SUMS -# 5. Extracts to: -# ~/.local/share/volca//volca -# ~/.local/share/volca/data//{flows.csv,...} -# and points ~/.local/share/volca/data/current at . +# 5. Extracts to the OS-native user data dir (matches platformdirs): +# Linux: ${XDG_DATA_HOME:-~/.local/share}/volca//volca +# macOS: ~/Library/Application Support/volca//volca +# with the data bundle under /data// and a +# /data/current symlink pointing at it. # 6. Installs a thin shim at ~/.local/bin/volca that sets VOLCA_DATA_DIR # and execs the real binary. The shim makes the data-bundle layout # invisible to users — they just `volca …`. # +# Override the install root with VOLCA_HOME=/full/path (skips OS detection). +# # Windows: # This script aborts. Use install.ps1 (or download from GH Releases). # ============================================================================= @@ -26,9 +29,7 @@ set -eu REPO="ccomb/volca" -PREFIX="${VOLCA_PREFIX:-$HOME/.local}" -SHARE_DIR="$PREFIX/share/volca" -BIN_DIR="$PREFIX/bin" +BIN_DIR="$HOME/.local/bin" SHIM="$BIN_DIR/volca" VERSION="${1:-}" @@ -67,6 +68,19 @@ case "$UNAME_S" in esac PLATFORM="${OS}-${ARCH}" +# --- Resolve install root ---------------------------------------------------- +# Mirrors platformdirs.user_data_dir("volca", appauthor=False) — the same root +# pyvolca.download() and install.ps1 use, so all three installers share it. + +if [ -n "${VOLCA_HOME:-}" ]; then + SHARE_DIR="$VOLCA_HOME" +else + case "$OS" in + linux) SHARE_DIR="${XDG_DATA_HOME:-$HOME/.local/share}/volca" ;; + macos) SHARE_DIR="$HOME/Library/Application Support/volca" ;; + esac +fi + # --- Tooling sanity ---------------------------------------------------------- for tool in curl tar; do diff --git a/pyvolca/README.md b/pyvolca/README.md index ffe3405..1387d8a 100644 --- a/pyvolca/README.md +++ b/pyvolca/README.md @@ -20,7 +20,7 @@ Requires Python ≥ 3.10 and a running VoLCA engine. Use `Server` (below) to run Most users should start with one of these two modes: - **You already have access to a VoLCA server** (for example a hosted server prepared by someone else): use `Client` only. You do not need `volca.toml`, and you do not need to install the VoLCA server locally. -- **You want Python to start a local VoLCA engine process for you**: use `download()` once to fetch the VoLCA engine binary and reference data into pyvolca's cache, then use `Server` to start it from Python. `volca.toml` is still a normal file path passed to `Server(config=...)`; put it in your project directory, or pass an absolute path. Do not put it inside your virtualenv or inside `site-packages`. +- **You want Python to start a local VoLCA engine process for you**: use `download()` once to fetch the VoLCA engine binary and reference data into the shared volca install dir (see [Where artefacts are installed](#where-artefacts-are-installed)), then use `Server` to start it from Python. `volca.toml` is still a normal file path passed to `Server(config=...)`; put it in your project directory, or pass an absolute path. Do not put it inside your virtualenv or inside `site-packages`. For a hosted server, the minimal connection looks like this: @@ -50,7 +50,21 @@ with Server(config="./volca.toml", binary=str(installed.binary)) as srv: print(c.list_databases()) ``` -In this local mode, `download()` stores the engine binary and reference data in pyvolca's user cache. `Server(config="./volca.toml")` still means “read `./volca.toml` relative to the current working directory”. +In this local mode, `download()` stores the engine binary and reference data in the shared volca install dir (see below). `Server(config="./volca.toml")` still means “read `./volca.toml` relative to the current working directory”. + +### Where artefacts are installed + +`download()` writes to the same OS-native location as the `install.sh` / `install.ps1` shell installers, so any of the three tools populate the same directory: + +| Platform | Default install root | +|---|---| +| Linux | `${XDG_DATA_HOME:-~/.local/share}/volca/` | +| macOS | `~/Library/Application Support/volca/` | +| Windows | `%LOCALAPPDATA%\volca\` | + +Override with `VOLCA_HOME=/full/path` (full path; skips OS detection). + +If you ran `install.sh` or `install.ps1` first, `Server()` finds the installed engine without an extra `download()` call. If you previously used `pyvolca < 0.4` it cached artefacts under `/pyvolca/` (Linux: `~/.cache/pyvolca/`); that directory is no longer read and can be removed (`rm -rf ~/.cache/pyvolca`). ## Local managed-server quick start @@ -285,7 +299,6 @@ Pyvolca dispatches dynamically against the engine's OpenAPI spec, so it ships wi ## API reference - _This reference is generated from the installed package. Run `python scripts/gen_api_md.py` to regenerate._ ## Classes @@ -476,7 +489,7 @@ An exchange with the environment (resource extraction or emission). Filter a supply-chain/consumers query by a classification (system, value, mode). -Matches one classification system entry (e.g. ("Category", "Agricultural\\Food", +Matches one classification system entry (e.g. ("Category", "Agricultural\Food", "exact")). Mode is "exact" (case-insensitive equality) or "contains" (substring). Multiple filters are AND-combined by the server. @@ -725,13 +738,13 @@ common case for "what does this variant consume differently?". Pass Download the volca binary + data bundle for the current platform. -Idempotent: if both artefacts are already extracted under the expected -cache paths and ``force=False``, returns immediately without network. +Idempotent: if both artefacts are already extracted under the install +root and ``force=False``, returns immediately without network. Args: version: GH Release tag (``v0.7.0``); ``None`` resolves the latest. repo: GitHub repo slug. Default ``ccomb/volca``. - force: Re-download even if the cache looks complete. + force: Re-download even if the install root looks complete. Returns: :class:`Installed` with the resolved paths and versions. @@ -742,6 +755,7 @@ Returns: Type alias: `Union[TechnosphereExchange, BiosphereExchange]`. + ## See also diff --git a/pyvolca/src/volca/_download.py b/pyvolca/src/volca/_download.py index c03908d..a7bdcc5 100644 --- a/pyvolca/src/volca/_download.py +++ b/pyvolca/src/volca/_download.py @@ -1,17 +1,25 @@ -"""Download cache for the volca binary + reference data bundle. +"""Download + install the volca binary + reference data bundle. -Public surface: :func:`download`. ``Server`` calls ``cached_binary`` and -``cached_data_dir`` to pick up downloaded artefacts when no explicit -binary path is configured. +Public surface: :func:`download`. ``Server`` calls ``installed_binary`` and +``installed_data_dir`` to pick up artefacts when no explicit binary path is +configured. -Cache layout (mirrors install.sh):: +The install layout is shared with ``install.sh`` and ``install.ps1`` — +running any of the three installers populates the same root:: - / + / /volca[.exe] data//{flows.csv, compartments.csv, units.csv, geographies.csv, flows.csv.cache.zst} data/current -> data/ (symlink, or copy on platforms without symlinks) + +Resolved per-platform: + - Linux: ${XDG_DATA_HOME:-~/.local/share}/volca/ + - macOS: ~/Library/Application Support/volca/ + - Windows: %LOCALAPPDATA%\\volca\\ + +Override the root with ``$VOLCA_HOME`` (full path, skips platform detection). """ from __future__ import annotations @@ -30,7 +38,7 @@ from pathlib import Path from typing import Iterator, Optional -from platformdirs import user_cache_dir +from platformdirs import user_data_dir if sys.platform == "win32": import msvcrt @@ -84,40 +92,82 @@ def _binary_name() -> str: # --------------------------------------------------------------------------- -# Cache paths +# Install paths # --------------------------------------------------------------------------- +_SEMVER_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)$") -def _cache_root() -> Path: - return Path(user_cache_dir("pyvolca")) + +def _install_root() -> Path: + """Resolve the volca install root. + + ``$VOLCA_HOME`` overrides everything (full path). Otherwise falls back + to ``platformdirs.user_data_dir("volca", appauthor=False)`` — same root + as install.sh / install.ps1. + """ + if home := os.environ.get("VOLCA_HOME"): + return Path(home) + return Path(user_data_dir("volca", appauthor=False)) def _manifest_path() -> Path: - return _cache_root() / "latest.json" + return _install_root() / "latest.json" + +def _scan_installed_versions(root: Path) -> list[tuple[tuple[int, int, int], Path]]: + """Find every ``/X.Y.Z/`` and rank by semver, highest first. -def cached_binary(version: Optional[str] = None) -> Optional[Path]: + Lets ``installed_binary()`` discover engines installed via install.sh / + install.ps1, which don't write ``latest.json``. + """ + if not root.is_dir(): + return [] + binary = _binary_name() + found: list[tuple[tuple[int, int, int], Path]] = [] + for child in root.iterdir(): + if not child.is_dir(): + continue + m = _SEMVER_RE.match(child.name) + if not m: + continue + bin_path = child / binary + if not bin_path.is_file(): + continue + key = (int(m.group(1)), int(m.group(2)), int(m.group(3))) + found.append((key, bin_path)) + found.sort(reverse=True) + return found + + +def installed_binary(version: Optional[str] = None) -> Optional[Path]: """Return the binary path for ``version`` if extracted, else ``None``. - If ``version`` is ``None``, returns the binary recorded in - ``latest.json`` by the most recent :func:`download` call. + With ``version=None``, prefers the version recorded in ``latest.json`` + by the most recent :func:`download` call. If that file is missing — e.g. + the user ran install.sh / install.ps1, which don't write a manifest — + falls back to scanning the install root for the highest-semver dir that + contains the binary. """ - if version is None: - manifest = _manifest_path() - if not manifest.is_file(): - return None + root = _install_root() + if version is not None: + p = root / version.lstrip("v") / _binary_name() + return p if p.is_file() else None + manifest = _manifest_path() + if manifest.is_file(): try: data = json.loads(manifest.read_text()) + ver = data.get("version") + if ver: + p = root / ver.lstrip("v") / _binary_name() + if p.is_file(): + return p except (OSError, json.JSONDecodeError): - return None - version = data.get("version") - if not version: - return None - p = _cache_root() / version.lstrip("v") / _binary_name() - return p if p.is_file() else None + pass + scanned = _scan_installed_versions(root) + return scanned[0][1] if scanned else None -def cached_data_dir() -> Optional[Path]: +def installed_data_dir() -> Optional[Path]: """Return the active data dir (``data/current``) if it exists. Resolves symlinks before returning, so a stale link to a removed @@ -125,7 +175,7 @@ def cached_data_dir() -> Optional[Path]: cannot read. ``Path.exists`` follows symlinks; ``is_symlink`` alone would happily return a broken pointer. """ - p = _cache_root() / "data" / "current" + p = _install_root() / "data" / "current" if p.exists() and p.is_dir(): return p return None @@ -197,11 +247,11 @@ def _extract(archive: Path, dest: Path) -> None: def _exclusive_lock(path: Path) -> Iterator[None]: """Hold a process-exclusive lock on ``path`` for the duration of the block. - Lets concurrent ``download()`` callers serialise around the cache writes: - one process actually downloads + extracts, the others wait, and on - re-entering the critical section see a fully-populated cache and short- - circuit. The OS releases the lock if the holder dies, so a crashed run - does not strand subsequent callers. + Lets concurrent ``download()`` callers serialise around the install + writes: one process actually downloads + extracts, the others wait, and + on re-entering the critical section see a fully-populated root and + short-circuit. The OS releases the lock if the holder dies, so a crashed + run does not strand subsequent callers. fcntl.flock on Unix; msvcrt.locking on Windows. Both are kernel-level. """ @@ -243,8 +293,8 @@ def _link_current(target: Path, link: Path) -> None: elif link.is_dir(): shutil.rmtree(link) try: - # Use a relative target so the cache stays portable if the parent - # dir gets moved (e.g. user's $HOME relocates). + # Use a relative target so the install dir stays portable if the + # parent gets moved (e.g. user's $HOME relocates). link.symlink_to(target.name, target_is_directory=True) except (OSError, NotImplementedError): shutil.copytree(target, link) @@ -263,13 +313,13 @@ def download( ) -> Installed: """Download the volca binary + data bundle for the current platform. - Idempotent: if both artefacts are already extracted under the expected - cache paths and ``force=False``, returns immediately without network. + Idempotent: if both artefacts are already extracted under the install + root and ``force=False``, returns immediately without network. Args: version: GH Release tag (``v0.7.0``); ``None`` resolves the latest. repo: GitHub repo slug. Default ``ccomb/volca``. - force: Re-download even if the cache looks complete. + force: Re-download even if the install root looks complete. Returns: :class:`Installed` with the resolved paths and versions. @@ -278,15 +328,15 @@ def download( tag = _resolve_tag(repo, version) plain_version = tag.lstrip("v") - cache = _cache_root() - cache.mkdir(parents=True, exist_ok=True) + root = _install_root() + root.mkdir(parents=True, exist_ok=True) - with _exclusive_lock(cache / ".lock"): - return _download_locked(cache, repo, tag, plain_version, slug, ext, force) + with _exclusive_lock(root / ".lock"): + return _download_locked(root, repo, tag, plain_version, slug, ext, force) def _download_locked( - cache: Path, + root: Path, repo: str, tag: str, plain_version: str, @@ -295,7 +345,7 @@ def _download_locked( force: bool, ) -> Installed: # ---- download SHA256SUMS first; it tells us which data version to fetch - sums_path = cache / f"SHA256SUMS-{tag}" + sums_path = root / f"SHA256SUMS-{tag}" sums_path.write_bytes(_http_get(_GH_RELEASES.format(repo=repo, tag=tag, asset="SHA256SUMS"))) sums = _parse_sha256sums(sums_path.read_text()) @@ -312,18 +362,18 @@ def _download_locked( raise DownloadError(f"cannot parse data version from {data_asset}") data_version = m.group(1) - binary_dir = cache / plain_version - data_dir = cache / "data" / data_version + binary_dir = root / plain_version + data_dir = root / "data" / data_version bin_path = binary_dir / _binary_name() - fully_cached = ( + already_installed = ( bin_path.is_file() and (data_dir / "flows.csv").is_file() and not force ) - if not fully_cached: + if not already_installed: # ---- binary - bin_arch = cache / f"_dl-{binary_asset}" + bin_arch = root / f"_dl-{binary_asset}" _download_asset(repo, tag, binary_asset, bin_arch) _verify(bin_arch, sums[binary_asset]) _extract(bin_arch, binary_dir) @@ -332,16 +382,17 @@ def _download_locked( os.chmod(bin_path, 0o755) # ---- data bundle - data_arch = cache / f"_dl-{data_asset}" + data_arch = root / f"_dl-{data_asset}" _download_asset(repo, tag, data_asset, data_arch) _verify(data_arch, sums[data_asset]) _extract(data_arch, data_dir) data_arch.unlink(missing_ok=True) - _link_current(data_dir, cache / "data" / "current") + _link_current(data_dir, root / "data" / "current") - # Manifest lets Server.start() find the cached binary without knowing - # which version was downloaded. Rewritten on every download() call. + # Manifest lets Server.start() find the binary without knowing which + # version was downloaded. install.sh / install.ps1 don't write this file + # — installed_binary() falls back to a semver scan in that case. _manifest_path().write_text( json.dumps( {"version": plain_version, "data_version": data_version, "binary": str(bin_path)}, @@ -351,7 +402,7 @@ def _download_locked( return Installed( binary=bin_path, - data_dir=cache / "data" / "current", + data_dir=root / "data" / "current", version=plain_version, data_version=data_version, ) diff --git a/pyvolca/src/volca/server.py b/pyvolca/src/volca/server.py index ad612f8..df2a10e 100644 --- a/pyvolca/src/volca/server.py +++ b/pyvolca/src/volca/server.py @@ -69,18 +69,18 @@ def _find_binary(self) -> str: Resolution order: 1. ``self.binary`` if it is an existing path. - 2. The most recent download cached by :func:`volca.download` — - so a script can ``volca.download()`` then ``Server()`` and - have the spawn pick up the cached engine without extra wiring. + 2. The shared install root (``platformdirs.user_data_dir``) — + populated by :func:`volca.download`, ``install.sh``, or + ``install.ps1`` interchangeably. 3. ``shutil.which(self.binary)`` — PATH lookup, including the ``~/.local/bin/volca`` shim that ``install.sh`` drops. 4. ``./volca`` / ``./dist/volca`` for ad-hoc dev trees. """ if Path(self.binary).exists(): return self.binary - cached = _download.cached_binary() - if cached is not None: - return str(cached) + installed = _download.installed_binary() + if installed is not None: + return str(installed) found = shutil.which(self.binary) if found: return found @@ -95,15 +95,15 @@ def _find_binary(self) -> str: def _subprocess_env(self) -> dict: """Subprocess env for the spawned engine. - When the data bundle has been downloaded into the pyvolca cache, + When the data bundle has been installed into the shared install root, export VOLCA_DATA_DIR so the engine resolves "data/flows.csv" and - friends against the cached bundle instead of the engine's CWD. + friends against the bundle instead of the engine's CWD. """ env = os.environ.copy() if "VOLCA_DATA_DIR" not in env: - cached = _download.cached_data_dir() - if cached is not None: - env["VOLCA_DATA_DIR"] = str(cached) + installed = _download.installed_data_dir() + if installed is not None: + env["VOLCA_DATA_DIR"] = str(installed) return env def is_alive(self) -> bool: diff --git a/pyvolca/tests/test_download.py b/pyvolca/tests/test_download.py index 9da3a88..962fc9c 100644 --- a/pyvolca/tests/test_download.py +++ b/pyvolca/tests/test_download.py @@ -108,34 +108,85 @@ def test_platform_slug_macos_x86_64_rejected(): # --------------------------------------------------------------------------- -# cached_binary +# _install_root # --------------------------------------------------------------------------- -def test_cached_binary_no_manifest(tmp_path: Path): - with mock.patch.object(_download, "_cache_root", return_value=tmp_path): - assert _download.cached_binary() is None +def test_install_root_default_uses_user_data_dir(tmp_path: Path, monkeypatch): + """Without ``$VOLCA_HOME``, defer to platformdirs.user_data_dir.""" + monkeypatch.delenv("VOLCA_HOME", raising=False) + with mock.patch("volca._download.user_data_dir", return_value=str(tmp_path / "platform")) as ud: + result = _download._install_root() + ud.assert_called_once_with("volca", appauthor=False) + assert result == tmp_path / "platform" -def test_cached_binary_explicit_version(tmp_path: Path): +def test_install_root_volca_home_overrides(tmp_path: Path, monkeypatch): + """``$VOLCA_HOME`` short-circuits platformdirs.""" + monkeypatch.setenv("VOLCA_HOME", str(tmp_path / "custom")) + with mock.patch("volca._download.user_data_dir") as ud: + result = _download._install_root() + ud.assert_not_called() + assert result == tmp_path / "custom" + + +# --------------------------------------------------------------------------- +# installed_binary +# --------------------------------------------------------------------------- + + +def test_installed_binary_no_manifest_no_dirs(tmp_path: Path): + """Empty install root → None.""" + with mock.patch.object(_download, "_install_root", return_value=tmp_path): + assert _download.installed_binary() is None + + +def test_installed_binary_explicit_version(tmp_path: Path): bin_dir = tmp_path / "0.7.0" bin_dir.mkdir() bin_path = bin_dir / "volca" bin_path.write_bytes(b"") - with mock.patch.object(_download, "_cache_root", return_value=tmp_path), \ + with mock.patch.object(_download, "_install_root", return_value=tmp_path), \ mock.patch.object(_download, "_binary_name", return_value="volca"): - assert _download.cached_binary("v0.7.0") == bin_path - assert _download.cached_binary("0.7.0") == bin_path + assert _download.installed_binary("v0.7.0") == bin_path + assert _download.installed_binary("0.7.0") == bin_path -def test_cached_binary_via_manifest(tmp_path: Path): +def test_installed_binary_via_manifest(tmp_path: Path): bin_dir = tmp_path / "0.7.0" bin_dir.mkdir() (bin_dir / "volca").write_bytes(b"") (tmp_path / "latest.json").write_text(json.dumps({"version": "0.7.0"})) - with mock.patch.object(_download, "_cache_root", return_value=tmp_path), \ + with mock.patch.object(_download, "_install_root", return_value=tmp_path), \ + mock.patch.object(_download, "_binary_name", return_value="volca"): + result = _download.installed_binary() + assert result == bin_dir / "volca" + + +def test_installed_binary_scan_fallback_when_no_manifest(tmp_path: Path): + """install.sh / install.ps1 don't write latest.json — fall back to a + semver scan and pick the highest version that contains the binary.""" + for ver in ("0.6.0", "0.7.0", "0.10.1"): + d = tmp_path / ver + d.mkdir() + (d / "volca").write_bytes(b"") + # A non-semver dir and a semver dir without the binary must both be ignored. + (tmp_path / "scratch").mkdir() + (tmp_path / "0.5.0").mkdir() # no binary + with mock.patch.object(_download, "_install_root", return_value=tmp_path), \ + mock.patch.object(_download, "_binary_name", return_value="volca"): + result = _download.installed_binary() + assert result == tmp_path / "0.10.1" / "volca" + + +def test_installed_binary_corrupt_manifest_falls_back_to_scan(tmp_path: Path): + bin_dir = tmp_path / "0.7.0" + bin_dir.mkdir() + (bin_dir / "volca").write_bytes(b"") + (tmp_path / "latest.json").write_text("not json {{{") + with mock.patch.object(_download, "_install_root", return_value=tmp_path), \ mock.patch.object(_download, "_binary_name", return_value="volca"): - result = _download.cached_binary() + result = _download.installed_binary() assert result == bin_dir / "volca"