From 39e34578f60e794becac35df66cd4f11bd59e8f9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:34:11 +0000 Subject: [PATCH 01/33] Fix reinstall-overwrites-kept-config: preserve config on plain reinstall after --keep-config Apply the remediation from the bug assessment on issue #3427. Before the unconditional shutil.rmtree(dest_dir), scan dest_dir for any *-config.yml and *-config.local.yml files and hold their contents in memory. After shutil.copytree succeeds, write them back so user-customized values always win over the packaged defaults. This mirrors the existing backup/restore logic for the --force reinstall path but handles the case where remove --keep-config left config files behind in an unregistered extension directory. Refs #3427 Assisted-by: GitHub Copilot (model: claude-sonnet-4.6, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/extensions/__init__.py | 20 +++++++++ tests/test_extensions.py | 58 +++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index e00b9c247a..8f62928387 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1402,6 +1402,22 @@ def install_from_directory( backup_config_dir.unlink() did_remove = self.remove(manifest.id) + # Rescue any config files left behind by a prior `remove --keep-config`. + # When an extension is removed with --keep-config, it is no longer in + # the registry but its config files remain in dest_dir. A subsequent + # plain (non-force) install would delete that directory unconditionally, + # silently discarding the preserved config. We read those files into + # memory now and write them back after copytree so the user's values + # always win over the packaged defaults. + stranded_configs: dict[str, bytes] = {} + if dest_dir.exists() and not self.registry.is_installed(manifest.id): + for cfg_file in ( + list(dest_dir.glob("*-config.yml")) + + list(dest_dir.glob("*-config.local.yml")) + ): + if cfg_file.is_file() and not cfg_file.is_symlink(): + stranded_configs[cfg_file.name] = cfg_file.read_bytes() + # Install extension (dest_dir computed above during self-install guard) if dest_dir.exists(): shutil.rmtree(dest_dir) @@ -1432,6 +1448,10 @@ def install_from_directory( hook_executor = HookExecutor(self.project_root) hook_executor.register_hooks(manifest) + # Restore stranded configs rescued before the rmtree above. + for filename, content in stranded_configs.items(): + (dest_dir / filename).write_bytes(content) + # Restore config files from backup when --force triggered a removal. # Only restore *.yml config files to match what remove() backs up, # so unexpected artifacts in .backup/ are not resurrected. diff --git a/tests/test_extensions.py b/tests/test_extensions.py index d2f4af8264..b1d85fdb2d 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1226,7 +1226,63 @@ def test_install_force_config_preserved(self, extension_dir, project_dir): assert new_config.exists() assert new_config.read_text() == "test: config" - def test_install_force_without_existing(self, extension_dir, project_dir): + def test_reinstall_after_keep_config_preserves_config( + self, extension_dir, project_dir + ): + """Reinstalling after `remove --keep-config` must not overwrite preserved config.""" + manager = ExtensionManager(project_dir) + + # Install once + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + # Customize the config file + ext_dir = project_dir / ".specify" / "extensions" / "test-ext" + config_file = ext_dir / "test-ext-config.yml" + config_file.write_text("model: custom-model\nmax_iterations: 99\n") + + # Remove while preserving config + manager.remove("test-ext", keep_config=True) + assert not manager.registry.is_installed("test-ext") + assert config_file.exists() + assert "custom-model" in config_file.read_text() + + # Plain reinstall (no --force) + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + # Preserved config must survive the reinstall + assert config_file.exists() + assert "custom-model" in config_file.read_text() + assert "99" in config_file.read_text() + + def test_reinstall_after_keep_config_preserves_local_config( + self, extension_dir, project_dir + ): + """Local config override files (*-config.local.yml) are also rescued on reinstall.""" + manager = ExtensionManager(project_dir) + + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + ext_dir = project_dir / ".specify" / "extensions" / "test-ext" + local_cfg = ext_dir / "test-ext-config.local.yml" + local_cfg.write_text("local_override: true\n") + + manager.remove("test-ext", keep_config=True) + assert local_cfg.exists() + + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + assert local_cfg.exists() + assert "local_override: true" in local_cfg.read_text() + + """Test force-install when extension is NOT already installed (works normally).""" manager = ExtensionManager(project_dir) From f43ae8c2e58fcd0ad00410408a1c33cc34dd75b0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:26:45 +0000 Subject: [PATCH 02/33] fix: restore method decl, move config restore before registration, preserve file mode - Restore missing `test_install_force_without_existing` method declaration in tests/test_extensions.py so pytest collects it as a separate test. - Move stranded-config restoration to immediately after `copytree`, before command/skill/hook registration, so a failed registration step can't leave preserved configs permanently lost. - Store `(bytes, mode)` tuples instead of bare bytes when rescuing stranded configs, and reapply the original file mode after writing so permission bits (e.g. 0600 for credential files) are faithfully restored. Assisted-by: GitHub Copilot (model: claude-sonnet-4.5, autonomous) --- src/specify_cli/extensions/__init__.py | 17 +++++++++++------ tests/test_extensions.py | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 8f62928387..06c6c04bed 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1409,14 +1409,17 @@ def install_from_directory( # silently discarding the preserved config. We read those files into # memory now and write them back after copytree so the user's values # always win over the packaged defaults. - stranded_configs: dict[str, bytes] = {} + stranded_configs: dict[str, tuple[bytes, int]] = {} if dest_dir.exists() and not self.registry.is_installed(manifest.id): for cfg_file in ( list(dest_dir.glob("*-config.yml")) + list(dest_dir.glob("*-config.local.yml")) ): if cfg_file.is_file() and not cfg_file.is_symlink(): - stranded_configs[cfg_file.name] = cfg_file.read_bytes() + stranded_configs[cfg_file.name] = ( + cfg_file.read_bytes(), + cfg_file.stat().st_mode, + ) # Install extension (dest_dir computed above during self-install guard) if dest_dir.exists(): @@ -1425,6 +1428,12 @@ def install_from_directory( ignore_fn = self._load_extensionignore(source_dir) shutil.copytree(source_dir, dest_dir, ignore=ignore_fn) + # Restore stranded configs rescued before the rmtree above. + for filename, (content, mode) in stranded_configs.items(): + target = dest_dir / filename + target.write_bytes(content) + target.chmod(mode) + # Register commands with AI agents registered_commands = {} if register_commands: @@ -1448,10 +1457,6 @@ def install_from_directory( hook_executor = HookExecutor(self.project_root) hook_executor.register_hooks(manifest) - # Restore stranded configs rescued before the rmtree above. - for filename, content in stranded_configs.items(): - (dest_dir / filename).write_bytes(content) - # Restore config files from backup when --force triggered a removal. # Only restore *.yml config files to match what remove() backs up, # so unexpected artifacts in .backup/ are not resurrected. diff --git a/tests/test_extensions.py b/tests/test_extensions.py index b1d85fdb2d..5c57851e3e 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1282,7 +1282,7 @@ def test_reinstall_after_keep_config_preserves_local_config( assert local_cfg.exists() assert "local_override: true" in local_cfg.read_text() - + def test_install_force_without_existing(self, extension_dir, project_dir): """Test force-install when extension is NOT already installed (works normally).""" manager = ExtensionManager(project_dir) From 0224824fdc89bcc5f97d2a9c568fc9981a5d1062 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:28:30 +0000 Subject: [PATCH 03/33] fix: mask setuid/setgid bits when restoring stranded config file mode Only preserve user/group read-write bits (mode & 0o660) to avoid restoring setuid, setgid, or world-writable permissions from a user-modified config file. Assisted-by: GitHub Copilot (model: claude-sonnet-4.5, autonomous) --- src/specify_cli/extensions/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 06c6c04bed..8c853a09e7 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1432,7 +1432,9 @@ def install_from_directory( for filename, (content, mode) in stranded_configs.items(): target = dest_dir / filename target.write_bytes(content) - target.chmod(mode) + # Mask to only user/group read-write bits to avoid restoring + # setuid/setgid or world-writable permissions. + target.chmod(mode & 0o660) # Register commands with AI agents registered_commands = {} From 0c86359647cb3f3dfcac95126a4154f91698d938 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Jul 2026 19:21:37 +0000 Subject: [PATCH 04/33] fix: add copytree rollback path and strengthen regression test with packaged default config - Wrap shutil.copytree in a try/except BaseException so stranded configs rescued before rmtree are written back even if copytree fails mid-way (addresses review comment: configs were permanently lost on copy failure) - Add a packaged default config to extension_dir in the regression test so a naive 'restore only when absent' implementation would fail; assert the user's customized values beat the packaged defaults after reinstall Assisted-by: GitHub Copilot (model: claude-sonnet-4.6, autonomous) --- src/specify_cli/extensions/__init__.py | 14 +++++++++++++- tests/test_extensions.py | 15 +++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 8c853a09e7..42b583fe89 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1426,7 +1426,19 @@ def install_from_directory( shutil.rmtree(dest_dir) ignore_fn = self._load_extensionignore(source_dir) - shutil.copytree(source_dir, dest_dir, ignore=ignore_fn) + try: + shutil.copytree(source_dir, dest_dir, ignore=ignore_fn) + except BaseException: + # copytree failed — dest_dir may be absent or only partially + # created. Write the rescued configs back now so they are not + # permanently lost even though the install did not complete. + if stranded_configs: + dest_dir.mkdir(parents=True, exist_ok=True) + for filename, (content, mode) in stranded_configs.items(): + target = dest_dir / filename + target.write_bytes(content) + target.chmod(mode & 0o660) + raise # Restore stranded configs rescued before the rmtree above. for filename, (content, mode) in stranded_configs.items(): diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 5c57851e3e..a9508bfb85 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1232,12 +1232,17 @@ def test_reinstall_after_keep_config_preserves_config( """Reinstalling after `remove --keep-config` must not overwrite preserved config.""" manager = ExtensionManager(project_dir) - # Install once + # Add a packaged default config so the reinstall has a file to overwrite. + # Without the fix, the packaged default would silently win on reinstall. + packaged_config = extension_dir / "test-ext-config.yml" + packaged_config.write_text("model: default-model\nmax_iterations: 1\n") + + # Install once (packaged default is copied into the installed directory) manager.install_from_directory( extension_dir, "0.1.0", register_commands=False ) - # Customize the config file + # Overwrite the installed config with user-customized values ext_dir = project_dir / ".specify" / "extensions" / "test-ext" config_file = ext_dir / "test-ext-config.yml" config_file.write_text("model: custom-model\nmax_iterations: 99\n") @@ -1248,15 +1253,17 @@ def test_reinstall_after_keep_config_preserves_config( assert config_file.exists() assert "custom-model" in config_file.read_text() - # Plain reinstall (no --force) + # Plain reinstall (no --force) — packaged default is still present in + # extension_dir, so a naive implementation would overwrite the custom values. manager.install_from_directory( extension_dir, "0.1.0", register_commands=False ) - # Preserved config must survive the reinstall + # Preserved config must survive the reinstall and beat the packaged default assert config_file.exists() assert "custom-model" in config_file.read_text() assert "99" in config_file.read_text() + assert "default-model" not in config_file.read_text() def test_reinstall_after_keep_config_preserves_local_config( self, extension_dir, project_dir From 79bce5493bc2e28329cbbb9c83a74f5e9f534634 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Jul 2026 21:25:49 +0000 Subject: [PATCH 05/33] fix: restore configs with secure atomic writes Assisted-by: GitHub Copilot (model: gpt-5, autonomous) --- src/specify_cli/extensions/__init__.py | 30 ++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 42b583fe89..bc93bfa86e 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1426,6 +1426,28 @@ def install_from_directory( shutil.rmtree(dest_dir) ignore_fn = self._load_extensionignore(source_dir) + + def _restore_stranded_config_file( + target: Path, content: bytes, preserved_mode: int + ) -> None: + mode = preserved_mode & 0o660 + tmp_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + dir=target.parent, + prefix=f".{target.name}.", + delete=False, + ) as tmp: + tmp_path = Path(tmp.name) + tmp_path.chmod(mode) + tmp.write(content) + os.replace(tmp_path, target) + except BaseException: + if tmp_path is not None and tmp_path.exists(): + tmp_path.unlink() + raise + try: shutil.copytree(source_dir, dest_dir, ignore=ignore_fn) except BaseException: @@ -1436,17 +1458,13 @@ def install_from_directory( dest_dir.mkdir(parents=True, exist_ok=True) for filename, (content, mode) in stranded_configs.items(): target = dest_dir / filename - target.write_bytes(content) - target.chmod(mode & 0o660) + _restore_stranded_config_file(target, content, mode) raise # Restore stranded configs rescued before the rmtree above. for filename, (content, mode) in stranded_configs.items(): target = dest_dir / filename - target.write_bytes(content) - # Mask to only user/group read-write bits to avoid restoring - # setuid/setgid or world-writable permissions. - target.chmod(mode & 0o660) + _restore_stranded_config_file(target, content, mode) # Register commands with AI agents registered_commands = {} From 532049b1ee47cafc0d7691f2f6d3f5524ad874d2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:40:53 +0000 Subject: [PATCH 06/33] fix: write secure temp file then chmod to preserved_mode; add copytree-failure test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _restore_stranded_config_file: write content while temp file is at its secure OS-default mode (typically 0600 on POSIX), then apply the original preserved_mode after the file is fully written and before the atomic os.replace. Removes the & 0o660 mask that was silently stripping world-read and executable bits (e.g. 0644 → 0640). - Add test_copytree_failure_restores_stranded_config: patches shutil.copytree to create a partial destination then raise OSError, then asserts that the preserved config bytes and file mode are restored by the rollback path and that the extension remains unregistered. Assisted-by: GitHub Copilot (model: claude-sonnet-4.6, autonomous) --- src/specify_cli/extensions/__init__.py | 3 +- tests/test_extensions.py | 66 ++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index bc93bfa86e..878c4c845d 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1430,7 +1430,6 @@ def install_from_directory( def _restore_stranded_config_file( target: Path, content: bytes, preserved_mode: int ) -> None: - mode = preserved_mode & 0o660 tmp_path: Path | None = None try: with tempfile.NamedTemporaryFile( @@ -1440,8 +1439,8 @@ def _restore_stranded_config_file( delete=False, ) as tmp: tmp_path = Path(tmp.name) - tmp_path.chmod(mode) tmp.write(content) + tmp_path.chmod(preserved_mode) os.replace(tmp_path, target) except BaseException: if tmp_path is not None and tmp_path.exists(): diff --git a/tests/test_extensions.py b/tests/test_extensions.py index a9508bfb85..f27335316d 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1289,6 +1289,72 @@ def test_reinstall_after_keep_config_preserves_local_config( assert local_cfg.exists() assert "local_override: true" in local_cfg.read_text() + def test_copytree_failure_restores_stranded_config( + self, extension_dir, project_dir, monkeypatch + ): + """A copytree failure must not permanently lose a preserved config. + + When copytree raises after the existing directory has been removed, the + rollback path must write the rescued bytes back to dest_dir and restore + the original file mode, while leaving the extension unregistered. + """ + import specify_cli.extensions as _ext_module + import stat + + manager = ExtensionManager(project_dir) + + # Add a packaged default config so copytree would overwrite it on success. + packaged_config = extension_dir / "test-ext-config.yml" + packaged_config.write_text("model: default-model\n") + + # Install once so the extension is on disk. + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + ext_dir = project_dir / ".specify" / "extensions" / "test-ext" + config_file = ext_dir / "test-ext-config.yml" + config_file.write_text("model: custom-model\nmax_iterations: 99\n") + + # Set a known, non-default mode so we can assert it survives the rollback. + if platform.system() != "Windows": + config_file.chmod(0o640) + original_bytes = config_file.read_bytes() + original_mode = config_file.stat().st_mode + + # Remove while preserving the config — it is now a stranded file. + manager.remove("test-ext", keep_config=True) + assert not manager.registry.is_installed("test-ext") + assert config_file.exists() + + # Make copytree create a partial destination then raise so the rollback + # path is exercised. + original_copytree = shutil.copytree + + def failing_copytree(src, dst, **kwargs): + Path(dst).mkdir(parents=True, exist_ok=True) + (Path(dst) / "_partial.txt").write_text("partial") + raise OSError("simulated disk full") + + monkeypatch.setattr(_ext_module.shutil, "copytree", failing_copytree) + + with pytest.raises(OSError, match="simulated disk full"): + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + # The preserved config must have been written back by the rollback path. + assert config_file.exists(), "rollback must recreate the config file" + assert config_file.read_bytes() == original_bytes + + # On POSIX, the original file mode must be faithfully restored. + if platform.system() != "Windows": + restored_mode = config_file.stat().st_mode + assert stat.S_IMODE(restored_mode) == stat.S_IMODE(original_mode) + + # The extension must remain unregistered after the failed install. + assert not manager.registry.is_installed("test-ext") + def test_install_force_without_existing(self, extension_dir, project_dir): """Test force-install when extension is NOT already installed (works normally).""" manager = ExtensionManager(project_dir) From c824cf3347205a9932e90c0bb2bfcaa43cfa0e80 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:56:29 -0500 Subject: [PATCH 07/33] Potential fix for pull request finding 'Unused local variable' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/test_extensions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index f27335316d..e00adb4974 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1329,7 +1329,6 @@ def test_copytree_failure_restores_stranded_config( # Make copytree create a partial destination then raise so the rollback # path is exercised. - original_copytree = shutil.copytree def failing_copytree(src, dst, **kwargs): Path(dst).mkdir(parents=True, exist_ok=True) From fdb9159f39bae5b46ef4b8f052f08384560b292c Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:56:41 -0500 Subject: [PATCH 08/33] Potential fix for pull request finding 'Module is imported with 'import' and 'import from'' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/test_extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index e00adb4974..ee7e6e68da 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -20,6 +20,7 @@ from pathlib import Path from datetime import datetime, timezone from unittest.mock import MagicMock +import specify_cli.extensions as _ext_module from tests.conftest import strip_ansi from specify_cli.extensions import ( @@ -1298,7 +1299,6 @@ def test_copytree_failure_restores_stranded_config( rollback path must write the rescued bytes back to dest_dir and restore the original file mode, while leaving the extension unregistered. """ - import specify_cli.extensions as _ext_module import stat manager = ExtensionManager(project_dir) From 0c782d7d5fdb9abf30f487b219c8d9173c2f0a4b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:24:23 +0000 Subject: [PATCH 09/33] fix: durable staging for stranded configs and import style fix - Stage stranded config files to a durable rescue_staging_dir (extensions_dir/.rescue-staging-) before rmtree so original bytes survive partial rmtree, copytree failure, or partial restore on retry. On retry the staging dir is detected and its content reused instead of whatever mix of packaged defaults and partial restores remains on disk. The staging dir is cleaned up only after every restore succeeds. - Fix CodeQL: change `import specify_cli.extensions as _ext_module` to `from specify_cli import extensions as _ext_module` in test file. Assisted-by: GitHub Copilot (model: claude-sonnet-4.6, autonomous) --- src/specify_cli/extensions/__init__.py | 48 ++++++++++++++++++++++++-- tests/test_extensions.py | 2 +- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 878c4c845d..92fa3bc2cb 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1407,10 +1407,27 @@ def install_from_directory( # the registry but its config files remain in dest_dir. A subsequent # plain (non-force) install would delete that directory unconditionally, # silently discarding the preserved config. We read those files into - # memory now and write them back after copytree so the user's values - # always win over the packaged defaults. + # memory and also write a durable staging copy outside dest_dir so + # that a partial rmtree, failed copytree, or partial restore cannot + # permanently discard the user's original bytes on a retry. The + # staging dir is removed only after every config has been successfully + # restored. stranded_configs: dict[str, tuple[bytes, int]] = {} - if dest_dir.exists() and not self.registry.is_installed(manifest.id): + rescue_staging_dir = self.extensions_dir / f".rescue-staging-{manifest.id}" + + if rescue_staging_dir.is_dir() and not self.registry.is_installed(manifest.id): + # A previous install attempt staged the configs but never + # completed cleanly. Reload from the durable backup so the + # original bytes are used on retry rather than whatever + # mixture of packaged defaults and partial restores remains + # on disk. + for staged_file in sorted(rescue_staging_dir.iterdir()): + if staged_file.is_file(): + stranded_configs[staged_file.name] = ( + staged_file.read_bytes(), + staged_file.stat().st_mode, + ) + elif dest_dir.exists() and not self.registry.is_installed(manifest.id): for cfg_file in ( list(dest_dir.glob("*-config.yml")) + list(dest_dir.glob("*-config.local.yml")) @@ -1421,6 +1438,26 @@ def install_from_directory( cfg_file.stat().st_mode, ) + if stranded_configs and not rescue_staging_dir.is_dir(): + # Write a durable backup outside dest_dir before any + # destructive operation so the original bytes survive a + # crash or partial failure at any later step. The staging + # dir is cleaned up only after every restore succeeds. + try: + rescue_staging_dir.mkdir(parents=True, exist_ok=True) + for filename, (content, mode) in stranded_configs.items(): + staged = rescue_staging_dir / filename + staged.write_bytes(content) + try: + staged.chmod(mode) + except (NotImplementedError, OSError): + pass + except BaseException: + # Staging failed — clean up any partial marker dir and + # fall back to in-memory protection only (same as the + # pre-staging behaviour). + shutil.rmtree(rescue_staging_dir, ignore_errors=True) + # Install extension (dest_dir computed above during self-install guard) if dest_dir.exists(): shutil.rmtree(dest_dir) @@ -1465,6 +1502,11 @@ def _restore_stranded_config_file( target = dest_dir / filename _restore_stranded_config_file(target, content, mode) + # Every stranded config has been restored successfully — the + # durable staging backup is no longer needed. + if rescue_staging_dir.is_dir(): + shutil.rmtree(rescue_staging_dir, ignore_errors=True) + # Register commands with AI agents registered_commands = {} if register_commands: diff --git a/tests/test_extensions.py b/tests/test_extensions.py index ee7e6e68da..ecc94d45b5 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -20,7 +20,7 @@ from pathlib import Path from datetime import datetime, timezone from unittest.mock import MagicMock -import specify_cli.extensions as _ext_module +from specify_cli import extensions as _ext_module from tests.conftest import strip_ansi from specify_cli.extensions import ( From 6106e0e02c214ec62a3e3c7a211cfb59f6d443e5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:39:18 +0000 Subject: [PATCH 10/33] fix: harden rescue staging dir - symlink checks, secure writes, cleanup errors - Thread 14: Change except BaseException to except Exception in the staging fallback block so KeyboardInterrupt/SystemExit propagate correctly - Thread 15: Add explanatory comment to the bare pass in the chmod except block to satisfy static analysis - Thread 16: Reject a symlinked staging directory and only reload non-symlinked files whose names match the two recognised config suffixes - Thread 17: Create each staging file via os.open with mode 0600 and O_CREAT|O_EXCL before writing so preserved bytes are never transiently exposed to other local users - Thread 18: Remove ignore_errors=True from the final staging-dir cleanup so a failed rmtree propagates rather than silently leaving a stale backup that could be misread on the next retry Assisted-by: GitHub Copilot (model: claude-sonnet-4.6, autonomous) --- src/specify_cli/extensions/__init__.py | 35 ++++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 92fa3bc2cb..6e9f313251 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1415,14 +1415,24 @@ def install_from_directory( stranded_configs: dict[str, tuple[bytes, int]] = {} rescue_staging_dir = self.extensions_dir / f".rescue-staging-{manifest.id}" - if rescue_staging_dir.is_dir() and not self.registry.is_installed(manifest.id): + if ( + rescue_staging_dir.is_dir() + and not rescue_staging_dir.is_symlink() + and not self.registry.is_installed(manifest.id) + ): # A previous install attempt staged the configs but never # completed cleanly. Reload from the durable backup so the # original bytes are used on retry rather than whatever # mixture of packaged defaults and partial restores remains - # on disk. + # on disk. Only load non-symlinked files whose names match + # the two recognised config suffixes so a tampered staging + # directory cannot inject arbitrary files. for staged_file in sorted(rescue_staging_dir.iterdir()): - if staged_file.is_file(): + if ( + staged_file.is_file() + and not staged_file.is_symlink() + and staged_file.name.endswith(("-config.yml", "-config.local.yml")) + ): stranded_configs[staged_file.name] = ( staged_file.read_bytes(), staged_file.stat().st_mode, @@ -1447,12 +1457,19 @@ def install_from_directory( rescue_staging_dir.mkdir(parents=True, exist_ok=True) for filename, (content, mode) in stranded_configs.items(): staged = rescue_staging_dir / filename - staged.write_bytes(content) + # Create the staging file with mode 0600 before writing so + # the preserved bytes are never transiently readable by other + # local users, even on a umask that would produce 0644. + fd = os.open(str(staged), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + os.write(fd, content) + finally: + os.close(fd) try: staged.chmod(mode) except (NotImplementedError, OSError): - pass - except BaseException: + pass # Best-effort; chmod may not be supported on all platforms. + except Exception: # Staging failed — clean up any partial marker dir and # fall back to in-memory protection only (same as the # pre-staging behaviour). @@ -1503,9 +1520,11 @@ def _restore_stranded_config_file( _restore_stranded_config_file(target, content, mode) # Every stranded config has been restored successfully — the - # durable staging backup is no longer needed. + # durable staging backup is no longer needed. Raise on failure so + # the install is not reported as successful while a stale backup + # that could be misread on the next retry remains on disk. if rescue_staging_dir.is_dir(): - shutil.rmtree(rescue_staging_dir, ignore_errors=True) + shutil.rmtree(rescue_staging_dir) # Register commands with AI agents registered_commands = {} From 3cca6e10cdf657d3d23852be2ce5ee322f051209 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:07:48 +0000 Subject: [PATCH 11/33] fix: mask file-type bits from chmod, harden staging dir symlink check - Add `import stat` to imports - Use `stat.S_IMODE(mode)` before chmod in staging write (thread 20, line 1464) - Use `stat.S_IMODE(preserved_mode)` and make chmod best-effort in `_restore_stranded_config_file` (thread 18, line 1492) - Add `not rescue_staging_dir.is_symlink()` guard to cleanup (thread 19, line 1522) Assisted-by: GitHub Copilot (model: claude-sonnet-4.6, autonomous) --- src/specify_cli/extensions/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 6e9f313251..4fe05173cd 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -14,6 +14,7 @@ import os import re import shutil +import stat import tempfile import zipfile from dataclasses import dataclass @@ -1466,7 +1467,7 @@ def install_from_directory( finally: os.close(fd) try: - staged.chmod(mode) + staged.chmod(stat.S_IMODE(mode)) except (NotImplementedError, OSError): pass # Best-effort; chmod may not be supported on all platforms. except Exception: @@ -1494,7 +1495,10 @@ def _restore_stranded_config_file( ) as tmp: tmp_path = Path(tmp.name) tmp.write(content) - tmp_path.chmod(preserved_mode) + try: + tmp_path.chmod(stat.S_IMODE(preserved_mode)) + except (NotImplementedError, OSError): + pass # Best-effort; chmod may not be supported on all platforms. os.replace(tmp_path, target) except BaseException: if tmp_path is not None and tmp_path.exists(): @@ -1523,7 +1527,7 @@ def _restore_stranded_config_file( # durable staging backup is no longer needed. Raise on failure so # the install is not reported as successful while a stale backup # that could be misread on the next retry remains on disk. - if rescue_staging_dir.is_dir(): + if rescue_staging_dir.is_dir() and not rescue_staging_dir.is_symlink(): shutil.rmtree(rescue_staging_dir) # Register commands with AI agents From a76c1818519e67ce63248ea4fc7bffeeda256046 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:30:18 +0000 Subject: [PATCH 12/33] fix: use completion marker for rescue staging, abort on staging failure, full os.write Assisted-by: GitHub Copilot (model: GPT-5.3-Codex, autonomous) --- src/specify_cli/extensions/__init__.py | 62 +++++++++++++++++++++----- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 4fe05173cd..234f9f0cf5 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1415,12 +1415,20 @@ def install_from_directory( # restored. stranded_configs: dict[str, tuple[bytes, int]] = {} rescue_staging_dir = self.extensions_dir / f".rescue-staging-{manifest.id}" - - if ( + # A staging directory is trusted only when this completion marker is + # present. The marker is written after every staged file is complete + # and removed before the non-atomic cleanup, so a crash mid-staging or + # mid-cleanup can never leave a partial directory that a retry mistakes + # for a complete durable backup. + rescue_complete_marker = rescue_staging_dir / ".rescue-complete" + staging_is_complete = ( rescue_staging_dir.is_dir() and not rescue_staging_dir.is_symlink() - and not self.registry.is_installed(manifest.id) - ): + and rescue_complete_marker.is_file() + and not rescue_complete_marker.is_symlink() + ) + + if staging_is_complete and not self.registry.is_installed(manifest.id): # A previous install attempt staged the configs but never # completed cleanly. Reload from the durable backup so the # original bytes are used on retry rather than whatever @@ -1449,11 +1457,21 @@ def install_from_directory( cfg_file.stat().st_mode, ) - if stranded_configs and not rescue_staging_dir.is_dir(): + if stranded_configs and not staging_is_complete: # Write a durable backup outside dest_dir before any # destructive operation so the original bytes survive a # crash or partial failure at any later step. The staging # dir is cleaned up only after every restore succeeds. + # + # Any pre-existing staging dir here lacks the completion marker + # (staging_is_complete is False), so it is a stale partial from an + # interrupted attempt — remove it first for a clean write. + if rescue_staging_dir.is_symlink(): + rescue_staging_dir.unlink() + elif rescue_staging_dir.is_dir(): + shutil.rmtree(rescue_staging_dir) + elif rescue_staging_dir.exists(): + rescue_staging_dir.unlink() try: rescue_staging_dir.mkdir(parents=True, exist_ok=True) for filename, (content, mode) in stranded_configs.items(): @@ -1463,18 +1481,38 @@ def install_from_directory( # local users, even on a umask that would produce 0644. fd = os.open(str(staged), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: - os.write(fd, content) + # os.write() may write fewer bytes than requested, so + # loop until the whole buffer is on disk — a truncated + # "durable" backup would be trusted over the intact + # config on a retry and cause silent data loss. + view = memoryview(content) + written = 0 + while written < len(view): + written += os.write(fd, view[written:]) finally: os.close(fd) try: staged.chmod(stat.S_IMODE(mode)) except (NotImplementedError, OSError): pass # Best-effort; chmod may not be supported on all platforms. - except Exception: - # Staging failed — clean up any partial marker dir and - # fall back to in-memory protection only (same as the - # pre-staging behaviour). + # Write the completion marker only after every staged file is + # fully written so a retry trusts staging only when it is whole. + marker_fd = os.open( + str(rescue_complete_marker), + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + ) + os.close(marker_fd) + except BaseException: + # Durable staging failed (or was interrupted). Continuing with + # only the in-memory copy would reintroduce the permanent-loss + # path this staging exists to close: the rmtree below could + # delete the originals and a later restore failure would leave + # no on-disk copy. dest_dir is still untouched here, so clean + # up the partial staging dir and abort the install instead of + # proceeding destructively. shutil.rmtree(rescue_staging_dir, ignore_errors=True) + raise # Install extension (dest_dir computed above during self-install guard) if dest_dir.exists(): @@ -1528,6 +1566,10 @@ def _restore_stranded_config_file( # the install is not reported as successful while a stale backup # that could be misread on the next retry remains on disk. if rescue_staging_dir.is_dir() and not rescue_staging_dir.is_symlink(): + # Remove the completion marker before the non-atomic rmtree so a + # crash mid-cleanup cannot leave a staging dir that a retry would + # wrongly trust as a complete durable backup. + rescue_complete_marker.unlink(missing_ok=True) shutil.rmtree(rescue_staging_dir) # Register commands with AI agents From 68e965c0c66f3088a0265c73791b6735fa07afec Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:31:36 -0500 Subject: [PATCH 13/33] fix(extensions): make rescue staging durable Assisted-by: GitHub Copilot (model: MAI-Code-1-Flash, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/extensions/__init__.py | 52 ++++++++++++++++++-- tests/test_extensions.py | 68 ++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 5 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 234f9f0cf5..3d75e8c5dc 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -102,6 +102,34 @@ def _load_core_command_names() -> frozenset[str]: CORE_COMMAND_NAMES = _load_core_command_names() +def _fsync_file(path: Path) -> None: + """Best-effort fsync for a regular file.""" + try: + with open(path, "rb") as handle: + os.fsync(handle.fileno()) + except (AttributeError, OSError, NotImplementedError): + pass + + +def _fsync_directory(path: Path) -> None: + """Best-effort fsync for a directory path.""" + if not path.exists(): + return + try: + dir_fd = os.open(str(path), os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + except (AttributeError, OSError, NotImplementedError): + try: + dir_fd = os.open(str(path), os.O_RDONLY) + except (AttributeError, OSError, NotImplementedError): + return + try: + os.fsync(dir_fd) + except (AttributeError, OSError, NotImplementedError): + pass + finally: + os.close(dir_fd) + + class ExtensionError(Exception): """Base exception for extension-related errors.""" @@ -1489,12 +1517,19 @@ def install_from_directory( written = 0 while written < len(view): written += os.write(fd, view[written:]) + try: + os.fchmod(fd, stat.S_IMODE(mode)) + except (AttributeError, NotImplementedError, OSError): + try: + staged.chmod(stat.S_IMODE(mode)) + except (NotImplementedError, OSError): + pass # Best-effort; chmod may not be supported on all platforms. + try: + os.fsync(fd) + except (AttributeError, OSError, NotImplementedError): + pass finally: os.close(fd) - try: - staged.chmod(stat.S_IMODE(mode)) - except (NotImplementedError, OSError): - pass # Best-effort; chmod may not be supported on all platforms. # Write the completion marker only after every staged file is # fully written so a retry trusts staging only when it is whole. marker_fd = os.open( @@ -1502,7 +1537,14 @@ def install_from_directory( os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600, ) - os.close(marker_fd) + try: + os.fsync(marker_fd) + except (AttributeError, OSError, NotImplementedError): + pass + finally: + os.close(marker_fd) + _fsync_directory(rescue_staging_dir) + _fsync_directory(rescue_staging_dir.parent) except BaseException: # Durable staging failed (or was interrupted). Continuing with # only the in-memory copy would reintroduce the permanent-loss diff --git a/tests/test_extensions.py b/tests/test_extensions.py index ecc94d45b5..001307fb9c 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1354,6 +1354,74 @@ def failing_copytree(src, dst, **kwargs): # The extension must remain unregistered after the failed install. assert not manager.registry.is_installed("test-ext") + def test_retry_after_staging_backup_restores_stranded_config( + self, extension_dir, project_dir, monkeypatch + ): + """A retry after an interrupted install should restore the rescued config from staging.""" + import stat + + manager = ExtensionManager(project_dir) + + packaged_config = extension_dir / "test-ext-config.yml" + packaged_config.write_text("model: default-model\n") + + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + ext_dir = project_dir / ".specify" / "extensions" / "test-ext" + config_file = ext_dir / "test-ext-config.yml" + config_file.write_text("model: custom-model\nmax_iterations: 99\n") + + if platform.system() != "Windows": + config_file.chmod(0o640) + original_bytes = config_file.read_bytes() + original_mode = config_file.stat().st_mode + + manager.remove("test-ext", keep_config=True) + assert not manager.registry.is_installed("test-ext") + assert config_file.exists() + + staging_dir = manager.extensions_dir / ".rescue-staging-test-ext" + assert not staging_dir.exists() + + original_copytree = shutil.copytree + copytree_calls = 0 + + def flaky_copytree(src, dst, **kwargs): + nonlocal copytree_calls + copytree_calls += 1 + if copytree_calls == 1: + Path(dst).mkdir(parents=True, exist_ok=True) + (Path(dst) / "_partial.txt").write_text("partial") + raise OSError("simulated disk full") + return original_copytree(src, dst, **kwargs) + + monkeypatch.setattr(_ext_module.shutil, "copytree", flaky_copytree) + + with pytest.raises(OSError, match="simulated disk full"): + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + assert staging_dir.exists() + assert (staging_dir / ".rescue-complete").exists() + assert (staging_dir / "test-ext-config.yml").exists() + + manifest = manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + assert manifest.id == "test-ext" + assert manager.registry.is_installed("test-ext") + assert config_file.read_bytes() == original_bytes + assert not (ext_dir / "_partial.txt").exists() + assert not staging_dir.exists() + + if platform.system() != "Windows": + restored_mode = config_file.stat().st_mode + assert stat.S_IMODE(restored_mode) == stat.S_IMODE(original_mode) + def test_install_force_without_existing(self, extension_dir, project_dir): """Test force-install when extension is NOT already installed (works normally).""" manager = ExtensionManager(project_dir) From 86f93b6505c606c7f69852b9eb3823dfc957b542 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:32:19 -0500 Subject: [PATCH 14/33] test(extensions): fix flaky copytree regression test Assisted-by: GitHub Copilot (model: MAI-Code-1-Flash, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_extensions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 001307fb9c..9fbc20340d 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1388,14 +1388,15 @@ def test_retry_after_staging_backup_restores_stranded_config( original_copytree = shutil.copytree copytree_calls = 0 - def flaky_copytree(src, dst, **kwargs): + def flaky_copytree(*args, **kwargs): nonlocal copytree_calls copytree_calls += 1 if copytree_calls == 1: + dst = args[1] Path(dst).mkdir(parents=True, exist_ok=True) (Path(dst) / "_partial.txt").write_text("partial") raise OSError("simulated disk full") - return original_copytree(src, dst, **kwargs) + return original_copytree(*args, **kwargs) monkeypatch.setattr(_ext_module.shutil, "copytree", flaky_copytree) From 9be19beb0b8fe204795c13cc928626b425d8d450 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:49:34 -0500 Subject: [PATCH 15/33] test(extensions): fix module import alias for review feedback Assisted-by: GitHub Copilot (model: MAI-Code-1-Flash, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 9fbc20340d..8aa623cc31 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -20,7 +20,7 @@ from pathlib import Path from datetime import datetime, timezone from unittest.mock import MagicMock -from specify_cli import extensions as _ext_module +import specify_cli.extensions as _ext_module from tests.conftest import strip_ansi from specify_cli.extensions import ( From 7030d91e80daed75707ed7ed15a7b1078e3f9f81 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:12:16 -0500 Subject: [PATCH 16/33] fix(workflows): keep cleanup warnings single-line and remove dead helper Assisted-by: GitHub Copilot (model: MAI-Code-1-Flash, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/extensions/__init__.py | 9 --------- src/specify_cli/workflows/_commands.py | 6 ++---- .../extensions/test_update_agent_context_feature_json.py | 1 - 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 3d75e8c5dc..184d8dece8 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -102,15 +102,6 @@ def _load_core_command_names() -> frozenset[str]: CORE_COMMAND_NAMES = _load_core_command_names() -def _fsync_file(path: Path) -> None: - """Best-effort fsync for a regular file.""" - try: - with open(path, "rb") as handle: - os.fsync(handle.fileno()) - except (AttributeError, OSError, NotImplementedError): - pass - - def _fsync_directory(path: Path) -> None: """Best-effort fsync for a directory path.""" if not path.exists(): diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 9ab199c023..f14433d1f6 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -1666,8 +1666,7 @@ def _validate_and_install_local( except OSError as cleanup_exc: console.print( "[yellow]Warning:[/yellow] Could not remove temporary " - f"download file {_escape_markup(str(tmp_path))}: " - f"{_escape_markup(str(cleanup_exc))}" + f"workflow download file: {_escape_markup(str(cleanup_exc))}" ) console.print(f"[red]Error:[/red] Failed to download workflow: {_escape_markup(str(exc))}") raise typer.Exit(1) @@ -1691,8 +1690,7 @@ def _validate_and_install_local( except OSError as exc: console.print( "[yellow]Warning:[/yellow] Could not remove temporary " - f"download file {_escape_markup(str(tmp_path))}: " - f"{_escape_markup(str(exc))}" + f"workflow download file: {_escape_markup(str(exc))}" ) return diff --git a/tests/extensions/test_update_agent_context_feature_json.py b/tests/extensions/test_update_agent_context_feature_json.py index 957415708c..25b5ff9457 100644 --- a/tests/extensions/test_update_agent_context_feature_json.py +++ b/tests/extensions/test_update_agent_context_feature_json.py @@ -11,7 +11,6 @@ from tests.conftest import requires_bash from tests.extensions.test_extension_agent_context import ( - BASH, POWERSHELL, _bash_posix_path, _run_bash_agent_context_script, From 02bc8d2a2d2a001bb22e7ae111ca19b32d627990 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:13:55 -0500 Subject: [PATCH 17/33] Potential fix for pull request finding 'Module is imported with 'import' and 'import from'' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/test_extensions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 8aa623cc31..9a21dc8b21 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -20,7 +20,6 @@ from pathlib import Path from datetime import datetime, timezone from unittest.mock import MagicMock -import specify_cli.extensions as _ext_module from tests.conftest import strip_ansi from specify_cli.extensions import ( From a80ac2b7fe19471ae538613ff1191316a3641403 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:36:45 -0500 Subject: [PATCH 18/33] Preserve rescued extension config across retry Assisted-by: GitHub Copilot (model: MAI-Code-1-Flash, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/extensions/__init__.py | 63 ++++++++++++++++++++------ tests/test_extensions.py | 7 +++ 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 184d8dece8..bf38619f2d 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -9,6 +9,7 @@ from __future__ import annotations import copy +import errno import hashlib import json import os @@ -102,23 +103,46 @@ def _load_core_command_names() -> frozenset[str]: CORE_COMMAND_NAMES = _load_core_command_names() +def _fsync_fd(fd: int) -> None: + """Sync a file descriptor, raising on real storage errors.""" + try: + os.fsync(fd) + except AttributeError: + return + except NotImplementedError: + return + except OSError as exc: + if exc.errno in {errno.ENOTSUP, errno.EOPNOTSUPP, errno.EINVAL, errno.EBADF}: + return + raise + + def _fsync_directory(path: Path) -> None: - """Best-effort fsync for a directory path.""" + """Sync a directory when the platform supports it.""" if not path.exists(): return try: dir_fd = os.open(str(path), os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) - except (AttributeError, OSError, NotImplementedError): + except (AttributeError, NotImplementedError): + return + except OSError as exc: + if exc.errno in {errno.ENOTSUP, errno.EOPNOTSUPP, errno.EINVAL, errno.EBADF}: + return try: dir_fd = os.open(str(path), os.O_RDONLY) - except (AttributeError, OSError, NotImplementedError): + except (AttributeError, NotImplementedError): return + except OSError as exc2: + if exc2.errno in {errno.ENOTSUP, errno.EOPNOTSUPP, errno.EINVAL, errno.EBADF}: + return + raise try: - os.fsync(dir_fd) - except (AttributeError, OSError, NotImplementedError): - pass + _fsync_fd(dir_fd) finally: - os.close(dir_fd) + try: + os.close(dir_fd) + except OSError: + pass class ExtensionError(Exception): @@ -1515,10 +1539,7 @@ def install_from_directory( staged.chmod(stat.S_IMODE(mode)) except (NotImplementedError, OSError): pass # Best-effort; chmod may not be supported on all platforms. - try: - os.fsync(fd) - except (AttributeError, OSError, NotImplementedError): - pass + _fsync_fd(fd) finally: os.close(fd) # Write the completion marker only after every staged file is @@ -1529,9 +1550,7 @@ def install_from_directory( 0o600, ) try: - os.fsync(marker_fd) - except (AttributeError, OSError, NotImplementedError): - pass + _fsync_fd(marker_fd) finally: os.close(marker_fd) _fsync_directory(rescue_staging_dir) @@ -1566,11 +1585,27 @@ def _restore_stranded_config_file( ) as tmp: tmp_path = Path(tmp.name) tmp.write(content) + tmp.flush() + _fsync_fd(tmp.fileno()) try: tmp_path.chmod(stat.S_IMODE(preserved_mode)) except (NotImplementedError, OSError): pass # Best-effort; chmod may not be supported on all platforms. os.replace(tmp_path, target) + try: + target_fd = os.open(str(target), os.O_RDONLY) + except (AttributeError, OSError, NotImplementedError): + target_fd = None + try: + if target_fd is not None: + _fsync_fd(target_fd) + finally: + if target_fd is not None: + try: + os.close(target_fd) + except OSError: + pass + _fsync_directory(target.parent) except BaseException: if tmp_path is not None and tmp_path.exists(): tmp_path.unlink() diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 9a21dc8b21..2dda61c50e 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -22,6 +22,7 @@ from unittest.mock import MagicMock from tests.conftest import strip_ansi +import specify_cli.extensions as _ext_module from specify_cli.extensions import ( CatalogEntry, CORE_COMMAND_NAMES, @@ -1408,6 +1409,12 @@ def flaky_copytree(*args, **kwargs): assert (staging_dir / ".rescue-complete").exists() assert (staging_dir / "test-ext-config.yml").exists() + # Corrupt the rollback-restored copy to prove the next retry must rely + # on the durable staging backup, not whatever was written to dest_dir + # by the earlier failed install attempt. + config_file.write_text("model: wrong-model\n", encoding="utf-8") + assert config_file.read_bytes() != original_bytes + manifest = manager.install_from_directory( extension_dir, "0.1.0", register_commands=False ) From eb28ce27f1e11afec3b0af65a8ef0955804a188f Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:02:08 -0500 Subject: [PATCH 19/33] Clarify ignored directory fsync cleanup errors Assisted-by: GitHub Copilot (model: MAI-Code-1-Flash, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/extensions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index bf38619f2d..428cd74f9b 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -142,6 +142,7 @@ def _fsync_directory(path: Path) -> None: try: os.close(dir_fd) except OSError: + # Cleanup after an fsync failure should not mask the original error. pass From c870beda520cd3eacd97c2be2b9864f85cd4bc7c Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:23:31 -0500 Subject: [PATCH 20/33] Fix reinstall durability and workflow cleanup warnings Assisted-by: GitHub Copilot (model: MAI-Code-1-Flash, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/extensions/__init__.py | 8 ++++++++ src/specify_cli/workflows/_commands.py | 6 ++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 428cd74f9b..aa5d0357c1 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -121,6 +121,8 @@ def _fsync_directory(path: Path) -> None: """Sync a directory when the platform supports it.""" if not path.exists(): return + if os.name == "nt": + return try: dir_fd = os.open(str(path), os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) except (AttributeError, NotImplementedError): @@ -1543,6 +1545,10 @@ def install_from_directory( _fsync_fd(fd) finally: os.close(fd) + # Flush the staging directory metadata before publishing the + # completion marker so a crash cannot leave a visible marker with + # only a subset of staged files. + _fsync_directory(rescue_staging_dir) # Write the completion marker only after every staged file is # fully written so a retry trusts staging only when it is whole. marker_fd = os.open( @@ -1639,7 +1645,9 @@ def _restore_stranded_config_file( # crash mid-cleanup cannot leave a staging dir that a retry would # wrongly trust as a complete durable backup. rescue_complete_marker.unlink(missing_ok=True) + _fsync_directory(rescue_staging_dir) shutil.rmtree(rescue_staging_dir) + _fsync_directory(rescue_staging_dir.parent) # Register commands with AI agents registered_commands = {} diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index f14433d1f6..080c39b3a2 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -1666,7 +1666,8 @@ def _validate_and_install_local( except OSError as cleanup_exc: console.print( "[yellow]Warning:[/yellow] Could not remove temporary " - f"workflow download file: {_escape_markup(str(cleanup_exc))}" + f"workflow download file: {_escape_markup(str(cleanup_exc))} " + f"(path: {_escape_markup(str(tmp_path))})" ) console.print(f"[red]Error:[/red] Failed to download workflow: {_escape_markup(str(exc))}") raise typer.Exit(1) @@ -1690,7 +1691,8 @@ def _validate_and_install_local( except OSError as exc: console.print( "[yellow]Warning:[/yellow] Could not remove temporary " - f"workflow download file: {_escape_markup(str(exc))}" + f"workflow download file: {_escape_markup(str(exc))} " + f"(path: {_escape_markup(str(tmp_path))})" ) return From e942583f8a871431a3ff4d7f91cf8c4d3de5b89e Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:10:26 -0500 Subject: [PATCH 21/33] Open rescue staging file in binary mode to fix Windows CRLF corruption On Windows os.open() defaults to text mode, so os.write() of preserved config bytes containing \r\n was translated to \r\r\n, corrupting the staged backup and failing the retry-restore regression test. Add O_BINARY (0 on POSIX) to the staging file open flags. Assisted-by: GitHub Copilot (model: Claude Opus 4.8, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/extensions/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index aa5d0357c1..7b85054630 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1525,7 +1525,14 @@ def install_from_directory( # Create the staging file with mode 0600 before writing so # the preserved bytes are never transiently readable by other # local users, even on a umask that would produce 0644. - fd = os.open(str(staged), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + # O_BINARY (0 on POSIX) is required so Windows does not open + # the descriptor in text mode and translate the preserved + # bytes' "\n" into "\r\n" as they are written. + fd = os.open( + str(staged), + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0), + 0o600, + ) try: # os.write() may write fewer bytes than requested, so # loop until the whole buffer is on disk — a truncated From 1f01cefc7d8bf04594223faca06ec636a4c8cc45 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:29:19 -0500 Subject: [PATCH 22/33] Load .extensionignore before deleting dest_dir on reinstall The .extensionignore loader can raise ValidationError (invalid UTF-8) or OSError. Previously it ran after dest_dir was removed, so such a failure left the kept config only in the hidden staging directory rather than its documented location. Load/validate it before the rmtree so every post-deletion failure path restores the config. Adds a regression test. Assisted-by: GitHub Copilot (model: Claude Opus 4.8, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/extensions/__init__.py | 8 +++-- tests/test_extensions.py | 42 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 7b85054630..c02cef02f5 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1580,12 +1580,16 @@ def install_from_directory( shutil.rmtree(rescue_staging_dir, ignore_errors=True) raise + # Load and validate .extensionignore BEFORE deleting dest_dir. The + # loader can raise ValidationError (invalid UTF-8) or OSError; doing it + # first means such a failure leaves the kept config untouched in its + # documented location rather than only in the hidden staging directory. + ignore_fn = self._load_extensionignore(source_dir) + # Install extension (dest_dir computed above during self-install guard) if dest_dir.exists(): shutil.rmtree(dest_dir) - ignore_fn = self._load_extensionignore(source_dir) - def _restore_stranded_config_file( target: Path, content: bytes, preserved_mode: int ) -> None: diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 2dda61c50e..c11084ebef 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1354,6 +1354,48 @@ def failing_copytree(src, dst, **kwargs): # The extension must remain unregistered after the failed install. assert not manager.registry.is_installed("test-ext") + def test_extensionignore_load_failure_preserves_kept_config( + self, extension_dir, project_dir + ): + """An .extensionignore load failure must not lose a preserved config. + + `.extensionignore` is loaded/validated before dest_dir is deleted, so a + ValidationError raised for invalid UTF-8 must abort the reinstall while + leaving the kept config untouched in its documented location rather than + only in the hidden staging directory. + """ + manager = ExtensionManager(project_dir) + + packaged_config = extension_dir / "test-ext-config.yml" + packaged_config.write_text("model: default-model\n") + + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + ext_dir = project_dir / ".specify" / "extensions" / "test-ext" + config_file = ext_dir / "test-ext-config.yml" + config_file.write_text("model: custom-model\nmax_iterations: 99\n") + original_bytes = config_file.read_bytes() + + manager.remove("test-ext", keep_config=True) + assert not manager.registry.is_installed("test-ext") + assert config_file.exists() + + # Author an .extensionignore that is not valid UTF-8 so the loader + # raises before dest_dir would be deleted. + (extension_dir / ".extensionignore").write_bytes(b"\xff\xfe invalid\n") + + with pytest.raises(ValidationError, match="not valid UTF-8"): + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + # The kept config must remain in its documented location, untouched. + assert config_file.exists(), "config must survive the ignore-load failure" + assert config_file.read_bytes() == original_bytes + assert not manager.registry.is_installed("test-ext") + def test_retry_after_staging_backup_restores_stranded_config( self, extension_dir, project_dir, monkeypatch ): From 57d90dc937b13ade98f7105d1e2f9e669b37c353 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Wed, 15 Jul 2026 07:27:20 -0500 Subject: [PATCH 23/33] Validate .extensionignore before publishing rescue staging Loading .extensionignore after the rescue staging directory was published meant a validation failure left a complete staging copy behind. A later retry (after the user fixed the ignore file and edited the kept config) would reload the stale staged bytes and silently overwrite the newer config. Move the loader ahead of reading/creating rescue staging so a failure aborts while the kept config is still authoritative on disk, and extend the regression test to prove no staging is published and a retry adopts the newer bytes. Assisted-by: GitHub Copilot (model: Claude Opus 4.8, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/extensions/__init__.py | 16 ++++++++------ tests/test_extensions.py | 30 ++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index c02cef02f5..875230d568 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1449,6 +1449,16 @@ def install_from_directory( backup_config_dir.unlink() did_remove = self.remove(manifest.id) + # Load and validate .extensionignore BEFORE reading/creating the rescue + # staging directory (and thus before deleting dest_dir). The loader can + # raise ValidationError (invalid UTF-8) or OSError; doing it first means + # such a failure aborts while the kept config is still authoritative in + # its documented location, rather than leaving a freshly published + # staging copy that a later retry (after the user edits the kept config) + # would reload and use to overwrite the newer bytes. Any staging left by + # an earlier destructive attempt is intentionally left intact here. + ignore_fn = self._load_extensionignore(source_dir) + # Rescue any config files left behind by a prior `remove --keep-config`. # When an extension is removed with --keep-config, it is no longer in # the registry but its config files remain in dest_dir. A subsequent @@ -1580,12 +1590,6 @@ def install_from_directory( shutil.rmtree(rescue_staging_dir, ignore_errors=True) raise - # Load and validate .extensionignore BEFORE deleting dest_dir. The - # loader can raise ValidationError (invalid UTF-8) or OSError; doing it - # first means such a failure leaves the kept config untouched in its - # documented location rather than only in the hidden staging directory. - ignore_fn = self._load_extensionignore(source_dir) - # Install extension (dest_dir computed above during self-install guard) if dest_dir.exists(): shutil.rmtree(dest_dir) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index c11084ebef..87c8a63aba 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1359,10 +1359,13 @@ def test_extensionignore_load_failure_preserves_kept_config( ): """An .extensionignore load failure must not lose a preserved config. - `.extensionignore` is loaded/validated before dest_dir is deleted, so a + `.extensionignore` is loaded/validated before the rescue staging + directory is read or created (and thus before dest_dir is deleted), so a ValidationError raised for invalid UTF-8 must abort the reinstall while - leaving the kept config untouched in its documented location rather than - only in the hidden staging directory. + leaving the kept config authoritative in its documented location. It must + NOT publish a stale staging copy that a later retry — after the user + fixes the ignore file and edits the kept config — would reload and use to + silently overwrite the newer bytes. """ manager = ExtensionManager(project_dir) @@ -1383,7 +1386,7 @@ def test_extensionignore_load_failure_preserves_kept_config( assert config_file.exists() # Author an .extensionignore that is not valid UTF-8 so the loader - # raises before dest_dir would be deleted. + # raises before rescue staging is read or created. (extension_dir / ".extensionignore").write_bytes(b"\xff\xfe invalid\n") with pytest.raises(ValidationError, match="not valid UTF-8"): @@ -1396,6 +1399,25 @@ def test_extensionignore_load_failure_preserves_kept_config( assert config_file.read_bytes() == original_bytes assert not manager.registry.is_installed("test-ext") + # No rescue staging may have been published, so a later retry reads the + # live (possibly newly edited) config rather than stale staged bytes. + staging_dir = manager.extensions_dir / ".rescue-staging-test-ext" + assert not staging_dir.exists() + + # Simulate the user fixing the ignore file and editing the kept config, + # then retrying: the retry must adopt the newer bytes, never a stale + # staged copy. + (extension_dir / ".extensionignore").write_text("*.log\n") + config_file.write_text("model: newer-model\n") + newer_bytes = config_file.read_bytes() + + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + assert manager.registry.is_installed("test-ext") + assert config_file.read_bytes() == newer_bytes + def test_retry_after_staging_backup_restores_stranded_config( self, extension_dir, project_dir, monkeypatch ): From defdd7cfdbf958e776fac4a334161ef8d5dc95c6 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:00:51 -0500 Subject: [PATCH 24/33] Harden preserved-config rescue against divergence and long names Address three review findings on the reinstall config-rescue path: - A complete .rescue-complete marker proves only that staging finished, not that dest_dir was modified. A crash after staging sync but before the rmtree leaves the live kept config intact; if the user edits it before retrying, preferring the staged bytes silently overwrote the newer config. The two copies are indistinguishable in provenance from disk, so detect divergence between a complete staging copy and the live config and abort (preserving both) instead of unconditionally choosing staging. - The staging directory embedded the full extension ID in one path component. Extension IDs are length-unbounded, so a valid long ID could install at dest_dir yet fail every reinstall-after-keep-config with ENAMETOOLONG. Derive the staging component from a fixed-length hash via a new _rescue_staging_dir() helper. - The stranded-config restore used the full config filename as a NamedTemporaryFile prefix; a name already near the component limit plus the random suffix raised ENAMETOOLONG. Use a short fixed prefix. Updates the retry regression test to the new divergence semantics and adds conflict-abort, long-ID, and fixed-prefix coverage. Assisted-by: GitHub Copilot (model: Claude Opus 4.8, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/extensions/__init__.py | 59 +++++++++++++- tests/test_extensions.py | 106 +++++++++++++++++++++++-- 2 files changed, 155 insertions(+), 10 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 875230d568..9cb82297ca 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -746,6 +746,20 @@ def __init__(self, project_root: Path): self.extensions_dir = project_root / ".specify" / "extensions" self.registry = ExtensionRegistry(self.extensions_dir) + def _rescue_staging_dir(self, extension_id: str) -> Path: + """Fixed-length staging directory path for a preserved-config rescue. + + The extension ID can be arbitrarily long (manifest validation caps only + the character set, not the length), so embedding it verbatim in a single + path component could push the ``.rescue-staging-`` directory past a + filesystem's per-component byte limit and make every reinstall after + ``--keep-config`` fail with ``ENAMETOOLONG`` even though the extension + installs fine at ``dest_dir``. Hash the ID to a fixed-length suffix so + the component length is bounded regardless of ID length. + """ + digest = hashlib.sha256(extension_id.encode("utf-8")).hexdigest()[:16] + return self.extensions_dir / f".rescue-staging-{digest}" + @staticmethod def _collect_manifest_command_names(manifest: ExtensionManifest) -> Dict[str, str]: """Collect command and alias names declared by a manifest. @@ -1470,7 +1484,7 @@ def install_from_directory( # staging dir is removed only after every config has been successfully # restored. stranded_configs: dict[str, tuple[bytes, int]] = {} - rescue_staging_dir = self.extensions_dir / f".rescue-staging-{manifest.id}" + rescue_staging_dir = self._rescue_staging_dir(manifest.id) # A staging directory is trusted only when this completion marker is # present. The marker is written after every staged file is complete # and removed before the non-atomic cleanup, so a crash mid-staging or @@ -1492,16 +1506,49 @@ def install_from_directory( # on disk. Only load non-symlinked files whose names match # the two recognised config suffixes so a tampered staging # directory cannot inject arbitrary files. + # + # A complete staging directory proves only that staging finished, + # not that dest_dir was ever modified: a crash after staging was + # synced but before the rmtree below leaves the live kept config + # intact. If the user then edits that live config before retrying, + # blindly preferring the staged bytes would silently overwrite the + # newer config. The staged and live copies are indistinguishable + # in provenance from disk alone (a genuine post-crash edit vs. a + # packaged default written by a partially-completed copytree), so + # when a live config disagrees with its staged copy we must not + # silently pick either — preserve both and abort, letting the user + # resolve it. dest_dir is still untouched here, so raising is safe. + conflicting: list[str] = [] for staged_file in sorted(rescue_staging_dir.iterdir()): if ( staged_file.is_file() and not staged_file.is_symlink() and staged_file.name.endswith(("-config.yml", "-config.local.yml")) ): + staged_bytes = staged_file.read_bytes() + live_file = dest_dir / staged_file.name + if live_file.is_file() and not live_file.is_symlink(): + try: + live_bytes = live_file.read_bytes() + except OSError: + live_bytes = None + if live_bytes is not None and live_bytes != staged_bytes: + conflicting.append(staged_file.name) stranded_configs[staged_file.name] = ( - staged_file.read_bytes(), + staged_bytes, staged_file.stat().st_mode, ) + if conflicting: + names = ", ".join(sorted(conflicting)) + raise ValidationError( + "Preserved extension config conflict for " + f"'{manifest.id}': the current config file(s) ({names}) in " + f"{dest_dir} differ from the rescued backup left by an " + f"interrupted install in {rescue_staging_dir}. Both copies " + "have been preserved. Resolve manually — keep the current " + f"file and delete {rescue_staging_dir}, or restore the " + "backup over the current file — then reinstall." + ) elif dest_dir.exists() and not self.registry.is_installed(manifest.id): for cfg_file in ( list(dest_dir.glob("*-config.yml")) @@ -1599,10 +1646,16 @@ def _restore_stranded_config_file( ) -> None: tmp_path: Path | None = None try: + # A short fixed prefix, not f".{target.name}.": the preserved + # config filename may itself already be near the filesystem's + # per-component byte limit, and NamedTemporaryFile appends a + # random suffix to the prefix — reusing the full name would push + # the temp file past the limit and raise ENAMETOOLONG on every + # retry. tempfile already guarantees collision avoidance. with tempfile.NamedTemporaryFile( mode="wb", dir=target.parent, - prefix=f".{target.name}.", + prefix=".cfg-restore.", delete=False, ) as tmp: tmp_path = Path(tmp.name) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 87c8a63aba..86f5a26ceb 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1421,7 +1421,12 @@ def test_extensionignore_load_failure_preserves_kept_config( def test_retry_after_staging_backup_restores_stranded_config( self, extension_dir, project_dir, monkeypatch ): - """A retry after an interrupted install should restore the rescued config from staging.""" + """A retry after an interrupted install restores the rescued config. + + When the live config is unchanged since the interrupted attempt (it + still matches the staged backup), the retry proceeds and yields the + preserved bytes, and the staging directory is cleaned up on success. + """ import stat manager = ExtensionManager(project_dir) @@ -1446,7 +1451,7 @@ def test_retry_after_staging_backup_restores_stranded_config( assert not manager.registry.is_installed("test-ext") assert config_file.exists() - staging_dir = manager.extensions_dir / ".rescue-staging-test-ext" + staging_dir = manager._rescue_staging_dir("test-ext") assert not staging_dir.exists() original_copytree = shutil.copytree @@ -1473,11 +1478,9 @@ def flaky_copytree(*args, **kwargs): assert (staging_dir / ".rescue-complete").exists() assert (staging_dir / "test-ext-config.yml").exists() - # Corrupt the rollback-restored copy to prove the next retry must rely - # on the durable staging backup, not whatever was written to dest_dir - # by the earlier failed install attempt. - config_file.write_text("model: wrong-model\n", encoding="utf-8") - assert config_file.read_bytes() != original_bytes + # The rollback restored the preserved bytes to the live config, so it + # still agrees with the staged backup — the retry proceeds normally. + assert config_file.read_bytes() == original_bytes manifest = manager.install_from_directory( extension_dir, "0.1.0", register_commands=False @@ -1493,6 +1496,95 @@ def flaky_copytree(*args, **kwargs): restored_mode = config_file.stat().st_mode assert stat.S_IMODE(restored_mode) == stat.S_IMODE(original_mode) + def test_retry_with_edited_live_config_aborts_and_preserves_both( + self, extension_dir, project_dir, monkeypatch + ): + """A retry must not silently overwrite a config edited after a crash. + + A complete staging directory proves only that staging finished, not + that dest_dir was modified. If the user edits the live kept config + before retrying, the retry must detect the divergence, preserve both + copies, and abort rather than blindly restoring the older staged bytes. + """ + manager = ExtensionManager(project_dir) + + packaged_config = extension_dir / "test-ext-config.yml" + packaged_config.write_text("model: default-model\n") + + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + ext_dir = project_dir / ".specify" / "extensions" / "test-ext" + config_file = ext_dir / "test-ext-config.yml" + config_file.write_text("model: custom-model\nmax_iterations: 99\n") + staged_bytes = config_file.read_bytes() + + manager.remove("test-ext", keep_config=True) + assert not manager.registry.is_installed("test-ext") + + staging_dir = manager._rescue_staging_dir("test-ext") + + original_copytree = shutil.copytree + copytree_calls = 0 + + def flaky_copytree(*args, **kwargs): + nonlocal copytree_calls + copytree_calls += 1 + if copytree_calls == 1: + dst = args[1] + Path(dst).mkdir(parents=True, exist_ok=True) + (Path(dst) / "_partial.txt").write_text("partial") + raise OSError("simulated disk full") + return original_copytree(*args, **kwargs) + + monkeypatch.setattr(_ext_module.shutil, "copytree", flaky_copytree) + + with pytest.raises(OSError, match="simulated disk full"): + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + assert staging_dir.exists() + assert (staging_dir / ".rescue-complete").exists() + + # Simulate the user editing the live config before retrying so it now + # diverges from the staged backup. + config_file.write_text("model: newer-edited-model\n") + edited_bytes = config_file.read_bytes() + assert edited_bytes != staged_bytes + + with pytest.raises(ValidationError, match="Preserved extension config conflict"): + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + # Both copies must survive: the edited live config and the staged backup. + assert config_file.read_bytes() == edited_bytes + assert staging_dir.exists() + assert (staging_dir / "test-ext-config.yml").read_bytes() == staged_bytes + assert not manager.registry.is_installed("test-ext") + + def test_rescue_staging_dir_is_fixed_length_for_long_ids(self, project_dir): + """The rescue staging component length must not grow with the ID length. + + Manifest validation caps the ID character set but not its length, so a + very long (but valid) ID must not lengthen the single staging path + component past a filesystem's per-component byte limit. + """ + manager = ExtensionManager(project_dir) + + short_dir = manager._rescue_staging_dir("a") + long_id = "a" * 250 + long_dir = manager._rescue_staging_dir(long_id) + + # Same fixed component length regardless of ID length. + assert len(short_dir.name) == len(long_dir.name) + # Comfortably within the common 255-byte component limit. + assert len(long_dir.name.encode("utf-8")) <= 255 + # Distinct IDs still map to distinct staging directories. + assert manager._rescue_staging_dir("b") != short_dir + def test_install_force_without_existing(self, extension_dir, project_dir): """Test force-install when extension is NOT already installed (works normally).""" manager = ExtensionManager(project_dir) From 899e7a6fe14221935353a988747d359b92812c30 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:45:01 +0000 Subject: [PATCH 25/33] Harden preserved-config rescue divergence check and fix test path Assisted-by: GitHub Copilot (model: GPT-5.6, autonomous) --- src/specify_cli/extensions/__init__.py | 70 ++++++++++++++++++-------- tests/test_extensions.py | 2 +- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 9cb82297ca..e9817396df 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1518,26 +1518,56 @@ def install_from_directory( # when a live config disagrees with its staged copy we must not # silently pick either — preserve both and abort, letting the user # resolve it. dest_dir is still untouched here, so raising is safe. - conflicting: list[str] = [] - for staged_file in sorted(rescue_staging_dir.iterdir()): - if ( - staged_file.is_file() - and not staged_file.is_symlink() - and staged_file.name.endswith(("-config.yml", "-config.local.yml")) - ): - staged_bytes = staged_file.read_bytes() - live_file = dest_dir / staged_file.name - if live_file.is_file() and not live_file.is_symlink(): - try: - live_bytes = live_file.read_bytes() - except OSError: - live_bytes = None - if live_bytes is not None and live_bytes != staged_bytes: - conflicting.append(staged_file.name) - stranded_configs[staged_file.name] = ( - staged_bytes, - staged_file.stat().st_mode, - ) + def _recognized_config_names(directory: Path) -> set[str]: + names: set[str] = set() + if not directory.is_dir(): + return names + for entry in directory.iterdir(): + if ( + entry.is_file() + and not entry.is_symlink() + and entry.name.endswith( + ("-config.yml", "-config.local.yml") + ) + ): + names.add(entry.name) + return names + + conflicting: set[str] = set() + staged_names = _recognized_config_names(rescue_staging_dir) + live_names = _recognized_config_names(dest_dir) + # A live-only config created after the interrupted attempt is not + # enumerated by staging, so without this it would be silently + # deleted by the rmtree below and its bytes lost. Treat it as a + # conflict so both locations are preserved and the user resolves it. + conflicting.update(live_names - staged_names) + for staged_name in sorted(staged_names): + staged_file = rescue_staging_dir / staged_name + staged_stat = staged_file.stat() + staged_bytes = staged_file.read_bytes() + live_file = dest_dir / staged_name + if live_file.is_file() and not live_file.is_symlink(): + # A live config that cannot be read or stat'ed must not be + # treated as non-conflicting: the rmtree below would delete + # it and restore the stale staged copy. Abort while dest_dir + # is untouched so no newer or permission-restricted config is + # lost. Divergence also includes permission-only edits (for + # example tightening a secret-bearing config from 0644 to + # 0600), which byte equality alone would miss and then revert. + try: + live_stat = live_file.stat() + live_bytes = live_file.read_bytes() + except OSError: + conflicting.add(staged_name) + else: + if live_bytes != staged_bytes or stat.S_IMODE( + live_stat.st_mode + ) != stat.S_IMODE(staged_stat.st_mode): + conflicting.add(staged_name) + stranded_configs[staged_name] = ( + staged_bytes, + staged_stat.st_mode, + ) if conflicting: names = ", ".join(sorted(conflicting)) raise ValidationError( diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 86f5a26ceb..50b2599123 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1401,7 +1401,7 @@ def test_extensionignore_load_failure_preserves_kept_config( # No rescue staging may have been published, so a later retry reads the # live (possibly newly edited) config rather than stale staged bytes. - staging_dir = manager.extensions_dir / ".rescue-staging-test-ext" + staging_dir = manager._rescue_staging_dir("test-ext") assert not staging_dir.exists() # Simulate the user fixing the ignore file and editing the kept config, From 34813e99268fd80cb9f6a459ce0605747df92452 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:01:43 +0000 Subject: [PATCH 26/33] fix: reject/flag symlinked preserved configs on reinstall Assisted-by: GitHub Copilot (model: GPT-5.6-Sol, autonomous) --- src/specify_cli/extensions/__init__.py | 27 ++++++- tests/test_extensions.py | 107 +++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index e9817396df..78dbe74664 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1546,7 +1546,15 @@ def _recognized_config_names(directory: Path) -> set[str]: staged_stat = staged_file.stat() staged_bytes = staged_file.read_bytes() live_file = dest_dir / staged_name - if live_file.is_file() and not live_file.is_symlink(): + if live_file.is_symlink(): + # A user may have replaced the live config with a symlink + # after the interrupted attempt. It cannot be compared by + # bytes/mode against the staged copy, and the rmtree below + # would silently delete this newer choice and restore the + # older staged file. Treat any live symlink as a conflict so + # both are preserved and the user resolves it. + conflicting.add(staged_name) + elif live_file.is_file(): # A live config that cannot be read or stat'ed must not be # treated as non-conflicting: the rmtree below would delete # it and restore the stale staged copy. Abort while dest_dir @@ -1584,7 +1592,22 @@ def _recognized_config_names(directory: Path) -> set[str]: list(dest_dir.glob("*-config.yml")) + list(dest_dir.glob("*-config.local.yml")) ): - if cfg_file.is_file() and not cfg_file.is_symlink(): + if cfg_file.is_symlink(): + # `remove --keep-config` preserves a symlinked config + # because Path.is_file() follows symlinks. Its bytes cannot + # be safely rescued (the target may live outside dest_dir), + # and the rmtree below would delete the link and silently + # discard the kept configuration. Reject the reinstall while + # dest_dir is untouched so the user resolves it rather than + # losing the linked config. + raise ValidationError( + "Preserved extension config for " + f"'{manifest.id}' is a symlink ({cfg_file.name}) in " + f"{dest_dir}, which cannot be safely rescued during " + "reinstall. Resolve manually — replace the symlink with " + "a regular file or remove it — then reinstall." + ) + if cfg_file.is_file(): stranded_configs[cfg_file.name] = ( cfg_file.read_bytes(), cfg_file.stat().st_mode, diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 50b2599123..f92a9a0f1b 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1290,6 +1290,113 @@ def test_reinstall_after_keep_config_preserves_local_config( assert local_cfg.exists() assert "local_override: true" in local_cfg.read_text() + def test_reinstall_with_symlinked_config_rejects_install( + self, extension_dir, project_dir + ): + """A preserved symlinked config must abort reinstall, not be deleted.""" + ext_dir = project_dir / ".specify" / "extensions" / "test-ext" + if not can_create_symlink(ext_dir.parent if ext_dir.parent.exists() else project_dir): + pytest.skip("Current platform/user cannot create symlinks") + + manager = ExtensionManager(project_dir) + packaged_config = extension_dir / "test-ext-config.yml" + packaged_config.write_text("model: default-model\n") + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + # Replace the installed config with a symlink to a file outside dest_dir. + config_file = ext_dir / "test-ext-config.yml" + external_target = project_dir / "external-config.yml" + external_target.write_text("model: linked-model\n") + config_file.unlink() + os.symlink(external_target, config_file) + assert config_file.is_symlink() + + # `remove --keep-config` follows the symlink via is_file() and keeps it. + manager.remove("test-ext", keep_config=True) + assert not manager.registry.is_installed("test-ext") + assert config_file.is_symlink() + + # Plain reinstall must reject rather than silently delete the link. + with pytest.raises(ValidationError, match="is a symlink"): + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + # The symlink and its target survive; nothing was silently discarded. + assert config_file.is_symlink() + assert external_target.read_text() == "model: linked-model\n" + assert not manager.registry.is_installed("test-ext") + + def test_retry_with_symlinked_live_config_aborts_and_preserves_both( + self, extension_dir, project_dir, monkeypatch + ): + """A live config replaced by a symlink on retry is a conflict, not overwritten.""" + ext_dir = project_dir / ".specify" / "extensions" / "test-ext" + if not can_create_symlink(project_dir): + pytest.skip("Current platform/user cannot create symlinks") + + manager = ExtensionManager(project_dir) + + packaged_config = extension_dir / "test-ext-config.yml" + packaged_config.write_text("model: default-model\n") + + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + config_file = ext_dir / "test-ext-config.yml" + config_file.write_text("model: custom-model\nmax_iterations: 99\n") + staged_bytes = config_file.read_bytes() + + manager.remove("test-ext", keep_config=True) + assert not manager.registry.is_installed("test-ext") + + staging_dir = manager._rescue_staging_dir("test-ext") + + original_copytree = shutil.copytree + copytree_calls = 0 + + def flaky_copytree(*args, **kwargs): + nonlocal copytree_calls + copytree_calls += 1 + if copytree_calls == 1: + dst = args[1] + Path(dst).mkdir(parents=True, exist_ok=True) + (Path(dst) / "_partial.txt").write_text("partial") + raise OSError("simulated disk full") + return original_copytree(*args, **kwargs) + + monkeypatch.setattr(_ext_module.shutil, "copytree", flaky_copytree) + + with pytest.raises(OSError, match="simulated disk full"): + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + assert staging_dir.exists() + assert (staging_dir / ".rescue-complete").exists() + + # Simulate the user replacing the live config with a symlink before retry. + external_target = project_dir / "external-config.yml" + external_target.write_text("model: newer-linked-model\n") + config_file.unlink() + os.symlink(external_target, config_file) + assert config_file.is_symlink() + + with pytest.raises(ValidationError, match="Preserved extension config conflict"): + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + # Both copies survive: the live symlink choice and the staged backup. + assert config_file.is_symlink() + assert external_target.read_text() == "model: newer-linked-model\n" + assert staging_dir.exists() + assert (staging_dir / "test-ext-config.yml").read_bytes() == staged_bytes + assert not manager.registry.is_installed("test-ext") + def test_copytree_failure_restores_stranded_config( self, extension_dir, project_dir, monkeypatch ): From 61edcc1259561b73ab7c4a584cba22a35fbbdcd9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:20:13 +0000 Subject: [PATCH 27/33] fix: include symlinks in live-dir config enumeration and address review feedback - _recognized_config_names() now accepts follow_symlinks=False for live dir so symlinked *-config.yml entries are detected and treated as conflicts rather than being silently deleted by rmtree. - Add explanatory comment to bare 'except OSError: pass' in _restore_stranded_config_file's finally block. - Resolve CodeQL dual-import style: use 'from specify_cli import extensions as _ext_module' instead of 'import specify_cli.extensions as _ext_module'. Assisted-by: GitHub Copilot (model: claude-sonnet-4, autonomous) --- src/specify_cli/extensions/__init__.py | 29 +++++++++++++++++--------- tests/test_extensions.py | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 78dbe74664..377a8855e7 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1518,24 +1518,33 @@ def install_from_directory( # when a live config disagrees with its staged copy we must not # silently pick either — preserve both and abort, letting the user # resolve it. dest_dir is still untouched here, so raising is safe. - def _recognized_config_names(directory: Path) -> set[str]: + def _recognized_config_names( + directory: Path, *, follow_symlinks: bool = True + ) -> set[str]: names: set[str] = set() if not directory.is_dir(): return names for entry in directory.iterdir(): - if ( - entry.is_file() - and not entry.is_symlink() - and entry.name.endswith( - ("-config.yml", "-config.local.yml") - ) + if not entry.name.endswith( + ("-config.yml", "-config.local.yml") ): - names.add(entry.name) + continue + if follow_symlinks: + if entry.is_file() and not entry.is_symlink(): + names.add(entry.name) + else: + # Include symlinks without following them so that + # live-only symlinked configs are detected and + # preserved rather than silently deleted. + if entry.is_file() or entry.is_symlink(): + names.add(entry.name) return names conflicting: set[str] = set() staged_names = _recognized_config_names(rescue_staging_dir) - live_names = _recognized_config_names(dest_dir) + live_names = _recognized_config_names( + dest_dir, follow_symlinks=False + ) # A live-only config created after the interrupted attempt is not # enumerated by staging, so without this it would be silently # deleted by the rmtree below and its bytes lost. Treat it as a @@ -1732,7 +1741,7 @@ def _restore_stranded_config_file( try: os.close(target_fd) except OSError: - pass + pass # best-effort close during cleanup; ignore errors _fsync_directory(target.parent) except BaseException: if tmp_path is not None and tmp_path.exists(): diff --git a/tests/test_extensions.py b/tests/test_extensions.py index f92a9a0f1b..81ff951dea 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -22,7 +22,7 @@ from unittest.mock import MagicMock from tests.conftest import strip_ansi -import specify_cli.extensions as _ext_module +from specify_cli import extensions as _ext_module from specify_cli.extensions import ( CatalogEntry, CORE_COMMAND_NAMES, From b75a748491361320939debf457e0dbc54b5b761c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:57:13 +0000 Subject: [PATCH 28/33] test: add staging-failure fault-injection test for rescue staging block Add test_staging_failure_aborts_before_dest_dir_removal covering three failure modes (mkdir, os.open/O_CREAT, fsync with EIO) in the rescue staging block. Each parametrized case verifies: - the install aborts before dest_dir is removed - the preserved config bytes remain authoritative - any partial staging is cleaned up and not left as complete - the extension stays unregistered Addresses review feedback on PRRT_kwDOPiFCnc6R351t. Assisted-by: GitHub Copilot (model: claude-sonnet-4.5, autonomous) --- tests/test_extensions.py | 87 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 81ff951dea..ae0073fc37 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1672,6 +1672,93 @@ def flaky_copytree(*args, **kwargs): assert (staging_dir / "test-ext-config.yml").read_bytes() == staged_bytes assert not manager.registry.is_installed("test-ext") + @pytest.mark.parametrize( + "failure_mode", + [ + pytest.param("mkdir", id="mkdir"), + pytest.param("os_open", id="os_open"), + pytest.param("fsync", id="fsync"), + ], + ) + def test_staging_failure_aborts_before_dest_dir_removal( + self, extension_dir, project_dir, monkeypatch, failure_mode + ): + """Staging failures abort the install before dest_dir is removed. + + When mkdir, os.open, or fsync fails while publishing rescue staging, + the install must abort before removing dest_dir so the preserved + config bytes remain authoritative, any partial staging is cleaned up + rather than trusted on retry, and the extension stays unregistered. + """ + import errno as _errno + + manager = ExtensionManager(project_dir) + + packaged_config = extension_dir / "test-ext-config.yml" + packaged_config.write_text("model: default-model\n") + + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + ext_dir = project_dir / ".specify" / "extensions" / "test-ext" + config_file = ext_dir / "test-ext-config.yml" + config_file.write_text("model: custom-model\nmax_iterations: 99\n") + original_bytes = config_file.read_bytes() + + manager.remove("test-ext", keep_config=True) + assert not manager.registry.is_installed("test-ext") + assert config_file.exists() + + staging_dir = manager._rescue_staging_dir("test-ext") + assert not staging_dir.exists() + + if failure_mode == "mkdir": + original_mkdir = Path.mkdir + + def failing_mkdir(self_path, *args, **kwargs): + if self_path == staging_dir: + raise OSError("staging mkdir failed") + return original_mkdir(self_path, *args, **kwargs) + + monkeypatch.setattr(Path, "mkdir", failing_mkdir) + + elif failure_mode == "os_open": + original_os_open = _ext_module.os.open + + def failing_os_open(path, flags, mode=0o777, *args, **kwargs): + # Fail only for file creation (O_CREAT) inside the staging + # directory so other os.open calls (e.g. directory fsync) are + # unaffected. + if str(staging_dir) in str(path) and (flags & os.O_CREAT): + raise OSError(_errno.ENOSPC, "No space left on device") + return original_os_open(path, flags, mode, *args, **kwargs) + + monkeypatch.setattr(_ext_module.os, "open", failing_os_open) + + else: # "fsync" + def failing_fsync_fd(fd: int) -> None: + # Simulate a real storage error (EIO) that the helper propagates. + raise OSError(_errno.EIO, "Input/output error") + + monkeypatch.setattr(_ext_module, "_fsync_fd", failing_fsync_fd) + + with pytest.raises(OSError): + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + # dest_dir must still exist — the install aborted before rmtree. + assert ext_dir.exists(), "dest_dir must survive a staging failure" + assert config_file.read_bytes() == original_bytes, ( + "preserved config must remain authoritative" + ) + # Partial staging must have been cleaned up and not left as complete. + assert not staging_dir.exists() or not ( + staging_dir / ".rescue-complete" + ).exists(), "incomplete staging must not be trusted" + assert not manager.registry.is_installed("test-ext") + def test_rescue_staging_dir_is_fixed_length_for_long_ids(self, project_dir): """The rescue staging component length must not grow with the ID length. From 09e35ea0b00c377c52e0e9500ce6c9bc9ddb62eb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:16:22 +0000 Subject: [PATCH 29/33] test: add test_retry_restores_config_from_staging_when_live_absent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exercises the retry-from-staging branch (if staging_is_complete at line 1505 of extensions/__init__.py) in a scenario where the live config is absent — simulating a power loss that interrupted the rollback before it could write the config back. When the live copy is gone, the live-dir fallback (elif dest_dir.exists()) finds no stranded configs and the packaged default would be kept. Only the staging-complete branch can restore the original bytes and mode. This proves staging (not the fallback) is used on retry. Addresses review feedback on PRRT_kwDOPiFCnc6SAL3L. Assisted-by: GitHub Copilot (model: claude-sonnet-4.5, autonomous) --- tests/test_extensions.py | 86 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tests/test_extensions.py b/tests/test_extensions.py index ae0073fc37..0f9eb3f346 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1603,6 +1603,92 @@ def flaky_copytree(*args, **kwargs): restored_mode = config_file.stat().st_mode assert stat.S_IMODE(restored_mode) == stat.S_IMODE(original_mode) + def test_retry_restores_config_from_staging_when_live_absent( + self, extension_dir, project_dir, monkeypatch + ): + """Retry succeeds using only staging when the live config is absent. + + After a copytree failure, staging is complete and the rollback writes + the config back to dest_dir. If a power loss interrupts that rollback + the config may be absent on the next attempt. The retry-from-staging + branch (``if staging_is_complete``) must restore the config from staging + alone so the original bytes and mode are recovered even when no live copy + is present. This is the critical path that distinguishes the staging + branch from the live-dir fallback. + """ + import stat + + manager = ExtensionManager(project_dir) + + packaged_config = extension_dir / "test-ext-config.yml" + packaged_config.write_text("model: default-model\n") + + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + ext_dir = project_dir / ".specify" / "extensions" / "test-ext" + config_file = ext_dir / "test-ext-config.yml" + config_file.write_text("model: custom-model\nmax_iterations: 99\n") + + if platform.system() != "Windows": + config_file.chmod(0o640) + original_bytes = config_file.read_bytes() + original_mode = config_file.stat().st_mode + + manager.remove("test-ext", keep_config=True) + assert not manager.registry.is_installed("test-ext") + assert config_file.exists() + + staging_dir = manager._rescue_staging_dir("test-ext") + assert not staging_dir.exists() + + original_copytree = shutil.copytree + copytree_calls = 0 + + def flaky_copytree(*args, **kwargs): + nonlocal copytree_calls + copytree_calls += 1 + if copytree_calls == 1: + dst = args[1] + Path(dst).mkdir(parents=True, exist_ok=True) + (Path(dst) / "_partial.txt").write_text("partial") + raise OSError("simulated disk full") + return original_copytree(*args, **kwargs) + + monkeypatch.setattr(_ext_module.shutil, "copytree", flaky_copytree) + + with pytest.raises(OSError, match="simulated disk full"): + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + # Staging is complete after the first failure. + assert staging_dir.exists() + assert (staging_dir / ".rescue-complete").exists() + assert (staging_dir / "test-ext-config.yml").exists() + + # Simulate a power loss that prevented the rollback from writing the + # config back: delete the live copy so the retry must rely on staging. + config_file.unlink() + assert not config_file.exists() + + manifest = manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + # The staging branch must restore the original bytes even though no live + # copy was present — proving staging (not the live-dir fallback) was used. + assert manifest.id == "test-ext" + assert manager.registry.is_installed("test-ext") + assert config_file.read_bytes() == original_bytes + assert not (ext_dir / "_partial.txt").exists() + assert not staging_dir.exists() + + if platform.system() != "Windows": + restored_mode = config_file.stat().st_mode + assert stat.S_IMODE(restored_mode) == stat.S_IMODE(original_mode) + def test_retry_with_edited_live_config_aborts_and_preserves_both( self, extension_dir, project_dir, monkeypatch ): From 752cd96dbab6296a33574d4d953c7f7f1b6e278e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 21:21:22 +0000 Subject: [PATCH 30/33] fix: keep staging files writable; record modes in .rescue-modes.json; fix live-only conflict message Thread 64: Remove os.fchmod/chmod from staged files to avoid Windows read-only attribute that prevents shutil.rmtree from cleaning up. Original permission bits are now written to a .rescue-modes.json sidecar in the staging dir and reloaded during retry, with a fall-back to the staged file's own mode for backwards-compat with pre-sidecar staging dirs. Thread 65: Split the ValidationError message for staging-vs-live conflicts into two accurate cases: files that diverged between both locations ("Both copies have been preserved") and live-only files that have no backup counterpart, which previously incorrectly claimed "Both copies have been preserved" and offered a restore instruction that was impossible. Assisted-by: GitHub Copilot (model: claude-sonnet-4.5, autonomous) --- src/specify_cli/extensions/__init__.py | 101 +++++++++++++++++++------ 1 file changed, 79 insertions(+), 22 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 377a8855e7..66ef219627 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1549,11 +1549,30 @@ def _recognized_config_names( # enumerated by staging, so without this it would be silently # deleted by the rmtree below and its bytes lost. Treat it as a # conflict so both locations are preserved and the user resolves it. - conflicting.update(live_names - staged_names) + live_only = live_names - staged_names + conflicting.update(live_only) + # Load original permission bits from the sidecar JSON written by + # the staging step. Staged files are kept at mode 0o600 so that + # rmtree always succeeds on Windows, so staged_stat.st_mode would + # always be 0o600 and must not be used for mode comparisons or + # restoration; the sidecar records the true original mode. + rescue_modes_file = rescue_staging_dir / ".rescue-modes.json" + _staged_modes: dict[str, int] = {} + if rescue_modes_file.is_file() and not rescue_modes_file.is_symlink(): + try: + _staged_modes = json.loads(rescue_modes_file.read_bytes()) + except (OSError, ValueError): + pass for staged_name in sorted(staged_names): staged_file = rescue_staging_dir / staged_name staged_stat = staged_file.stat() staged_bytes = staged_file.read_bytes() + # Prefer the sidecar-recorded mode; fall back to the staged + # file's own mode for backwards-compat with staging dirs + # written before the sidecar was introduced. + staged_mode = _staged_modes.get( + staged_name, stat.S_IMODE(staged_stat.st_mode) + ) live_file = dest_dir / staged_name if live_file.is_symlink(): # A user may have replaced the live config with a symlink @@ -1579,23 +1598,38 @@ def _recognized_config_names( else: if live_bytes != staged_bytes or stat.S_IMODE( live_stat.st_mode - ) != stat.S_IMODE(staged_stat.st_mode): + ) != staged_mode: conflicting.add(staged_name) - stranded_configs[staged_name] = ( - staged_bytes, - staged_stat.st_mode, - ) + stranded_configs[staged_name] = (staged_bytes, staged_mode) if conflicting: - names = ", ".join(sorted(conflicting)) - raise ValidationError( - "Preserved extension config conflict for " - f"'{manifest.id}': the current config file(s) ({names}) in " - f"{dest_dir} differ from the rescued backup left by an " - f"interrupted install in {rescue_staging_dir}. Both copies " - "have been preserved. Resolve manually — keep the current " - f"file and delete {rescue_staging_dir}, or restore the " - "backup over the current file — then reinstall." + # Split into two cases for accurate user guidance: files that + # exist in both locations but have diverged, and files that + # exist only in the live directory with no rescue-backup copy. + both_diverged = conflicting - live_only + live_only_conflict = conflicting & live_only + msg_parts: list[str] = [ + f"Preserved extension config conflict for '{manifest.id}':" + ] + if both_diverged: + names = ", ".join(sorted(both_diverged)) + msg_parts.append( + f"The current config(s) ({names}) in {dest_dir} differ" + f" from their rescued backup in {rescue_staging_dir}." + " Both copies have been preserved." + ) + if live_only_conflict: + names = ", ".join(sorted(live_only_conflict)) + msg_parts.append( + f"The config(s) ({names}) exist only in {dest_dir}" + f" with no counterpart in the rescued backup at" + f" {rescue_staging_dir}." + ) + msg_parts.append( + f"Reconcile {dest_dir} and {rescue_staging_dir} to the" + f" desired final state, delete {rescue_staging_dir}," + " then reinstall." ) + raise ValidationError(" ".join(msg_parts)) elif dest_dir.exists() and not self.registry.is_installed(manifest.id): for cfg_file in ( list(dest_dir.glob("*-config.yml")) @@ -1661,16 +1695,39 @@ def _recognized_config_names( written = 0 while written < len(view): written += os.write(fd, view[written:]) - try: - os.fchmod(fd, stat.S_IMODE(mode)) - except (AttributeError, NotImplementedError, OSError): - try: - staged.chmod(stat.S_IMODE(mode)) - except (NotImplementedError, OSError): - pass # Best-effort; chmod may not be supported on all platforms. + # Do NOT chmod the staged file: setting a read-only + # mode (e.g. 0o444) makes the file undeletable on + # Windows and causes shutil.rmtree to fail during + # cleanup. Original modes are recorded separately in + # .rescue-modes.json so they can be reapplied when the + # config is actually restored. _fsync_fd(fd) finally: os.close(fd) + # Persist the original permission bits in a sidecar JSON file + # so a retry can correctly reapply them even though the staged + # files themselves are kept at their creation mode (0o600). + rescue_modes_file = rescue_staging_dir / ".rescue-modes.json" + modes_payload = json.dumps( + { + filename: stat.S_IMODE(mode) + for filename, (_, mode) in stranded_configs.items() + }, + sort_keys=True, + ).encode() + modes_fd = os.open( + str(rescue_modes_file), + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0), + 0o600, + ) + try: + view = memoryview(modes_payload) + written = 0 + while written < len(view): + written += os.write(modes_fd, view[written:]) + _fsync_fd(modes_fd) + finally: + os.close(modes_fd) # Flush the staging directory metadata before publishing the # completion marker so a crash cannot leave a visible marker with # only a subset of staged files. From 88bc55b075bbaef5cbbe2590d61aabbd1a8c6ce7 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:26:24 -0500 Subject: [PATCH 31/33] Potential fix for pull request finding 'Empty except' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- src/specify_cli/extensions/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 66ef219627..22fe3e2fb2 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1562,6 +1562,8 @@ def _recognized_config_names( try: _staged_modes = json.loads(rescue_modes_file.read_bytes()) except (OSError, ValueError): + # Ignore unreadable/invalid sidecar metadata and fall back + # to each staged file's mode for compatibility. pass for staged_name in sorted(staged_names): staged_file = rescue_staging_dir / staged_name From dece98dd4b6c4b800325c9b6c59ca9a913b32a00 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 21:58:07 +0000 Subject: [PATCH 32/33] fix: add .keep-config provenance marker to guard rescue path against partially-failed installs When `remove --keep-config` strands config files, write a `.keep-config` marker into the extension directory. `install_from_directory` now only enters the rescue path when that marker is present, preventing a partially- failed install (which also leaves dest_dir with no registry entry but no marker) from having its packaged default configs treated as user-preserved data on a retry from an updated package. Refs: https://github.com/github/spec-kit/pull/3449#discussion_r3606283457 Assisted-by: GitHub Copilot (model: claude-sonnet-4.5, autonomous) --- src/specify_cli/extensions/__init__.py | 12 ++++++- tests/test_extensions.py | 46 ++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index 22fe3e2fb2..ef480dbe94 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -1632,7 +1632,12 @@ def _recognized_config_names( " then reinstall." ) raise ValidationError(" ".join(msg_parts)) - elif dest_dir.exists() and not self.registry.is_installed(manifest.id): + elif ( + dest_dir.exists() + and not self.registry.is_installed(manifest.id) + and (dest_dir / ".keep-config").is_file() + and not (dest_dir / ".keep-config").is_symlink() + ): for cfg_file in ( list(dest_dir.glob("*-config.yml")) + list(dest_dir.glob("*-config.local.yml")) @@ -2015,6 +2020,11 @@ def remove(self, extension_id: str, keep_config: bool = False) -> bool: shutil.rmtree(child) else: child.unlink() + # Write a provenance marker so install_from_directory can + # distinguish this --keep-config leftover from a directory left + # by a partially-failed install (which must not have its + # packaged default configs treated as user-preserved data). + (extension_dir / ".keep-config").write_text("") else: # Backup config files before deleting if extension_dir.exists(): diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 0f9eb3f346..e2baee6414 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -1865,6 +1865,52 @@ def test_rescue_staging_dir_is_fixed_length_for_long_ids(self, project_dir): # Distinct IDs still map to distinct staging directories. assert manager._rescue_staging_dir("b") != short_dir + def test_failed_install_without_keep_config_does_not_rescue_defaults( + self, extension_dir, project_dir, monkeypatch + ): + """A dir left by a partially-failed install must not trigger the rescue path. + + Any install that copies files but then fails during command, skill, or + hook registration also leaves a complete dest_dir with no registry entry. + On a later retry from an updated package this branch must not treat the + previous package's default config as user-preserved data and restore it + over the new defaults. Only directories explicitly left by + ``remove --keep-config`` (which writes a ``.keep-config`` marker) should + trigger the rescue path. + """ + manager = ExtensionManager(project_dir) + + packaged_config = extension_dir / "test-ext-config.yml" + packaged_config.write_text("model: default-model\n") + + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + ext_dir = project_dir / ".specify" / "extensions" / "test-ext" + config_file = ext_dir / "test-ext-config.yml" + + # Simulate a partially-failed install: the extension directory is present + # with the packaged default config but there is no .keep-config marker and + # the extension is not in the registry. This matches what happens when + # copytree succeeds but command/hook registration raises afterwards. + manager.registry.remove("test-ext") + assert not manager.registry.is_installed("test-ext") + assert ext_dir.exists() + assert not (ext_dir / ".keep-config").exists() + + # Update the packaged config so a retry with the new package would use + # different defaults — the old defaults must NOT be rescued. + packaged_config.write_text("model: updated-default-model\n") + + manager.install_from_directory( + extension_dir, "0.1.0", register_commands=False + ) + + assert manager.registry.is_installed("test-ext") + # The new packaged default must win; the old default was not user data. + assert config_file.read_text() == "model: updated-default-model\n" + def test_install_force_without_existing(self, extension_dir, project_dir): """Test force-install when extension is NOT already installed (works normally).""" manager = ExtensionManager(project_dir) From 8a80b4574cfae7373a552b75f0ddb75f8db9d26b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 22:00:25 +0000 Subject: [PATCH 33/33] refactor: extract _has_keep_config_marker helper and document empty-content choice Assisted-by: GitHub Copilot (model: claude-sonnet-4.5, autonomous) --- src/specify_cli/extensions/__init__.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/extensions/__init__.py b/src/specify_cli/extensions/__init__.py index ef480dbe94..f33c1578b1 100644 --- a/src/specify_cli/extensions/__init__.py +++ b/src/specify_cli/extensions/__init__.py @@ -760,6 +760,18 @@ def _rescue_staging_dir(self, extension_id: str) -> Path: digest = hashlib.sha256(extension_id.encode("utf-8")).hexdigest()[:16] return self.extensions_dir / f".rescue-staging-{digest}" + @staticmethod + def _has_keep_config_marker(directory: Path) -> bool: + """Return True when *directory* contains a valid ``.keep-config`` marker. + + The marker is a regular (non-symlink) file written by + ``remove(..., keep_config=True)`` to record explicit provenance. Its + content is intentionally empty — only presence matters, not content. + The symlink guard prevents a crafted symlink from fooling the check. + """ + marker = directory / ".keep-config" + return marker.is_file() and not marker.is_symlink() + @staticmethod def _collect_manifest_command_names(manifest: ExtensionManifest) -> Dict[str, str]: """Collect command and alias names declared by a manifest. @@ -1635,8 +1647,7 @@ def _recognized_config_names( elif ( dest_dir.exists() and not self.registry.is_installed(manifest.id) - and (dest_dir / ".keep-config").is_file() - and not (dest_dir / ".keep-config").is_symlink() + and self._has_keep_config_marker(dest_dir) ): for cfg_file in ( list(dest_dir.glob("*-config.yml")) @@ -2024,6 +2035,7 @@ def remove(self, extension_id: str, keep_config: bool = False) -> bool: # distinguish this --keep-config leftover from a directory left # by a partially-failed install (which must not have its # packaged default configs treated as user-preserved data). + # Content is intentionally empty — only presence matters. (extension_dir / ".keep-config").write_text("") else: # Backup config files before deleting