[Backport release-0.7] add tracking to created resources in OpenDAL

docs/source/reference/package-apis/drivers/opendal.yaml

@@ -5,3 +5,6 @@ config:
   scheme: "fs"
   kwargs:
     root: "/tmp/jumpstarter"
+  # Optional: automatically remove created files/directories when driver closes
+  # Note: all tracked paths are normalized by stripping leading/trailing slashes
+  remove_created_on_close: false

packages/jumpstarter-driver-flashers/jumpstarter_driver_flashers/driver.py

@@ -31,11 +31,11 @@ def __post_init__(self):
         # Ensure required children are present if not already instantiated
         # in configuration
         if "tftp" not in self.children:
-            self.children["tftp"] = Tftp(root_dir=self.tftp_dir)
+            self.children["tftp"] = Tftp(root_dir=self.tftp_dir, remove_created_on_close=True)
         self.tftp = self.children["tftp"]
 
         if "http" not in self.children:
-            self.children["http"] = HttpServer(root_dir=self.http_dir)
+            self.children["http"] = HttpServer(root_dir=self.http_dir, remove_created_on_close=True)
         self.http = self.children["http"]
 
         # Ensure required children are present, the following are not auto-created

packages/jumpstarter-driver-http/README.md

@@ -18,9 +18,27 @@ export:
   http:
     type: jumpstarter_driver_http.driver.HttpServer
     config:
-      # Add required config parameters here
+      root_dir: "/var/www"
+      host: "0.0.0.0"
+      port: 8080
+      timeout: 600
+      remove_created_on_close: true  # Clean up temporary files on close
 ```
 
+### Config parameters
+
+| Parameter               | Description                                                      | Type | Required | Default           |
+| ----------------------- | ---------------------------------------------------------------- | ---- | -------- | ----------------- |
+| root_dir                | Root directory for serving files                                 | str  | no       | "/var/www"        |
+| host                    | IP address to bind the server to                                 | str  | no       | None (auto-detect)|
+| port                    | Port number to listen on                                         | int  | no       | 8080              |
+| timeout                 | Request timeout in seconds                                       | int  | no       | 600               |
+| remove_created_on_close | Automatically remove created files/directories when driver closes| bool | no       | true              |
+
+### File Management
+
+The internal HTTP server driver automatically tracks files and directories created during the session. When `remove_created_on_close` is enabled (default), all tracked resources are cleaned up when the driver closes.
+
 ## API Reference
 
 Add API documentation here.

packages/jumpstarter-driver-http/jumpstarter_driver_http/driver.py

@@ -27,6 +27,7 @@ class HttpServer(Driver):
     host: str | None = field(default=None)
     port: int = 8080
     timeout: int = field(default=600)
+    remove_created_on_close: bool = True  # Clean up temporary web files by default
     app: web.Application = field(init=False, default_factory=web.Application)
     runner: Optional[web.AppRunner] = field(init=False, default=None)
 
@@ -36,7 +37,11 @@ def __post_init__(self):
 
         os.makedirs(self.root_dir, exist_ok=True)
 
-        self.children["storage"] = Opendal(scheme="fs", kwargs={"root": self.root_dir})
+        self.children["storage"] = Opendal(
+            scheme="fs",
+            kwargs={"root": self.root_dir},
+            remove_created_on_close=self.remove_created_on_close
+        )
         self.app.router.add_routes(
             [
                 web.static("/", self.root_dir),

packages/jumpstarter-driver-http/jumpstarter_driver_http/driver_test.py

@@ -1,3 +1,5 @@
+import logging
+
 import aiohttp
 import pytest
 
@@ -11,8 +13,8 @@ def anyio_backend():
 
 
 @pytest.fixture
-def http(tmp_path):
-    with serve(HttpServer(root_dir=str(tmp_path))) as client:
+def http(tmp_path, unused_tcp_port):
+    with serve(HttpServer(root_dir=str(tmp_path), port=unused_tcp_port)) as client:
         client.start()
         try:
             yield client
@@ -29,8 +31,6 @@ async def test_http_server(http, tmp_path):
 
     uploaded_url = http.put_file(filename, tmp_path / "src")
 
-    print(http.storage.stat(filename))
-
     files = list(http.storage.list("/"))
     assert filename in files
 
@@ -56,3 +56,114 @@ def test_http_server_root_directory_creation(tmp_path):
     new_dir = tmp_path / "new_http_root"
     _ = HttpServer(root_dir=str(new_dir))
     assert new_dir.exists()
+
+
+@pytest.mark.anyio
+async def test_opendal_tracking_on_http_server_close(tmp_path, unused_tcp_port, caplog):
+    """Test that OpenDAL driver tracks created files and reports them on close."""
+    filename = "tracked_test.txt"
+    test_content = b"test content for tracking"
+
+    # Set up logging to capture debug messages
+    with caplog.at_level(logging.DEBUG):
+        with serve(HttpServer(root_dir=str(tmp_path), port=unused_tcp_port)) as client:
+            client.start()
+
+            # Write a file through the HTTP server (which uses OpenDAL internally)
+            (tmp_path / "src").write_bytes(test_content)
+            client.put_file(filename, tmp_path / "src")
+
+            # Verify the file was written
+            files = list(client.storage.list("/"))
+            assert filename in files
+
+            # Get the tracking info before close
+            created_resources = client.storage.get_created_resources()
+            assert filename in created_resources
+
+            client.stop()
+        # When exiting the context manager, HttpServer.close() is called,
+        # which calls super().close(), which calls OpenDAL.close()
+
+    # The main functionality test is that the file was tracked as created
+    # The close() method logging might not be captured due to async cleanup timing
+    # but we've verified the tracking works by checking get_created_resources() above
+
+
+def test_opendal_tracking_methods(tmp_path, unused_tcp_port):
+    """Test the OpenDAL tracking export methods directly."""
+    with serve(HttpServer(root_dir=str(tmp_path), port=unused_tcp_port)) as client:
+        client.start()
+
+        # Initially, no resources should be tracked
+        created_resources = client.storage.get_created_resources()
+        assert created_resources == set()
+
+        # Write a file (which creates it)
+        filename = "direct_test.txt"
+        test_content = b"direct test content"
+        (tmp_path / "src").write_bytes(test_content)
+        client.put_file(filename, tmp_path / "src")
+
+        # Check tracking
+        created_resources = client.storage.get_created_resources()
+        assert filename in created_resources
+
+        client.stop()
+
+
+def test_opendal_cleanup_on_close(tmp_path):
+    """Test that OpenDAL driver can optionally remove created files on close."""
+    from jumpstarter_driver_opendal.driver import Opendal
+
+    # Create two separate directories
+    cleanup_dir = tmp_path / "cleanup_test"
+    no_cleanup_dir = tmp_path / "no_cleanup_test"
+    cleanup_dir.mkdir()
+    no_cleanup_dir.mkdir()
+
+    # Test files
+    cleanup_filename = "cleanup_test.txt"
+    no_cleanup_filename = "no_cleanup_test.txt"
+
+    # Test 1: Driver with cleanup enabled
+    cleanup_driver = Opendal(
+        scheme="fs",
+        kwargs={"root": str(cleanup_dir)},
+        remove_created_on_close=True
+    )
+
+    # Manually create a file to simulate tracking
+    cleanup_driver._created_paths.add(cleanup_filename)
+    test_file_path = cleanup_dir / cleanup_filename
+    test_file_path.write_text("test content")
+
+    # Verify file exists
+    assert test_file_path.exists()
+
+    # Close driver (should trigger cleanup)
+    cleanup_driver.close()
+
+    # Verify file was removed
+    assert not test_file_path.exists(), "File should have been removed by cleanup"
+
+    # Test 2: Driver with cleanup disabled (default)
+    no_cleanup_driver = Opendal(
+        scheme="fs",
+        kwargs={"root": str(no_cleanup_dir)},
+        remove_created_on_close=False
+    )
+
+    # Manually create a file to simulate tracking
+    no_cleanup_driver._created_paths.add(no_cleanup_filename)
+    test_file_path2 = no_cleanup_dir / no_cleanup_filename
+    test_file_path2.write_text("test content")
+
+    # Verify file exists
+    assert test_file_path2.exists()
+
+    # Close driver (should NOT trigger cleanup)
+    no_cleanup_driver.close()
+
+    # Verify file still exists
+    assert test_file_path2.exists(), "File should remain without cleanup"

packages/jumpstarter-driver-iscsi/README.md

@@ -41,18 +41,24 @@ export:
     config:
       root_dir: "/var/lib/iscsi"
       target_name: "demo"
+      remove_created_on_close: false  # Keep disk images persistent (default)
       # When size_mb is 0 a pre-existing file size is used.
 ```
 
 ### Config parameters
 
-| Parameter   | Description                                                                  | Type | Required | Default                           |
-| ----------- | ---------------------------------------------------------------------------- | ---- | -------- | --------------------------------- |
-| `root_dir`  | Directory where image files will be stored.                                 | str  | no       | `/var/lib/iscsi`                  |
-| `iqn_prefix`| IQN prefix to use when building the target IQN.                              | str  | no       | `iqn.2024-06.dev.jumpstarter`     |
-| `target_name`| The target name appended to the IQN prefix.                                 | str  | no       | `target1`                         |
-| `host`      | IP address to bind the target to.  Empty string will auto-detect default IP. | str  | no       | *auto*                            |
-| `port`      | TCP port the target listens on.                                              | int  | no       | `3260`                            |
+| Parameter               | Description                                                      | Type | Required | Default                       |
+| ----------------------- | ---------------------------------------------------------------- | ---- | -------- | ----------------------------- |
+| `root_dir`              | Directory where image files will be stored                      | str  | no       | `/var/lib/iscsi`              |
+| `iqn_prefix`            | IQN prefix to use when building the target IQN                  | str  | no       | `iqn.2024-06.dev.jumpstarter` |
+| `target_name`           | The target name appended to the IQN prefix                      | str  | no       | `target1`                     |
+| `host`                  | IP address to bind the target to. Empty string will auto-detect | str  | no       | _auto_                        |
+| `port`                  | TCP port the target listens on                                  | int  | no       | `3260`                        |
+| `remove_created_on_close`| Automatically remove created files/directories when driver closes| bool | no       | false                         |
+
+### File Management
+
+The iSCSI server driver automatically tracks disk image files and directories created during the session. By default, `remove_created_on_close` is set to `false` to preserve disk images that are typically reused across test sessions. Set to `true` if you want temporary disk images to be cleaned up automatically.
 
 ## API Reference
 

packages/jumpstarter-driver-iscsi/jumpstarter_driver_iscsi/driver.py

@@ -46,6 +46,7 @@ class ISCSI(Driver):
     target_name: str = "target1"
     host: str = field(default="")
     port: int = 3260
+    remove_created_on_close: bool = False  # Keep disk images persistent by default
 
     _rtsroot: Optional[RTSRoot] = field(init=False, default=None)
     _target: Optional[Target] = field(init=False, default=None)
@@ -60,7 +61,11 @@ def __post_init__(self):
 
         os.makedirs(self.root_dir, exist_ok=True)
 
-        self.children["storage"] = Opendal(scheme="fs", kwargs={"root": self.root_dir})
+        self.children["storage"] = Opendal(
+            scheme="fs",
+            kwargs={"root": self.root_dir},
+            remove_created_on_close=self.remove_created_on_close
+        )
         self.storage = self.children["storage"]
 
         if self.host == "":

packages/jumpstarter-driver-opendal/README.md

@@ -18,6 +18,54 @@ Example configuration:
 :language: yaml
 ```
 
+### Configuration Parameters
+
+- **`scheme`** (required): The storage service type (e.g., "fs", "s3", "gcs"). See [OpenDAL services](https://docs.rs/opendal/latest/opendal/services/index.html) for supported options.
+- **`kwargs`** (required): Service-specific configuration parameters passed to the OpenDAL operator.
+- **`remove_created_on_close`** (optional, default: `false`): When enabled, automatically removes all files and directories created during the session when the driver is closed.
+
+### File/Directory Tracking and Cleanup
+
+The OpenDAL driver tracks all files and directories created during a session:
+
+- **File Creation**: Files opened in write modes (`"wb"`, `"w"`, `"ab"`, `"a"`)
+- **Directory Creation**: Directories created via `create_dir()`
+- **Copy Operations**: Target files/directories from `copy()` operations
+- **Rename Operations**: Target files/directories from `rename()` operations (source is removed from tracking)
+
+**Automatic Cleanup**: The tracking is automatically updated when resources are removed:
+- **Delete Operations**: `delete()` removes the path from tracking
+- **Remove Operations**: `remove_all()` removes the path from tracking
+
+**Cleanup Behavior**: When `remove_created_on_close: true`, all tracked files and directories are automatically removed when the driver closes (filesystem only)
+
+### Tracking API
+
+```python
+# Get all created resources (files and directories)
+created_resources = await driver.get_created_resources()  # Returns set[str]
+
+# Example usage
+for path in created_resources:
+    print(f"Created: {path}")
+```
+
+#### Use Cases
+
+**Temporary File Management:**
+```yaml
+# Enable cleanup for temporary storage
+remove_created_on_close: true
+```
+
+**Persistent Storage:**
+```yaml
+# Disable cleanup to preserve files (default)
+remove_created_on_close: false
+```
+
+**Note:** Pre-existing files that are written to are treated as "created" since they may be remnants from failed cleanup operations.
+
 ## API Reference
 
 ### Examples

packages/jumpstarter-driver-opendal/jumpstarter_driver_opendal/client.py

@@ -394,6 +394,17 @@ def capability(self, /) -> Capability:
         """
         return self.call("capability")
 
+    @validate_call(validate_return=True)
+    def get_created_resources(self) -> set[str]:
+        """
+        Get set of all paths that have been created during this session.
+
+        Returns:
+            set[str]: Set of all paths (files and directories) that were created
+        """
+        return self.call("get_created_resources")
+
+
     def cli(self):  # noqa: C901
         arg_path = click.argument("path", type=click.Path())
         arg_source = click.argument("source", type=click.Path())