From e2bc79f9d2ee14d72836b817420bff38bb0d5c83 Mon Sep 17 00:00:00 2001 From: Tamir Date: Fri, 8 Aug 2025 10:39:22 +0300 Subject: [PATCH 01/10] Refactor volume mount configuration and add shared mount Use separate classes for each volume mount type. Added support for shared filesystem volume mounts. The base VolumeMount class now serves as an abstract base, and specific volume mount types like ScratchMount, SecretMount, MemoryMount, and SharedFilesystemMount inherit from it. --- datacrunch/containers/containers.py | 75 ++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 7 deletions(-) diff --git a/datacrunch/containers/containers.py b/datacrunch/containers/containers.py index d64c0ac..aaee368 100644 --- a/datacrunch/containers/containers.py +++ b/datacrunch/containers/containers.py @@ -43,6 +43,7 @@ class VolumeMountType(str, Enum): SCRATCH = "scratch" SECRET = "secret" MEMORY = "memory" + SHARED = "shared" class ContainerRegistryType(str, Enum): @@ -119,25 +120,85 @@ class EnvVar: @dataclass_json(undefined=Undefined.EXCLUDE) @dataclass class VolumeMount: - """Volume mount configuration for containers. + """Base class for volume mount configurations. Attributes: type: Type of volume mount. mount_path: Path where the volume should be mounted in the container. - size_in_mb: Size of the memory volume in megabytes, only used for memory volume mounts """ type: VolumeMountType mount_path: str - size_in_mb: Optional[int] = None -@dataclass_json +@dataclass_json(undefined=Undefined.EXCLUDE) @dataclass -class SecretMount: - mount_path: str +class ScratchMount(VolumeMount): + """Scratch volume mount configuration. + """ + + def __init__(self, mount_path: str): + """Initialize a scratch volume mount. + + Args: + mount_path: Path where the volume should be mounted in the container. + """ + super().__init__(type=VolumeMountType.SCRATCH, mount_path=mount_path) + + +@dataclass_json(undefined=Undefined.EXCLUDE) +@dataclass +class SecretMount(VolumeMount): + """Secret volume mount configuration. + + A secret volume mount allows mounting secret files into the container. + + Attributes: + secret_name: Name of the fileset secret to mount + file_names: List of file names that are part of the fileset secret + """ + secret_name: str - type: VolumeMountType = VolumeMountType.SECRET + file_names: Optional[List[str]] = None + + def __init__(self, mount_path: str, secret_name: str, file_names: Optional[List[str]] = None): + super().__init__(type=VolumeMountType.SECRET, mount_path=mount_path) + self.secret_name = secret_name + self.file_names = file_names + + +@dataclass_json(undefined=Undefined.EXCLUDE) +@dataclass +class MemoryMount(VolumeMount): + """Memory volume mount configuration. + + A memory volume mount provides high-speed, ephemeral in-memory storage inside your container. + The mount path is currently hardcoded to /dev/shm and cannot be changed. + + Attributes: + size_in_mb: Size of the memory volume in megabytes. + """ + + size_in_mb: int + + def __init__(self, size_in_mb: int): + super().__init__(type=VolumeMountType.MEMORY, mount_path='/dev/shm') + self.size_in_mb = size_in_mb + + +@dataclass_json(undefined=Undefined.EXCLUDE) +@dataclass +class SharedFilesystemMount(VolumeMount): + """Shared filesystem volume mount configuration. + + A shared filesystem volume mount allows mounting a shared filesystem into the container. + """ + + volume_id: str # The ID of the shared filesystem volume to mount, needs to be created first + + def __init__(self, mount_path: str, volume_id: str): + super().__init__(type=VolumeMountType.SHARED, mount_path=mount_path) + self.volume_id = volume_id @dataclass_json From 0eeff2f2c832806444be65617c446b5245f9bf51 Mon Sep 17 00:00:00 2001 From: Tamir Date: Fri, 8 Aug 2025 12:10:11 +0300 Subject: [PATCH 02/10] Container image can be also dict, --- datacrunch/containers/__init__.py | 2 ++ datacrunch/containers/containers.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/datacrunch/containers/__init__.py b/datacrunch/containers/__init__.py index 6a95373..91751f8 100644 --- a/datacrunch/containers/__init__.py +++ b/datacrunch/containers/__init__.py @@ -7,6 +7,8 @@ EntrypointOverridesSettings, VolumeMount, SecretMount, + SharedFilesystemMount, + ScratchMount, VolumeMountType, Container, ContainerRegistryCredentials, diff --git a/datacrunch/containers/containers.py b/datacrunch/containers/containers.py index aaee368..d3bf233 100644 --- a/datacrunch/containers/containers.py +++ b/datacrunch/containers/containers.py @@ -8,7 +8,7 @@ import os from dataclasses import dataclass, field from dataclasses_json import dataclass_json, Undefined # type: ignore -from typing import List, Optional, Dict, Any +from typing import List, Optional, Dict, Any, Union from enum import Enum from datacrunch.http_client.http_client import HTTPClient @@ -216,7 +216,7 @@ class Container: volume_mounts: Optional list of volume mounts. """ - image: str + image: Union[str, dict] exposed_port: int name: Optional[str] = None healthcheck: Optional[HealthcheckSettings] = None From 3c9e858eed492030c25caf7f998c59125e94b169 Mon Sep 17 00:00:00 2001 From: Tamir Date: Fri, 8 Aug 2025 15:44:13 +0300 Subject: [PATCH 03/10] added to example --- .../containers/container_deployments_example.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/examples/containers/container_deployments_example.py b/examples/containers/container_deployments_example.py index 8001fb4..1b8b5a1 100644 --- a/examples/containers/container_deployments_example.py +++ b/examples/containers/container_deployments_example.py @@ -20,11 +20,11 @@ QueueLoadScalingTrigger, UtilizationScalingTrigger, HealthcheckSettings, - VolumeMount, + ScratchMount, SecretMount, + SharedFilesystemMount, ContainerRegistrySettings, Deployment, - VolumeMountType, ContainerDeploymentStatus, ) @@ -97,23 +97,24 @@ def main() -> None: path="/health" ), volume_mounts=[ - # Shared memory volume - VolumeMount( - type=VolumeMountType.SCRATCH, + ScratchMount( mount_path="/data" ), - # Fileset secret + # Optional: Fileset secret SecretMount( mount_path="/path/to/mount", secret_name="my-fileset-secret" # This fileset secret must be created beforehand - ) + ), + # Optional: Mount an existing shared filesystem volume + SharedFilesystemMount( + mount_path="/sfs", volume_id=""), ], env=[ # Secret environment variables needed to be added beforehand EnvVar( name="HF_TOKEN", # This is a reference to a secret already created - value_or_reference_to_secret="hf_token", + value_or_reference_to_secret="hf-token", type=EnvVarType.SECRET ), # Plain environment variables can be added directly From f0d63bb1a09a030f80570cd3ca57cc5a8f18d9c2 Mon Sep 17 00:00:00 2001 From: Tamir Date: Wed, 13 Aug 2025 09:29:56 +0300 Subject: [PATCH 04/10] re-add deprecated field size_in_mb --- datacrunch/containers/containers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datacrunch/containers/containers.py b/datacrunch/containers/containers.py index d3bf233..e5cd0ef 100644 --- a/datacrunch/containers/containers.py +++ b/datacrunch/containers/containers.py @@ -125,10 +125,13 @@ class VolumeMount: Attributes: type: Type of volume mount. mount_path: Path where the volume should be mounted in the container. + size_in_mb: Size of the volume in megabytes. Deprecated: use MemoryMount for memory volumes instead. """ type: VolumeMountType mount_path: str + # Deprecated: use MemoryMount for memory volumes instead. + size_in_mb: Optional[int] = None @dataclass_json(undefined=Undefined.EXCLUDE) From 80f97b5f7f292bc15887970fde1895c12ea12530 Mon Sep 17 00:00:00 2001 From: Tamir Date: Wed, 13 Aug 2025 11:59:28 +0300 Subject: [PATCH 05/10] use kw_only=True to prevent to fix class init non-default argument err --- datacrunch/containers/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datacrunch/containers/containers.py b/datacrunch/containers/containers.py index e5cd0ef..cf15edc 100644 --- a/datacrunch/containers/containers.py +++ b/datacrunch/containers/containers.py @@ -131,7 +131,7 @@ class VolumeMount: type: VolumeMountType mount_path: str # Deprecated: use MemoryMount for memory volumes instead. - size_in_mb: Optional[int] = None + size_in_mb: Optional[int] = field(default=None, kw_only=True) @dataclass_json(undefined=Undefined.EXCLUDE) @@ -165,9 +165,9 @@ class SecretMount(VolumeMount): file_names: Optional[List[str]] = None def __init__(self, mount_path: str, secret_name: str, file_names: Optional[List[str]] = None): - super().__init__(type=VolumeMountType.SECRET, mount_path=mount_path) self.secret_name = secret_name self.file_names = file_names + super().__init__(type=VolumeMountType.SECRET, mount_path=mount_path) @dataclass_json(undefined=Undefined.EXCLUDE) From f94f626784316ec1eaa0b74fb5e29c11f9a90ffc Mon Sep 17 00:00:00 2001 From: Tamir Date: Wed, 13 Aug 2025 13:40:41 +0300 Subject: [PATCH 06/10] removed support for python v3.9, renamed classname --- .github/workflows/code_style.yml | 2 +- .github/workflows/unit_tests.yml | 2 +- datacrunch/containers/__init__.py | 2 +- datacrunch/containers/containers.py | 2 +- examples/containers/container_deployments_example.py | 6 +++--- setup.py | 3 +-- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 2d36207..b0053a9 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 850e765..6060541 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v2 diff --git a/datacrunch/containers/__init__.py b/datacrunch/containers/__init__.py index 91751f8..5100d25 100644 --- a/datacrunch/containers/__init__.py +++ b/datacrunch/containers/__init__.py @@ -7,7 +7,7 @@ EntrypointOverridesSettings, VolumeMount, SecretMount, - SharedFilesystemMount, + SharedFileSystemMount, ScratchMount, VolumeMountType, Container, diff --git a/datacrunch/containers/containers.py b/datacrunch/containers/containers.py index cf15edc..40cb331 100644 --- a/datacrunch/containers/containers.py +++ b/datacrunch/containers/containers.py @@ -191,7 +191,7 @@ def __init__(self, size_in_mb: int): @dataclass_json(undefined=Undefined.EXCLUDE) @dataclass -class SharedFilesystemMount(VolumeMount): +class SharedFileSystemMount(VolumeMount): """Shared filesystem volume mount configuration. A shared filesystem volume mount allows mounting a shared filesystem into the container. diff --git a/examples/containers/container_deployments_example.py b/examples/containers/container_deployments_example.py index 1b8b5a1..ba5ff74 100644 --- a/examples/containers/container_deployments_example.py +++ b/examples/containers/container_deployments_example.py @@ -22,7 +22,7 @@ HealthcheckSettings, ScratchMount, SecretMount, - SharedFilesystemMount, + SharedFileSystemMount, ContainerRegistrySettings, Deployment, ContainerDeploymentStatus, @@ -106,8 +106,8 @@ def main() -> None: secret_name="my-fileset-secret" # This fileset secret must be created beforehand ), # Optional: Mount an existing shared filesystem volume - SharedFilesystemMount( - mount_path="/sfs", volume_id=""), + SharedFileSystemMount( + mount_path="/sfs", volume_id=""), ], env=[ # Secret environment variables needed to be added beforehand diff --git a/setup.py b/setup.py index 99d5608..8d1b925 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ }, classifiers=[ "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', @@ -41,5 +40,5 @@ "Operating System :: OS Independent", "Natural Language :: English" ], - python_requires='>=3.9', + python_requires='>=3.10', ) From f2140411a067e3e78564b96a8cb40be27f219a19 Mon Sep 17 00:00:00 2001 From: Tamir Date: Wed, 13 Aug 2025 13:45:51 +0300 Subject: [PATCH 07/10] better doc --- datacrunch/containers/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datacrunch/containers/containers.py b/datacrunch/containers/containers.py index 40cb331..2b826b1 100644 --- a/datacrunch/containers/containers.py +++ b/datacrunch/containers/containers.py @@ -157,8 +157,8 @@ class SecretMount(VolumeMount): A secret volume mount allows mounting secret files into the container. Attributes: - secret_name: Name of the fileset secret to mount - file_names: List of file names that are part of the fileset secret + secret_name: The name of the fileset secret to mount. This secret must be created in advance, for example using `create_fileset_secret_from_file_paths` + file_names: List of file names that are part of the fileset secret. """ secret_name: str From bcde87e75832f166dbbd5a9a50dbb8f87fc1fa82 Mon Sep 17 00:00:00 2001 From: Tamir Date: Wed, 13 Aug 2025 13:57:58 +0300 Subject: [PATCH 08/10] changelog entry --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c97ae6d..4fc43d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added `SharedFileSystemMount` class for container sfs support +- Added `SecretMount` and `ScratchMount` classes that inherit from base `VolumeMount` + +### Changed + +- Removed support for python 3.9 as it doesn't support `kw_only` and reaches EOS state in 2 months + ## [1.13.2] - 2025-06-04 ### Changed From 26d091019d86ecaf00f659577f4151821c980437 Mon Sep 17 00:00:00 2001 From: Tamir Date: Wed, 13 Aug 2025 14:23:19 +0300 Subject: [PATCH 09/10] renamed `ScratchMount` to `GeneralStorageMount` --- CHANGELOG.md | 2 +- datacrunch/containers/containers.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fc43d5..7c89be8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added `SharedFileSystemMount` class for container sfs support -- Added `SecretMount` and `ScratchMount` classes that inherit from base `VolumeMount` +- Added `SecretMount` and `GeneralStorageMount` classes that inherit from base `VolumeMount` ### Changed diff --git a/datacrunch/containers/containers.py b/datacrunch/containers/containers.py index 2b826b1..d614041 100644 --- a/datacrunch/containers/containers.py +++ b/datacrunch/containers/containers.py @@ -136,12 +136,12 @@ class VolumeMount: @dataclass_json(undefined=Undefined.EXCLUDE) @dataclass -class ScratchMount(VolumeMount): - """Scratch volume mount configuration. +class GeneralStorageMount(VolumeMount): + """General storage volume mount configuration. """ def __init__(self, mount_path: str): - """Initialize a scratch volume mount. + """Initialize a general scratch volume mount. Args: mount_path: Path where the volume should be mounted in the container. From 6b1e6968392b77e476ac68b4ca29c0bfe1f68b40 Mon Sep 17 00:00:00 2001 From: Tamir Date: Wed, 13 Aug 2025 14:41:18 +0300 Subject: [PATCH 10/10] fixed imports --- datacrunch/containers/__init__.py | 2 +- examples/containers/container_deployments_example.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datacrunch/containers/__init__.py b/datacrunch/containers/__init__.py index 5100d25..dc7a4ab 100644 --- a/datacrunch/containers/__init__.py +++ b/datacrunch/containers/__init__.py @@ -8,7 +8,7 @@ VolumeMount, SecretMount, SharedFileSystemMount, - ScratchMount, + GeneralStorageMount, VolumeMountType, Container, ContainerRegistryCredentials, diff --git a/examples/containers/container_deployments_example.py b/examples/containers/container_deployments_example.py index ba5ff74..e40c4f2 100644 --- a/examples/containers/container_deployments_example.py +++ b/examples/containers/container_deployments_example.py @@ -20,7 +20,7 @@ QueueLoadScalingTrigger, UtilizationScalingTrigger, HealthcheckSettings, - ScratchMount, + GeneralStorageMount, SecretMount, SharedFileSystemMount, ContainerRegistrySettings, @@ -97,7 +97,7 @@ def main() -> None: path="/health" ), volume_mounts=[ - ScratchMount( + GeneralStorageMount( mount_path="/data" ), # Optional: Fileset secret