Integrate ignore_patterns into Models upload

src/kaggle/api/kaggle_api_extended.py

@@ -416,30 +416,105 @@ def __str__(self):
         return self.value
 
 
+DEFAULT_IGNORE_PATTERNS = [
+    ".git/",
+    "*/.git/",
+    ".cache/",
+    ".huggingface/",
+]
+
+
+def should_ignore(rel_path: str, is_dir: bool, patterns: List[str]) -> bool:
+    """Helper to check if a path should be ignored based on patterns."""
+    import fnmatch
+
+    rel_path = rel_path.replace(os.path.sep, "/")
+    match_path = rel_path + "/" if is_dir and not rel_path.endswith("/") else rel_path
+
+    for pattern in patterns:
+        pattern = pattern.replace(os.path.sep, "/")
+        if pattern.endswith("/"):
+            if not is_dir:
+                continue
+        if fnmatch.fnmatch(match_path, pattern):
+            return True
+    return False
+
+
 class DirectoryArchive(object):
-    """
-    Context manager for handling directory archives.
+    """Context manager for handling directory archives with filtering.
 
-    This class provides a context manager for working with directory archives in various formats.
-    It manages the lifecycle of the archive, including opening and closing resources as needed.
+    This class provides a context manager for working with directory archives in
+    various formats. It manages the lifecycle of the archive, including opening
+    and closing resources as needed. It supports filtering files based on
+    ignore patterns.
     """
 
-    def __init__(self, fullpath, fmt):
+    def __init__(self, fullpath, fmt, ignore_patterns=None):
         self._fullpath = fullpath
         self._format = fmt
+        self._ignore_patterns = ignore_patterns or []
         self.name = None
         self.path = None
 
     def __enter__(self):
         self._temp_dir = tempfile.mkdtemp()
         _, dir_name = os.path.split(self._fullpath)
-        self.path = shutil.make_archive(os.path.join(self._temp_dir, dir_name), self._format, self._fullpath)
+        archive_base_path = os.path.join(self._temp_dir, dir_name)
+
+        if self._ignore_patterns:
+            self.path = self._create_archive_with_filters(archive_base_path)
+        else:
+            self.path = shutil.make_archive(archive_base_path, self._format, self._fullpath)
+
         _, self.name = os.path.split(self.path)
         return self
 
     def __exit__(self, *args):
         shutil.rmtree(self._temp_dir)
 
+    def _create_archive_with_filters(self, base_name):
+        if self._format == "zip":
+            archive_path = base_name + ".zip"
+            with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as zipf:
+                for root, dirs, files in os.walk(self._fullpath):
+                    dirs[:] = [
+                        d
+                        for d in dirs
+                        if not should_ignore(
+                            os.path.relpath(os.path.join(root, d), self._fullpath),
+                            True,
+                            self._ignore_patterns,
+                        )
+                    ]
+                    for file in files:
+                        file_path = os.path.join(root, file)
+                        rel_path = os.path.relpath(file_path, self._fullpath)
+                        if not should_ignore(rel_path, False, self._ignore_patterns):
+                            zipf.write(file_path, rel_path)
+            return archive_path
+        elif self._format == "tar":
+            archive_path = base_name + ".tar"
+            with tarfile.open(archive_path, "w") as tarf:
+                for root, dirs, files in os.walk(self._fullpath):
+                    dirs[:] = [
+                        d
+                        for d in dirs
+                        if not should_ignore(
+                            os.path.relpath(os.path.join(root, d), self._fullpath),
+                            True,
+                            self._ignore_patterns,
+                        )
+                    ]
+                    for file in files:
+                        file_path = os.path.join(root, file)
+                        rel_path = os.path.relpath(file_path, self._fullpath)
+                        if not should_ignore(rel_path, False, self._ignore_patterns):
+                            tarf.add(file_path, rel_path)
+            return archive_path
+        else:
+            raise ValueError(f"Unsupported archive format: {self._format}")
+
 
 class ResumableUploadContext(object):
     """
@@ -7020,7 +7095,13 @@ def model_instance_initialize_cli(self, folder):
         folder = folder or os.getcwd()
         self.model_instance_initialize(folder)
 
-    def model_instance_create(self, folder: str, quiet: bool = False, dir_mode: str = "skip") -> ApiCreateModelResponse:
+    def model_instance_create(
+        self,
+        folder: str,
+        quiet: bool = False,
+        dir_mode: str = "skip",
+        ignore_patterns: Optional[List[str]] = None,
+    ) -> ApiCreateModelResponse:
         """Creates a new model instance.
 
         Args:
@@ -7090,20 +7171,30 @@ def model_instance_create(self, folder: str, quiet: bool = False, dir_mode: str
             request.body = body
             message = kaggle.models.model_api_client.create_model_instance
             with ResumableUploadContext() as upload_context:
-                self.upload_files(body, None, folder, ApiBlobType.MODEL, upload_context, quiet, dir_mode)
+                self.upload_files(
+                    body,
+                    None,
+                    folder,
+                    ApiBlobType.MODEL,
+                    upload_context,
+                    quiet,
+                    dir_mode,
+                    ignore_patterns,
+                )
                 response = cast(ApiCreateModelResponse, self.with_retry(message)(request))
                 return response
 
-    def model_instance_create_cli(self, folder, quiet=False, dir_mode="skip"):
+    def model_instance_create_cli(self, folder, quiet=False, dir_mode="skip", ignore_patterns=None):
         """A client wrapper for creating a new model instance.
 
         Args:
             folder: The folder from which to get the metadata file.
             quiet: Suppress verbose output (default is False).
             dir_mode: What to do with directories: "skip" - ignore; "zip" - compress and upload.
+            ignore_patterns: Patterns of files/dirs to ignore.
         """
         folder = folder or os.getcwd()
-        result = self.model_instance_create(folder, quiet, dir_mode)
+        result = self.model_instance_create(folder, quiet, dir_mode, ignore_patterns)
 
         if result.id:
             print("Your model instance was created. Id={}. Url={}".format(result.id, result.url))
@@ -7358,7 +7449,13 @@ def model_instance_update_cli(self, folder=None):
             print("Model update error: " + result.error)
 
     def model_instance_version_create(
-        self, model_instance: str, folder: str, version_notes: str = "", quiet: bool = False, dir_mode: str = "skip"
+        self,
+        model_instance: str,
+        folder: str,
+        version_notes: str = "",
+        quiet: bool = False,
+        dir_mode: str = "skip",
+        ignore_patterns: Optional[List[str]] = None,
     ) -> ApiCreateModelResponse:
         """Creates a new model instance version.
 
@@ -7385,11 +7482,28 @@ def model_instance_version_create(
         with self.build_kaggle_client() as kaggle:
             message = kaggle.models.model_api_client.create_model_instance_version
             with ResumableUploadContext() as upload_context:
-                self.upload_files(body, None, folder, ApiBlobType.MODEL, upload_context, quiet, dir_mode)
+                self.upload_files(
+                    body,
+                    None,
+                    folder,
+                    ApiBlobType.MODEL,
+                    upload_context,
+                    quiet,
+                    dir_mode,
+                    ignore_patterns,
+                )
                 response = cast(ApiCreateModelResponse, self.with_retry(message)(request))
                 return response
 
-    def model_instance_version_create_cli(self, model_instance, folder, version_notes="", quiet=False, dir_mode="skip"):
+    def model_instance_version_create_cli(
+        self,
+        model_instance,
+        folder,
+        version_notes="",
+        quiet=False,
+        dir_mode="skip",
+        ignore_patterns=None,
+    ):
         """A client wrapper for creating a new version of a model instance.
 
         Args:
@@ -7399,8 +7513,11 @@ def model_instance_version_create_cli(self, model_instance, folder, version_note
             version_notes: The version notes to record for this new version.
             quiet: Suppress verbose output (default is False).
             dir_mode: What to do with directories: "skip" - ignore; "zip" - compress and upload.
+            ignore_patterns: Patterns of files/dirs to ignore.
         """
-        result = self.model_instance_version_create(model_instance, folder, version_notes, quiet, dir_mode)
+        result = self.model_instance_version_create(
+            model_instance, folder, version_notes, quiet, dir_mode, ignore_patterns
+        )
 
         if result.id != 0:
             print("Your model instance version was created. Url={}".format(result.url))
@@ -7938,6 +8055,7 @@ def upload_files(
         upload_context: ResumableUploadContext,
         quiet: bool = False,
         dir_mode: str = "skip",
+        ignore_patterns: Optional[List[str]] = None,
     ) -> None:
         """Uploads files in a folder.
 
@@ -7949,10 +8067,12 @@ def upload_files(
             upload_context (ResumableUploadContext): The context for resumable uploads.
             quiet (bool): Suppress verbose output (default is False).
             dir_mode (str): What to do with directories: "skip" - ignore; "zip" - compress and upload.
+            ignore_patterns (Optional[List[str]]): Patterns of files/dirs to ignore.
 
         Returns:
             None:
         """
+        patterns = DEFAULT_IGNORE_PATTERNS + (ignore_patterns or [])
         for file_name in os.listdir(folder):
             if file_name in [
                 self.DATASET_METADATA_FILE,
@@ -7963,8 +8083,21 @@ def upload_files(
                 self.MODEL_INSTANCE_METADATA_FILE,
             ]:
                 continue
+
+            full_path = os.path.join(folder, file_name)
+            is_dir = os.path.isdir(full_path)
+            if should_ignore(file_name, is_dir, patterns):
+                continue
+
             upload_file = self._upload_file_or_folder(
-                folder, file_name, blob_type, upload_context, dir_mode, quiet, resources
+                folder,
+                file_name,
+                blob_type,
+                upload_context,
+                dir_mode,
+                quiet,
+                resources,
+                patterns,
             )
             if upload_file is not None:
                 files = request.files
@@ -7988,16 +8121,29 @@ def _upload_file_or_folder(
         dir_mode: str,
         quiet: bool = False,
         resources: Optional[List[Dict[str, Union[str, Dict[str, List[Dict[str, str]]]]]]] = None,
+        ignore_patterns: Optional[List[str]] = None,
     ) -> Union[UploadFile, None]:
         full_path = os.path.join(parent_path, file_or_folder_name)
         upload_file = None
         if os.path.isfile(full_path):
-            upload_file = self._upload_file(file_or_folder_name, full_path, blob_type, upload_context, quiet, resources)
+            upload_file = self._upload_file(
+                file_or_folder_name,
+                full_path,
+                blob_type,
+                upload_context,
+                quiet,
+                resources,
+            )
         elif os.path.isdir(full_path):
             if dir_mode in ["zip", "tar"]:
-                with DirectoryArchive(full_path, dir_mode) as archive:
+                with DirectoryArchive(full_path, dir_mode, ignore_patterns=ignore_patterns) as archive:
                     upload_file = self._upload_file(
-                        archive.name, archive.path, blob_type, upload_context, quiet, resources
+                        archive.name,
+                        archive.path,
+                        blob_type,
+                        upload_context,
+                        quiet,
+                        resources,
                     )
             elif not quiet:
                 print("Skipping folder: " + file_or_folder_name + "; use '--dir-mode' to upload folders")

src/kaggle/cli.py

@@ -1571,6 +1571,13 @@ def parse_model_instances(subparsers) -> None:
     parser_model_instances_create_optional.add_argument(
         "-r", "--dir-mode", dest="dir_mode", choices=["skip", "zip", "tar"], default="skip", help=Help.param_dir_mode
     )
+    parser_model_instances_create_optional.add_argument(
+        "--ignore-patterns",
+        dest="ignore_patterns",
+        action="append",
+        required=False,
+        help="Patterns to ignore when uploading files/dirs",
+    )
     parser_model_instances_create._action_groups.append(parser_model_instances_create_optional)
     parser_model_instances_create.set_defaults(func=api.model_instance_create_cli)
 
@@ -1676,6 +1683,13 @@ def parse_model_instance_versions(subparsers) -> None:
     parser_model_instance_versions_create_optional.add_argument(
         "-r", "--dir-mode", dest="dir_mode", choices=["skip", "zip", "tar"], default="skip", help=Help.param_dir_mode
     )
+    parser_model_instance_versions_create_optional.add_argument(
+        "--ignore-patterns",
+        dest="ignore_patterns",
+        action="append",
+        required=False,
+        help="Patterns to ignore when uploading files/dirs",
+    )
     parser_model_instance_versions_create._action_groups.append(parser_model_instance_versions_create_optional)
     parser_model_instance_versions_create.set_defaults(func=api.model_instance_version_create_cli)
 

tests/unit/test_cli_models.py

@@ -203,6 +203,23 @@ def test_model_instances_create_parser_with_flags_succeeds(parser):
     assert kwargs["folder"] == "/path/to/instance"
     assert kwargs["quiet"] is True
     assert kwargs["dir_mode"] == "zip"
+    assert kwargs.get("ignore_patterns") is None
+
+
+def test_model_instances_create_parser_with_ignore_patterns_succeeds(parser):
+    func, kwargs = parser.dispatch(
+        [
+            "models",
+            "instances",
+            "create",
+            "--ignore-patterns",
+            "*.tmp",
+            "--ignore-patterns",
+            "temp/",
+        ]
+    )
+    assert func.__name__ == "model_instance_create_cli"
+    assert kwargs["ignore_patterns"] == ["*.tmp", "temp/"]
 
 
 def test_model_instances_files_parser_missing_instance_fails(parser):
@@ -346,6 +363,28 @@ def test_model_instance_versions_create_parser_succeeds(parser):
     assert kwargs["version_notes"] == "version notes"
     assert kwargs["quiet"] is True
     assert kwargs["dir_mode"] == "tar"
+    assert kwargs.get("ignore_patterns") is None
+
+
+def test_model_instance_versions_create_parser_with_ignore_patterns_succeeds(
+    parser,
+):
+    func, kwargs = parser.dispatch(
+        [
+            "models",
+            "instances",
+            "versions",
+            "create",
+            "owner/model/framework/variation",
+            "--ignore-patterns",
+            "*.tmp",
+            "--ignore-patterns",
+            "temp/",
+        ]
+    )
+    assert func.__name__ == "model_instance_version_create_cli"
+    assert kwargs["model_instance"] == "owner/model/framework/variation"
+    assert kwargs["ignore_patterns"] == ["*.tmp", "temp/"]
 
 
 def test_model_instance_versions_download_parser_missing_version_fails(parser):