From 715c74bbea7ed1e5e44c6381f2170d9dc63bf106 Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Thu, 16 Jul 2026 20:14:43 +0000 Subject: [PATCH 1/5] Implement ignore_patterns core logic and unit tests - Define DEFAULT_IGNORE_PATTERNS. - Implement should_ignore helper. - Update DirectoryArchive to support filtering for zip/tar formats. - Add unit tests for should_ignore and DirectoryArchive. TAG=agy CONV=2a820637-152b-46a6-9a45-e29918f1e322 --- src/kaggle/api/kaggle_api_extended.py | 101 +++++++++++++- tests/unit/test_ignore_patterns.py | 182 ++++++++++++++++++++++++++ 2 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_ignore_patterns.py diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index e52aace7..7eb1f1a9 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -416,30 +416,119 @@ def __str__(self): return self.value +DEFAULT_IGNORE_PATTERNS = [ + ".git/", + "*/.git/", + ".cache/", + ".huggingface/", +] + + +def should_ignore(rel_path: str, is_dir: bool, patterns: List[str]) -> bool: + """Helper to check if a path should be ignored based on patterns.""" + import fnmatch + + rel_path = rel_path.replace(os.path.sep, "/") + match_path = ( + rel_path + "/" if is_dir and not rel_path.endswith("/") else rel_path + ) + + for pattern in patterns: + pattern = pattern.replace(os.path.sep, "/") + if pattern.endswith("/"): + if not is_dir: + continue + if fnmatch.fnmatch(match_path, pattern): + return True + return False + + class DirectoryArchive(object): - """ - Context manager for handling directory archives. + """Context manager for handling directory archives with filtering. - This class provides a context manager for working with directory archives in various formats. - It manages the lifecycle of the archive, including opening and closing resources as needed. + This class provides a context manager for working with directory archives in + various formats. It manages the lifecycle of the archive, including opening + and closing resources as needed. It supports filtering files based on + ignore patterns. """ - def __init__(self, fullpath, fmt): + def __init__(self, fullpath, fmt, ignore_patterns=None): self._fullpath = fullpath self._format = fmt + self._ignore_patterns = ignore_patterns or [] self.name = None self.path = None def __enter__(self): self._temp_dir = tempfile.mkdtemp() _, dir_name = os.path.split(self._fullpath) - self.path = shutil.make_archive(os.path.join(self._temp_dir, dir_name), self._format, self._fullpath) + archive_base_path = os.path.join(self._temp_dir, dir_name) + + if self._ignore_patterns: + self.path = self._create_archive_with_filters(archive_base_path) + else: + self.path = shutil.make_archive( + archive_base_path, self._format, self._fullpath + ) + _, self.name = os.path.split(self.path) return self def __exit__(self, *args): shutil.rmtree(self._temp_dir) + def _create_archive_with_filters(self, base_name): + if self._format == "zip": + archive_path = base_name + ".zip" + with zipfile.ZipFile( + archive_path, "w", zipfile.ZIP_DEFLATED + ) as zipf: + for root, dirs, files in os.walk(self._fullpath): + dirs[:] = [ + d + for d in dirs + if not should_ignore( + os.path.relpath( + os.path.join(root, d), self._fullpath + ), + True, + self._ignore_patterns, + ) + ] + for file in files: + file_path = os.path.join(root, file) + rel_path = os.path.relpath(file_path, self._fullpath) + if not should_ignore( + rel_path, False, self._ignore_patterns + ): + zipf.write(file_path, rel_path) + return archive_path + elif self._format == "tar": + archive_path = base_name + ".tar" + with tarfile.open(archive_path, "w") as tarf: + for root, dirs, files in os.walk(self._fullpath): + dirs[:] = [ + d + for d in dirs + if not should_ignore( + os.path.relpath( + os.path.join(root, d), self._fullpath + ), + True, + self._ignore_patterns, + ) + ] + for file in files: + file_path = os.path.join(root, file) + rel_path = os.path.relpath(file_path, self._fullpath) + if not should_ignore( + rel_path, False, self._ignore_patterns + ): + tarf.add(file_path, rel_path) + return archive_path + else: + raise ValueError(f"Unsupported archive format: {self._format}") + class ResumableUploadContext(object): """ diff --git a/tests/unit/test_ignore_patterns.py b/tests/unit/test_ignore_patterns.py new file mode 100644 index 00000000..06312726 --- /dev/null +++ b/tests/unit/test_ignore_patterns.py @@ -0,0 +1,182 @@ +import os +import shutil +import tempfile +import unittest +import zipfile +import tarfile + +from kaggle.api.kaggle_api_extended import should_ignore, DirectoryArchive, DEFAULT_IGNORE_PATTERNS + + +class TestIgnorePatterns(unittest.TestCase): + + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_should_ignore_default_patterns(self): + # .git/ at root should be ignored + self.assertTrue(should_ignore(".git", True, DEFAULT_IGNORE_PATTERNS)) + self.assertTrue(should_ignore(".git/", True, DEFAULT_IGNORE_PATTERNS)) + + # .git/ in sub dir should be ignored (due to */.git/) + self.assertTrue(should_ignore("sub/.git", True, DEFAULT_IGNORE_PATTERNS)) + self.assertTrue( + should_ignore("sub/.git/", True, DEFAULT_IGNORE_PATTERNS) + ) + self.assertTrue( + should_ignore("sub/dir/.git", True, DEFAULT_IGNORE_PATTERNS) + ) + + # .git file (not dir) should NOT be ignored + self.assertFalse(should_ignore(".git", False, DEFAULT_IGNORE_PATTERNS)) + self.assertFalse( + should_ignore("sub/.git", False, DEFAULT_IGNORE_PATTERNS) + ) + + # .cache/ at root should be ignored + self.assertTrue(should_ignore(".cache", True, DEFAULT_IGNORE_PATTERNS)) + + # .cache/ in sub dir should NOT be ignored (no */.cache/ in defaults) + self.assertFalse( + should_ignore("sub/.cache", True, DEFAULT_IGNORE_PATTERNS) + ) + + # .huggingface/ at root should be ignored + self.assertTrue( + should_ignore(".huggingface", True, DEFAULT_IGNORE_PATTERNS) + ) + + # .huggingface/ in sub dir should NOT be ignored + self.assertFalse( + should_ignore("sub/.huggingface", True, DEFAULT_IGNORE_PATTERNS) + ) + + def test_should_ignore_custom_patterns(self): + patterns = ["*.tmp", "ignore_dir/", "*/nested_ignore_dir/", "exact_file.txt"] + + # Wildcard file pattern + self.assertTrue(should_ignore("foo.tmp", False, patterns)) + self.assertTrue(should_ignore("sub/foo.tmp", False, patterns)) + self.assertFalse(should_ignore("foo.tmp.bak", False, patterns)) + + # Directory pattern (root only) + self.assertTrue(should_ignore("ignore_dir", True, patterns)) + self.assertFalse(should_ignore("sub/ignore_dir", True, patterns)) + # Directory pattern (root only) should not match file + self.assertFalse(should_ignore("ignore_dir", False, patterns)) + + # Directory pattern (nested) + self.assertTrue(should_ignore("sub/nested_ignore_dir", True, patterns)) + self.assertTrue( + should_ignore("sub/dir/nested_ignore_dir", True, patterns) + ) + self.assertFalse(should_ignore("nested_ignore_dir", True, patterns)) + + # Exact file pattern + self.assertTrue(should_ignore("exact_file.txt", False, patterns)) + # fnmatch "exact_file.txt" will not match "sub/exact_file.txt" if we don't have wildcard + # Wait, fnmatch("sub/exact_file.txt", "exact_file.txt") is False. + self.assertFalse(should_ignore("sub/exact_file.txt", False, patterns)) + + def test_directory_archive_zip_filtering(self): + # Create dummy directory structure + src_dir = os.path.join(self.temp_dir, "src") + os.makedirs(src_dir) + os.makedirs(os.path.join(src_dir, ".git")) + os.makedirs(os.path.join(src_dir, "sub")) + os.makedirs(os.path.join(src_dir, "sub", ".git")) + os.makedirs(os.path.join(src_dir, "ignore_dir")) + + # Create files + self._create_file(os.path.join(src_dir, "keep.txt")) + self._create_file(os.path.join(src_dir, ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "keep2.txt")) + self._create_file(os.path.join(src_dir, "sub", ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "temp.tmp")) + self._create_file(os.path.join(src_dir, "ignore_dir", "file.txt")) + + patterns = DEFAULT_IGNORE_PATTERNS + ["*.tmp", "ignore_dir/"] + + # Archive + with DirectoryArchive(src_dir, "zip", ignore_patterns=patterns) as ar: + archive_path = ar.path + self.assertTrue(os.path.exists(archive_path)) + + # Extract and verify + extract_dir = os.path.join(self.temp_dir, "extract") + os.makedirs(extract_dir) + with zipfile.ZipFile(archive_path, "r") as zip_ref: + zip_ref.extractall(extract_dir) + + # Check files + self.assertTrue(os.path.exists(os.path.join(extract_dir, "keep.txt"))) + self.assertTrue( + os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt")) + ) + + # Checked ignored + self.assertFalse(os.path.exists(os.path.join(extract_dir, ".git"))) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "sub", ".git")) + ) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp")) + ) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "ignore_dir")) + ) + + def test_directory_archive_tar_filtering(self): + # Create dummy directory structure + src_dir = os.path.join(self.temp_dir, "src") + os.makedirs(src_dir) + os.makedirs(os.path.join(src_dir, ".git")) + os.makedirs(os.path.join(src_dir, "sub")) + os.makedirs(os.path.join(src_dir, "sub", ".git")) + os.makedirs(os.path.join(src_dir, "ignore_dir")) + + # Create files + self._create_file(os.path.join(src_dir, "keep.txt")) + self._create_file(os.path.join(src_dir, ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "keep2.txt")) + self._create_file(os.path.join(src_dir, "sub", ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "temp.tmp")) + self._create_file(os.path.join(src_dir, "ignore_dir", "file.txt")) + + patterns = DEFAULT_IGNORE_PATTERNS + ["*.tmp", "ignore_dir/"] + + # Archive + with DirectoryArchive(src_dir, "tar", ignore_patterns=patterns) as ar: + archive_path = ar.path + self.assertTrue(os.path.exists(archive_path)) + + # Extract and verify + extract_dir = os.path.join(self.temp_dir, "extract") + os.makedirs(extract_dir) + with tarfile.open(archive_path, "r") as tar_ref: + tar_ref.extractall(extract_dir) + + # Check files + self.assertTrue(os.path.exists(os.path.join(extract_dir, "keep.txt"))) + self.assertTrue( + os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt")) + ) + + # Checked ignored + self.assertFalse(os.path.exists(os.path.join(extract_dir, ".git"))) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "sub", ".git")) + ) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp")) + ) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "ignore_dir")) + ) + + def _create_file(self, path): + with open(path, "w") as f: + f.write("dummy content") From 3a98ee56ae1e92678d80d2283cf7d6e10156e746 Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Thu, 16 Jul 2026 20:28:49 +0000 Subject: [PATCH 2/5] Reformat Python files using black TAG=agy CONV=2a820637-152b-46a6-9a45-e29918f1e322 --- src/kaggle/api/kaggle_api_extended.py | 28 ++++--------- tests/unit/test_ignore_patterns.py | 60 +++++++-------------------- 2 files changed, 22 insertions(+), 66 deletions(-) diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index 7eb1f1a9..4918c3b5 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -429,9 +429,7 @@ def should_ignore(rel_path: str, is_dir: bool, patterns: List[str]) -> bool: import fnmatch rel_path = rel_path.replace(os.path.sep, "/") - match_path = ( - rel_path + "/" if is_dir and not rel_path.endswith("/") else rel_path - ) + match_path = rel_path + "/" if is_dir and not rel_path.endswith("/") else rel_path for pattern in patterns: pattern = pattern.replace(os.path.sep, "/") @@ -467,9 +465,7 @@ def __enter__(self): if self._ignore_patterns: self.path = self._create_archive_with_filters(archive_base_path) else: - self.path = shutil.make_archive( - archive_base_path, self._format, self._fullpath - ) + self.path = shutil.make_archive(archive_base_path, self._format, self._fullpath) _, self.name = os.path.split(self.path) return self @@ -480,17 +476,13 @@ def __exit__(self, *args): def _create_archive_with_filters(self, base_name): if self._format == "zip": archive_path = base_name + ".zip" - with zipfile.ZipFile( - archive_path, "w", zipfile.ZIP_DEFLATED - ) as zipf: + with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as zipf: for root, dirs, files in os.walk(self._fullpath): dirs[:] = [ d for d in dirs if not should_ignore( - os.path.relpath( - os.path.join(root, d), self._fullpath - ), + os.path.relpath(os.path.join(root, d), self._fullpath), True, self._ignore_patterns, ) @@ -498,9 +490,7 @@ def _create_archive_with_filters(self, base_name): for file in files: file_path = os.path.join(root, file) rel_path = os.path.relpath(file_path, self._fullpath) - if not should_ignore( - rel_path, False, self._ignore_patterns - ): + if not should_ignore(rel_path, False, self._ignore_patterns): zipf.write(file_path, rel_path) return archive_path elif self._format == "tar": @@ -511,9 +501,7 @@ def _create_archive_with_filters(self, base_name): d for d in dirs if not should_ignore( - os.path.relpath( - os.path.join(root, d), self._fullpath - ), + os.path.relpath(os.path.join(root, d), self._fullpath), True, self._ignore_patterns, ) @@ -521,9 +509,7 @@ def _create_archive_with_filters(self, base_name): for file in files: file_path = os.path.join(root, file) rel_path = os.path.relpath(file_path, self._fullpath) - if not should_ignore( - rel_path, False, self._ignore_patterns - ): + if not should_ignore(rel_path, False, self._ignore_patterns): tarf.add(file_path, rel_path) return archive_path else: diff --git a/tests/unit/test_ignore_patterns.py b/tests/unit/test_ignore_patterns.py index 06312726..41f1ed68 100644 --- a/tests/unit/test_ignore_patterns.py +++ b/tests/unit/test_ignore_patterns.py @@ -23,36 +23,24 @@ def test_should_ignore_default_patterns(self): # .git/ in sub dir should be ignored (due to */.git/) self.assertTrue(should_ignore("sub/.git", True, DEFAULT_IGNORE_PATTERNS)) - self.assertTrue( - should_ignore("sub/.git/", True, DEFAULT_IGNORE_PATTERNS) - ) - self.assertTrue( - should_ignore("sub/dir/.git", True, DEFAULT_IGNORE_PATTERNS) - ) + self.assertTrue(should_ignore("sub/.git/", True, DEFAULT_IGNORE_PATTERNS)) + self.assertTrue(should_ignore("sub/dir/.git", True, DEFAULT_IGNORE_PATTERNS)) # .git file (not dir) should NOT be ignored self.assertFalse(should_ignore(".git", False, DEFAULT_IGNORE_PATTERNS)) - self.assertFalse( - should_ignore("sub/.git", False, DEFAULT_IGNORE_PATTERNS) - ) + self.assertFalse(should_ignore("sub/.git", False, DEFAULT_IGNORE_PATTERNS)) # .cache/ at root should be ignored self.assertTrue(should_ignore(".cache", True, DEFAULT_IGNORE_PATTERNS)) # .cache/ in sub dir should NOT be ignored (no */.cache/ in defaults) - self.assertFalse( - should_ignore("sub/.cache", True, DEFAULT_IGNORE_PATTERNS) - ) + self.assertFalse(should_ignore("sub/.cache", True, DEFAULT_IGNORE_PATTERNS)) # .huggingface/ at root should be ignored - self.assertTrue( - should_ignore(".huggingface", True, DEFAULT_IGNORE_PATTERNS) - ) + self.assertTrue(should_ignore(".huggingface", True, DEFAULT_IGNORE_PATTERNS)) # .huggingface/ in sub dir should NOT be ignored - self.assertFalse( - should_ignore("sub/.huggingface", True, DEFAULT_IGNORE_PATTERNS) - ) + self.assertFalse(should_ignore("sub/.huggingface", True, DEFAULT_IGNORE_PATTERNS)) def test_should_ignore_custom_patterns(self): patterns = ["*.tmp", "ignore_dir/", "*/nested_ignore_dir/", "exact_file.txt"] @@ -70,9 +58,7 @@ def test_should_ignore_custom_patterns(self): # Directory pattern (nested) self.assertTrue(should_ignore("sub/nested_ignore_dir", True, patterns)) - self.assertTrue( - should_ignore("sub/dir/nested_ignore_dir", True, patterns) - ) + self.assertTrue(should_ignore("sub/dir/nested_ignore_dir", True, patterns)) self.assertFalse(should_ignore("nested_ignore_dir", True, patterns)) # Exact file pattern @@ -113,21 +99,13 @@ def test_directory_archive_zip_filtering(self): # Check files self.assertTrue(os.path.exists(os.path.join(extract_dir, "keep.txt"))) - self.assertTrue( - os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt")) - ) + self.assertTrue(os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt"))) # Checked ignored self.assertFalse(os.path.exists(os.path.join(extract_dir, ".git"))) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "sub", ".git")) - ) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp")) - ) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "ignore_dir")) - ) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", ".git"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "ignore_dir"))) def test_directory_archive_tar_filtering(self): # Create dummy directory structure @@ -161,21 +139,13 @@ def test_directory_archive_tar_filtering(self): # Check files self.assertTrue(os.path.exists(os.path.join(extract_dir, "keep.txt"))) - self.assertTrue( - os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt")) - ) + self.assertTrue(os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt"))) # Checked ignored self.assertFalse(os.path.exists(os.path.join(extract_dir, ".git"))) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "sub", ".git")) - ) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp")) - ) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "ignore_dir")) - ) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", ".git"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "ignore_dir"))) def _create_file(self, path): with open(path, "w") as f: From 12b112e64cbb337263f8a0cc4d35a9bf9f9bdc83 Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Thu, 16 Jul 2026 20:18:24 +0000 Subject: [PATCH 3/5] Integrate ignore_patterns into Models upload - Update KaggleApi.upload_files to accept ignore_patterns. - Update model_instance_create and model_instance_version_create to accept and pass ignore_patterns. - Add --ignore-patterns CLI argument to model creation commands. - Add unit tests for model creation with ignore_patterns. TAG=agy CONV=2a820637-152b-46a6-9a45-e29918f1e322 --- src/kaggle/api/kaggle_api_extended.py | 116 +++++++++++++++++++++---- src/kaggle/cli.py | 14 +++ tests/unit/test_model_create.py | 120 ++++++++++++++++++++++++++ 3 files changed, 235 insertions(+), 15 deletions(-) diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index 4918c3b5..0cd0b683 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -7095,7 +7095,13 @@ def model_instance_initialize_cli(self, folder): folder = folder or os.getcwd() self.model_instance_initialize(folder) - def model_instance_create(self, folder: str, quiet: bool = False, dir_mode: str = "skip") -> ApiCreateModelResponse: + def model_instance_create( + self, + folder: str, + quiet: bool = False, + dir_mode: str = "skip", + ignore_patterns: Optional[List[str]] = None, + ) -> ApiCreateModelResponse: """Creates a new model instance. Args: @@ -7165,20 +7171,34 @@ def model_instance_create(self, folder: str, quiet: bool = False, dir_mode: str request.body = body message = kaggle.models.model_api_client.create_model_instance with ResumableUploadContext() as upload_context: - self.upload_files(body, None, folder, ApiBlobType.MODEL, upload_context, quiet, dir_mode) + self.upload_files( + body, + None, + folder, + ApiBlobType.MODEL, + upload_context, + quiet, + dir_mode, + ignore_patterns, + ) response = cast(ApiCreateModelResponse, self.with_retry(message)(request)) return response - def model_instance_create_cli(self, folder, quiet=False, dir_mode="skip"): + def model_instance_create_cli( + self, folder, quiet=False, dir_mode="skip", ignore_patterns=None + ): """A client wrapper for creating a new model instance. Args: folder: The folder from which to get the metadata file. quiet: Suppress verbose output (default is False). dir_mode: What to do with directories: "skip" - ignore; "zip" - compress and upload. + ignore_patterns: Patterns of files/dirs to ignore. """ folder = folder or os.getcwd() - result = self.model_instance_create(folder, quiet, dir_mode) + result = self.model_instance_create( + folder, quiet, dir_mode, ignore_patterns + ) if result.id: print("Your model instance was created. Id={}. Url={}".format(result.id, result.url)) @@ -7433,7 +7453,13 @@ def model_instance_update_cli(self, folder=None): print("Model update error: " + result.error) def model_instance_version_create( - self, model_instance: str, folder: str, version_notes: str = "", quiet: bool = False, dir_mode: str = "skip" + self, + model_instance: str, + folder: str, + version_notes: str = "", + quiet: bool = False, + dir_mode: str = "skip", + ignore_patterns: Optional[List[str]] = None, ) -> ApiCreateModelResponse: """Creates a new model instance version. @@ -7460,11 +7486,28 @@ def model_instance_version_create( with self.build_kaggle_client() as kaggle: message = kaggle.models.model_api_client.create_model_instance_version with ResumableUploadContext() as upload_context: - self.upload_files(body, None, folder, ApiBlobType.MODEL, upload_context, quiet, dir_mode) + self.upload_files( + body, + None, + folder, + ApiBlobType.MODEL, + upload_context, + quiet, + dir_mode, + ignore_patterns, + ) response = cast(ApiCreateModelResponse, self.with_retry(message)(request)) return response - def model_instance_version_create_cli(self, model_instance, folder, version_notes="", quiet=False, dir_mode="skip"): + def model_instance_version_create_cli( + self, + model_instance, + folder, + version_notes="", + quiet=False, + dir_mode="skip", + ignore_patterns=None, + ): """A client wrapper for creating a new version of a model instance. Args: @@ -7474,8 +7517,11 @@ def model_instance_version_create_cli(self, model_instance, folder, version_note version_notes: The version notes to record for this new version. quiet: Suppress verbose output (default is False). dir_mode: What to do with directories: "skip" - ignore; "zip" - compress and upload. + ignore_patterns: Patterns of files/dirs to ignore. """ - result = self.model_instance_version_create(model_instance, folder, version_notes, quiet, dir_mode) + result = self.model_instance_version_create( + model_instance, folder, version_notes, quiet, dir_mode, ignore_patterns + ) if result.id != 0: print("Your model instance version was created. Url={}".format(result.url)) @@ -8013,6 +8059,7 @@ def upload_files( upload_context: ResumableUploadContext, quiet: bool = False, dir_mode: str = "skip", + ignore_patterns: Optional[List[str]] = None, ) -> None: """Uploads files in a folder. @@ -8024,10 +8071,12 @@ def upload_files( upload_context (ResumableUploadContext): The context for resumable uploads. quiet (bool): Suppress verbose output (default is False). dir_mode (str): What to do with directories: "skip" - ignore; "zip" - compress and upload. + ignore_patterns (Optional[List[str]]): Patterns of files/dirs to ignore. Returns: None: """ + patterns = DEFAULT_IGNORE_PATTERNS + (ignore_patterns or []) for file_name in os.listdir(folder): if file_name in [ self.DATASET_METADATA_FILE, @@ -8038,8 +8087,21 @@ def upload_files( self.MODEL_INSTANCE_METADATA_FILE, ]: continue + + full_path = os.path.join(folder, file_name) + is_dir = os.path.isdir(full_path) + if should_ignore(file_name, is_dir, patterns): + continue + upload_file = self._upload_file_or_folder( - folder, file_name, blob_type, upload_context, dir_mode, quiet, resources + folder, + file_name, + blob_type, + upload_context, + dir_mode, + quiet, + resources, + patterns, ) if upload_file is not None: files = request.files @@ -8051,7 +8113,10 @@ def _new_file(self, file: UploadFile) -> ApiDatasetNewFile: new_file.token = file.token new_file.description = file.description if file.columns: - new_file.columns = [ApiDatasetColumn.from_dict(file.to_dict()) for file in file.columns] + new_file.columns = [ + ApiDatasetColumn.from_dict(file.to_dict()) + for file in file.columns + ] return new_file def _upload_file_or_folder( @@ -8062,20 +8127,41 @@ def _upload_file_or_folder( upload_context: ResumableUploadContext, dir_mode: str, quiet: bool = False, - resources: Optional[List[Dict[str, Union[str, Dict[str, List[Dict[str, str]]]]]]] = None, + resources: Optional[ + List[Dict[str, Union[str, Dict[str, List[Dict[str, str]]]]]] + ] = None, + ignore_patterns: Optional[List[str]] = None, ) -> Union[UploadFile, None]: full_path = os.path.join(parent_path, file_or_folder_name) upload_file = None if os.path.isfile(full_path): - upload_file = self._upload_file(file_or_folder_name, full_path, blob_type, upload_context, quiet, resources) + upload_file = self._upload_file( + file_or_folder_name, + full_path, + blob_type, + upload_context, + quiet, + resources, + ) elif os.path.isdir(full_path): if dir_mode in ["zip", "tar"]: - with DirectoryArchive(full_path, dir_mode) as archive: + with DirectoryArchive( + full_path, dir_mode, ignore_patterns=ignore_patterns + ) as archive: upload_file = self._upload_file( - archive.name, archive.path, blob_type, upload_context, quiet, resources + archive.name, + archive.path, + blob_type, + upload_context, + quiet, + resources, ) elif not quiet: - print("Skipping folder: " + file_or_folder_name + "; use '--dir-mode' to upload folders") + print( + "Skipping folder: " + + file_or_folder_name + + "; use '--dir-mode' to upload folders" + ) else: if not quiet: print("Skipping: " + file_or_folder_name) diff --git a/src/kaggle/cli.py b/src/kaggle/cli.py index 175fd981..652a5d31 100644 --- a/src/kaggle/cli.py +++ b/src/kaggle/cli.py @@ -1571,6 +1571,13 @@ def parse_model_instances(subparsers) -> None: parser_model_instances_create_optional.add_argument( "-r", "--dir-mode", dest="dir_mode", choices=["skip", "zip", "tar"], default="skip", help=Help.param_dir_mode ) + parser_model_instances_create_optional.add_argument( + "--ignore-patterns", + dest="ignore_patterns", + action="append", + required=False, + help="Patterns to ignore when uploading files/dirs", + ) parser_model_instances_create._action_groups.append(parser_model_instances_create_optional) parser_model_instances_create.set_defaults(func=api.model_instance_create_cli) @@ -1676,6 +1683,13 @@ def parse_model_instance_versions(subparsers) -> None: parser_model_instance_versions_create_optional.add_argument( "-r", "--dir-mode", dest="dir_mode", choices=["skip", "zip", "tar"], default="skip", help=Help.param_dir_mode ) + parser_model_instance_versions_create_optional.add_argument( + "--ignore-patterns", + dest="ignore_patterns", + action="append", + required=False, + help="Patterns to ignore when uploading files/dirs", + ) parser_model_instance_versions_create._action_groups.append(parser_model_instance_versions_create_optional) parser_model_instance_versions_create.set_defaults(func=api.model_instance_version_create_cli) diff --git a/tests/unit/test_model_create.py b/tests/unit/test_model_create.py index 2adcbd89..e647e23e 100644 --- a/tests/unit/test_model_create.py +++ b/tests/unit/test_model_create.py @@ -156,6 +156,38 @@ def test_model_instance_create_valid_metadata_succeeds(self, mock_client, mock_u self.assertEqual(body.base_model_instance, "base-instance") self.assertEqual(body.external_base_model_url, "http://example.com") + @patch.object(KaggleApi, "upload_files") + @patch.object(KaggleApi, "build_kaggle_client") + def test_model_instance_create_with_ignore_patterns_succeeds( + self, mock_client, mock_upload + ): + mock_kaggle = MagicMock() + mock_response = ApiCreateModelResponse() + mock_kaggle.models.model_api_client.create_model_instance.return_value = mock_response + mock_client.return_value.__enter__ = MagicMock(return_value=mock_kaggle) + mock_client.return_value.__exit__ = MagicMock(return_value=False) + + metadata = self._get_valid_instance_metadata() + ignore_patterns = ["*.tmp", "temp/"] + + with tempfile.TemporaryDirectory() as tmpdir: + self._write_metadata(tmpdir, metadata) + response = self.api.model_instance_create( + tmpdir, + quiet=True, + dir_mode="zip", + ignore_patterns=ignore_patterns, + ) + + self.assertEqual(response, mock_response) + mock_upload.assert_called_once() + call_args = mock_upload.call_args[0] + self.assertEqual(call_args[2], tmpdir) + self.assertEqual(call_args[3], ApiBlobType.MODEL) + self.assertTrue(call_args[5]) + self.assertEqual(call_args[6], "zip") + self.assertEqual(call_args[7], ignore_patterns) + def test_model_instance_update_invalid_folder_fails(self): with self.assertRaises(ValueError) as context: self.api.model_instance_update("/non/existent/folder") @@ -488,6 +520,94 @@ def test_model_update_partial_fields_succeeds(self, mock_client): self.assertIsNotNone(request.update_mask) self.assertEqual(list(request.update_mask.paths), ["title"]) +class TestModelInstanceVersionCreate(unittest.TestCase): + """Tests for model_instance_version_create.""" + + def setUp(self): + self.api = KaggleApi.__new__(KaggleApi) + self.api.config_values = {"username": "testuser"} + self.api.already_printed_version_warning = True + + @patch.object(KaggleApi, "upload_files") + @patch.object(KaggleApi, "build_kaggle_client") + def test_model_instance_version_create_succeeds( + self, mock_client, mock_upload + ): + mock_kaggle = MagicMock() + mock_response = ApiCreateModelResponse() + mock_kaggle.models.model_api_client.create_model_instance_version.return_value = ( + mock_response + ) + mock_client.return_value.__enter__ = MagicMock( + return_value=mock_kaggle + ) + mock_client.return_value.__exit__ = MagicMock(return_value=False) + + model_instance = "testuser/test-model/keras/test-instance" + + with tempfile.TemporaryDirectory() as tmpdir: + response = self.api.model_instance_version_create( + model_instance, + tmpdir, + version_notes="test version", + quiet=True, + dir_mode="zip", + ) + + self.assertEqual(response, mock_response) + mock_upload.assert_called_once() + call_args = mock_upload.call_args[0] + self.assertEqual(call_args[2], tmpdir) + self.assertEqual(call_args[3], ApiBlobType.MODEL) + self.assertTrue(call_args[5]) + self.assertEqual(call_args[6], "zip") + self.assertIsNone(mock_upload.call_args[1].get("ignore_patterns")) + + mock_kaggle.models.model_api_client.create_model_instance_version.assert_called_once() + request = mock_kaggle.models.model_api_client.create_model_instance_version.call_args[ + 0 + ][0] + self.assertEqual(request.owner_slug, "testuser") + self.assertEqual(request.model_slug, "test-model") + self.assertEqual(request.instance_slug, "test-instance") + self.assertEqual(request.body.version_notes, "test version") + + @patch.object(KaggleApi, "upload_files") + @patch.object(KaggleApi, "build_kaggle_client") + def test_model_instance_version_create_with_ignore_patterns_succeeds( + self, mock_client, mock_upload + ): + mock_kaggle = MagicMock() + mock_response = ApiCreateModelResponse() + mock_kaggle.models.model_api_client.create_model_instance_version.return_value = ( + mock_response + ) + mock_client.return_value.__enter__ = MagicMock( + return_value=mock_kaggle + ) + mock_client.return_value.__exit__ = MagicMock(return_value=False) + + model_instance = "testuser/test-model/keras/test-instance" + ignore_patterns = ["*.tmp", "temp/"] + + with tempfile.TemporaryDirectory() as tmpdir: + response = self.api.model_instance_version_create( + model_instance, + tmpdir, + version_notes="test version", + quiet=True, + dir_mode="zip", + ignore_patterns=ignore_patterns, + ) + + self.assertEqual(response, mock_response) + mock_upload.assert_called_once() + call_args = mock_upload.call_args[0] + self.assertEqual(call_args[2], tmpdir) + self.assertEqual(call_args[3], ApiBlobType.MODEL) + self.assertTrue(call_args[5]) + self.assertEqual(call_args[6], "zip") + self.assertEqual(call_args[7], ignore_patterns) if __name__ == "__main__": From 0d0aac6db244c21221a27172acbfdf94d2a3f33b Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Thu, 16 Jul 2026 20:18:50 +0000 Subject: [PATCH 4/5] Add CLI parser tests for model ignore_patterns TAG=agy CONV=2a820637-152b-46a6-9a45-e29918f1e322 --- tests/unit/test_cli_models.py | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/unit/test_cli_models.py b/tests/unit/test_cli_models.py index 05171b0a..75177279 100644 --- a/tests/unit/test_cli_models.py +++ b/tests/unit/test_cli_models.py @@ -203,6 +203,23 @@ def test_model_instances_create_parser_with_flags_succeeds(parser): assert kwargs["folder"] == "/path/to/instance" assert kwargs["quiet"] is True assert kwargs["dir_mode"] == "zip" + assert kwargs.get("ignore_patterns") is None + + +def test_model_instances_create_parser_with_ignore_patterns_succeeds(parser): + func, kwargs = parser.dispatch( + [ + "models", + "instances", + "create", + "--ignore-patterns", + "*.tmp", + "--ignore-patterns", + "temp/", + ] + ) + assert func.__name__ == "model_instance_create_cli" + assert kwargs["ignore_patterns"] == ["*.tmp", "temp/"] def test_model_instances_files_parser_missing_instance_fails(parser): @@ -346,6 +363,28 @@ def test_model_instance_versions_create_parser_succeeds(parser): assert kwargs["version_notes"] == "version notes" assert kwargs["quiet"] is True assert kwargs["dir_mode"] == "tar" + assert kwargs.get("ignore_patterns") is None + + +def test_model_instance_versions_create_parser_with_ignore_patterns_succeeds( + parser, +): + func, kwargs = parser.dispatch( + [ + "models", + "instances", + "versions", + "create", + "owner/model/framework/variation", + "--ignore-patterns", + "*.tmp", + "--ignore-patterns", + "temp/", + ] + ) + assert func.__name__ == "model_instance_version_create_cli" + assert kwargs["model_instance"] == "owner/model/framework/variation" + assert kwargs["ignore_patterns"] == ["*.tmp", "temp/"] def test_model_instance_versions_download_parser_missing_version_fails(parser): From 6053df02d97739c9ccd0b5b4dc7aa11253169202 Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Thu, 16 Jul 2026 20:29:10 +0000 Subject: [PATCH 5/5] Reformat Python files using black TAG=agy CONV=2a820637-152b-46a6-9a45-e29918f1e322 --- src/kaggle/api/kaggle_api_extended.py | 27 +++++---------------- tests/unit/test_model_create.py | 34 ++++++++------------------- 2 files changed, 16 insertions(+), 45 deletions(-) diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index 0cd0b683..a484e4bd 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -7184,9 +7184,7 @@ def model_instance_create( response = cast(ApiCreateModelResponse, self.with_retry(message)(request)) return response - def model_instance_create_cli( - self, folder, quiet=False, dir_mode="skip", ignore_patterns=None - ): + def model_instance_create_cli(self, folder, quiet=False, dir_mode="skip", ignore_patterns=None): """A client wrapper for creating a new model instance. Args: @@ -7196,9 +7194,7 @@ def model_instance_create_cli( ignore_patterns: Patterns of files/dirs to ignore. """ folder = folder or os.getcwd() - result = self.model_instance_create( - folder, quiet, dir_mode, ignore_patterns - ) + result = self.model_instance_create(folder, quiet, dir_mode, ignore_patterns) if result.id: print("Your model instance was created. Id={}. Url={}".format(result.id, result.url)) @@ -8113,10 +8109,7 @@ def _new_file(self, file: UploadFile) -> ApiDatasetNewFile: new_file.token = file.token new_file.description = file.description if file.columns: - new_file.columns = [ - ApiDatasetColumn.from_dict(file.to_dict()) - for file in file.columns - ] + new_file.columns = [ApiDatasetColumn.from_dict(file.to_dict()) for file in file.columns] return new_file def _upload_file_or_folder( @@ -8127,9 +8120,7 @@ def _upload_file_or_folder( upload_context: ResumableUploadContext, dir_mode: str, quiet: bool = False, - resources: Optional[ - List[Dict[str, Union[str, Dict[str, List[Dict[str, str]]]]]] - ] = None, + resources: Optional[List[Dict[str, Union[str, Dict[str, List[Dict[str, str]]]]]]] = None, ignore_patterns: Optional[List[str]] = None, ) -> Union[UploadFile, None]: full_path = os.path.join(parent_path, file_or_folder_name) @@ -8145,9 +8136,7 @@ def _upload_file_or_folder( ) elif os.path.isdir(full_path): if dir_mode in ["zip", "tar"]: - with DirectoryArchive( - full_path, dir_mode, ignore_patterns=ignore_patterns - ) as archive: + with DirectoryArchive(full_path, dir_mode, ignore_patterns=ignore_patterns) as archive: upload_file = self._upload_file( archive.name, archive.path, @@ -8157,11 +8146,7 @@ def _upload_file_or_folder( resources, ) elif not quiet: - print( - "Skipping folder: " - + file_or_folder_name - + "; use '--dir-mode' to upload folders" - ) + print("Skipping folder: " + file_or_folder_name + "; use '--dir-mode' to upload folders") else: if not quiet: print("Skipping: " + file_or_folder_name) diff --git a/tests/unit/test_model_create.py b/tests/unit/test_model_create.py index e647e23e..361baeed 100644 --- a/tests/unit/test_model_create.py +++ b/tests/unit/test_model_create.py @@ -158,9 +158,7 @@ def test_model_instance_create_valid_metadata_succeeds(self, mock_client, mock_u @patch.object(KaggleApi, "upload_files") @patch.object(KaggleApi, "build_kaggle_client") - def test_model_instance_create_with_ignore_patterns_succeeds( - self, mock_client, mock_upload - ): + def test_model_instance_create_with_ignore_patterns_succeeds(self, mock_client, mock_upload): mock_kaggle = MagicMock() mock_response = ApiCreateModelResponse() mock_kaggle.models.model_api_client.create_model_instance.return_value = mock_response @@ -520,6 +518,8 @@ def test_model_update_partial_fields_succeeds(self, mock_client): self.assertIsNotNone(request.update_mask) self.assertEqual(list(request.update_mask.paths), ["title"]) + + class TestModelInstanceVersionCreate(unittest.TestCase): """Tests for model_instance_version_create.""" @@ -530,17 +530,11 @@ def setUp(self): @patch.object(KaggleApi, "upload_files") @patch.object(KaggleApi, "build_kaggle_client") - def test_model_instance_version_create_succeeds( - self, mock_client, mock_upload - ): + def test_model_instance_version_create_succeeds(self, mock_client, mock_upload): mock_kaggle = MagicMock() mock_response = ApiCreateModelResponse() - mock_kaggle.models.model_api_client.create_model_instance_version.return_value = ( - mock_response - ) - mock_client.return_value.__enter__ = MagicMock( - return_value=mock_kaggle - ) + mock_kaggle.models.model_api_client.create_model_instance_version.return_value = mock_response + mock_client.return_value.__enter__ = MagicMock(return_value=mock_kaggle) mock_client.return_value.__exit__ = MagicMock(return_value=False) model_instance = "testuser/test-model/keras/test-instance" @@ -564,9 +558,7 @@ def test_model_instance_version_create_succeeds( self.assertIsNone(mock_upload.call_args[1].get("ignore_patterns")) mock_kaggle.models.model_api_client.create_model_instance_version.assert_called_once() - request = mock_kaggle.models.model_api_client.create_model_instance_version.call_args[ - 0 - ][0] + request = mock_kaggle.models.model_api_client.create_model_instance_version.call_args[0][0] self.assertEqual(request.owner_slug, "testuser") self.assertEqual(request.model_slug, "test-model") self.assertEqual(request.instance_slug, "test-instance") @@ -574,17 +566,11 @@ def test_model_instance_version_create_succeeds( @patch.object(KaggleApi, "upload_files") @patch.object(KaggleApi, "build_kaggle_client") - def test_model_instance_version_create_with_ignore_patterns_succeeds( - self, mock_client, mock_upload - ): + def test_model_instance_version_create_with_ignore_patterns_succeeds(self, mock_client, mock_upload): mock_kaggle = MagicMock() mock_response = ApiCreateModelResponse() - mock_kaggle.models.model_api_client.create_model_instance_version.return_value = ( - mock_response - ) - mock_client.return_value.__enter__ = MagicMock( - return_value=mock_kaggle - ) + mock_kaggle.models.model_api_client.create_model_instance_version.return_value = mock_response + mock_client.return_value.__enter__ = MagicMock(return_value=mock_kaggle) mock_client.return_value.__exit__ = MagicMock(return_value=False) model_instance = "testuser/test-model/keras/test-instance"