From 715c74bbea7ed1e5e44c6381f2170d9dc63bf106 Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Thu, 16 Jul 2026 20:14:43 +0000 Subject: [PATCH 1/5] Implement ignore_patterns core logic and unit tests - Define DEFAULT_IGNORE_PATTERNS. - Implement should_ignore helper. - Update DirectoryArchive to support filtering for zip/tar formats. - Add unit tests for should_ignore and DirectoryArchive. TAG=agy CONV=2a820637-152b-46a6-9a45-e29918f1e322 --- src/kaggle/api/kaggle_api_extended.py | 101 +++++++++++++- tests/unit/test_ignore_patterns.py | 182 ++++++++++++++++++++++++++ 2 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_ignore_patterns.py diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index e52aace7..7eb1f1a9 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -416,30 +416,119 @@ def __str__(self): return self.value +DEFAULT_IGNORE_PATTERNS = [ + ".git/", + "*/.git/", + ".cache/", + ".huggingface/", +] + + +def should_ignore(rel_path: str, is_dir: bool, patterns: List[str]) -> bool: + """Helper to check if a path should be ignored based on patterns.""" + import fnmatch + + rel_path = rel_path.replace(os.path.sep, "/") + match_path = ( + rel_path + "/" if is_dir and not rel_path.endswith("/") else rel_path + ) + + for pattern in patterns: + pattern = pattern.replace(os.path.sep, "/") + if pattern.endswith("/"): + if not is_dir: + continue + if fnmatch.fnmatch(match_path, pattern): + return True + return False + + class DirectoryArchive(object): - """ - Context manager for handling directory archives. + """Context manager for handling directory archives with filtering. - This class provides a context manager for working with directory archives in various formats. - It manages the lifecycle of the archive, including opening and closing resources as needed. + This class provides a context manager for working with directory archives in + various formats. It manages the lifecycle of the archive, including opening + and closing resources as needed. It supports filtering files based on + ignore patterns. """ - def __init__(self, fullpath, fmt): + def __init__(self, fullpath, fmt, ignore_patterns=None): self._fullpath = fullpath self._format = fmt + self._ignore_patterns = ignore_patterns or [] self.name = None self.path = None def __enter__(self): self._temp_dir = tempfile.mkdtemp() _, dir_name = os.path.split(self._fullpath) - self.path = shutil.make_archive(os.path.join(self._temp_dir, dir_name), self._format, self._fullpath) + archive_base_path = os.path.join(self._temp_dir, dir_name) + + if self._ignore_patterns: + self.path = self._create_archive_with_filters(archive_base_path) + else: + self.path = shutil.make_archive( + archive_base_path, self._format, self._fullpath + ) + _, self.name = os.path.split(self.path) return self def __exit__(self, *args): shutil.rmtree(self._temp_dir) + def _create_archive_with_filters(self, base_name): + if self._format == "zip": + archive_path = base_name + ".zip" + with zipfile.ZipFile( + archive_path, "w", zipfile.ZIP_DEFLATED + ) as zipf: + for root, dirs, files in os.walk(self._fullpath): + dirs[:] = [ + d + for d in dirs + if not should_ignore( + os.path.relpath( + os.path.join(root, d), self._fullpath + ), + True, + self._ignore_patterns, + ) + ] + for file in files: + file_path = os.path.join(root, file) + rel_path = os.path.relpath(file_path, self._fullpath) + if not should_ignore( + rel_path, False, self._ignore_patterns + ): + zipf.write(file_path, rel_path) + return archive_path + elif self._format == "tar": + archive_path = base_name + ".tar" + with tarfile.open(archive_path, "w") as tarf: + for root, dirs, files in os.walk(self._fullpath): + dirs[:] = [ + d + for d in dirs + if not should_ignore( + os.path.relpath( + os.path.join(root, d), self._fullpath + ), + True, + self._ignore_patterns, + ) + ] + for file in files: + file_path = os.path.join(root, file) + rel_path = os.path.relpath(file_path, self._fullpath) + if not should_ignore( + rel_path, False, self._ignore_patterns + ): + tarf.add(file_path, rel_path) + return archive_path + else: + raise ValueError(f"Unsupported archive format: {self._format}") + class ResumableUploadContext(object): """ diff --git a/tests/unit/test_ignore_patterns.py b/tests/unit/test_ignore_patterns.py new file mode 100644 index 00000000..06312726 --- /dev/null +++ b/tests/unit/test_ignore_patterns.py @@ -0,0 +1,182 @@ +import os +import shutil +import tempfile +import unittest +import zipfile +import tarfile + +from kaggle.api.kaggle_api_extended import should_ignore, DirectoryArchive, DEFAULT_IGNORE_PATTERNS + + +class TestIgnorePatterns(unittest.TestCase): + + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_should_ignore_default_patterns(self): + # .git/ at root should be ignored + self.assertTrue(should_ignore(".git", True, DEFAULT_IGNORE_PATTERNS)) + self.assertTrue(should_ignore(".git/", True, DEFAULT_IGNORE_PATTERNS)) + + # .git/ in sub dir should be ignored (due to */.git/) + self.assertTrue(should_ignore("sub/.git", True, DEFAULT_IGNORE_PATTERNS)) + self.assertTrue( + should_ignore("sub/.git/", True, DEFAULT_IGNORE_PATTERNS) + ) + self.assertTrue( + should_ignore("sub/dir/.git", True, DEFAULT_IGNORE_PATTERNS) + ) + + # .git file (not dir) should NOT be ignored + self.assertFalse(should_ignore(".git", False, DEFAULT_IGNORE_PATTERNS)) + self.assertFalse( + should_ignore("sub/.git", False, DEFAULT_IGNORE_PATTERNS) + ) + + # .cache/ at root should be ignored + self.assertTrue(should_ignore(".cache", True, DEFAULT_IGNORE_PATTERNS)) + + # .cache/ in sub dir should NOT be ignored (no */.cache/ in defaults) + self.assertFalse( + should_ignore("sub/.cache", True, DEFAULT_IGNORE_PATTERNS) + ) + + # .huggingface/ at root should be ignored + self.assertTrue( + should_ignore(".huggingface", True, DEFAULT_IGNORE_PATTERNS) + ) + + # .huggingface/ in sub dir should NOT be ignored + self.assertFalse( + should_ignore("sub/.huggingface", True, DEFAULT_IGNORE_PATTERNS) + ) + + def test_should_ignore_custom_patterns(self): + patterns = ["*.tmp", "ignore_dir/", "*/nested_ignore_dir/", "exact_file.txt"] + + # Wildcard file pattern + self.assertTrue(should_ignore("foo.tmp", False, patterns)) + self.assertTrue(should_ignore("sub/foo.tmp", False, patterns)) + self.assertFalse(should_ignore("foo.tmp.bak", False, patterns)) + + # Directory pattern (root only) + self.assertTrue(should_ignore("ignore_dir", True, patterns)) + self.assertFalse(should_ignore("sub/ignore_dir", True, patterns)) + # Directory pattern (root only) should not match file + self.assertFalse(should_ignore("ignore_dir", False, patterns)) + + # Directory pattern (nested) + self.assertTrue(should_ignore("sub/nested_ignore_dir", True, patterns)) + self.assertTrue( + should_ignore("sub/dir/nested_ignore_dir", True, patterns) + ) + self.assertFalse(should_ignore("nested_ignore_dir", True, patterns)) + + # Exact file pattern + self.assertTrue(should_ignore("exact_file.txt", False, patterns)) + # fnmatch "exact_file.txt" will not match "sub/exact_file.txt" if we don't have wildcard + # Wait, fnmatch("sub/exact_file.txt", "exact_file.txt") is False. + self.assertFalse(should_ignore("sub/exact_file.txt", False, patterns)) + + def test_directory_archive_zip_filtering(self): + # Create dummy directory structure + src_dir = os.path.join(self.temp_dir, "src") + os.makedirs(src_dir) + os.makedirs(os.path.join(src_dir, ".git")) + os.makedirs(os.path.join(src_dir, "sub")) + os.makedirs(os.path.join(src_dir, "sub", ".git")) + os.makedirs(os.path.join(src_dir, "ignore_dir")) + + # Create files + self._create_file(os.path.join(src_dir, "keep.txt")) + self._create_file(os.path.join(src_dir, ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "keep2.txt")) + self._create_file(os.path.join(src_dir, "sub", ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "temp.tmp")) + self._create_file(os.path.join(src_dir, "ignore_dir", "file.txt")) + + patterns = DEFAULT_IGNORE_PATTERNS + ["*.tmp", "ignore_dir/"] + + # Archive + with DirectoryArchive(src_dir, "zip", ignore_patterns=patterns) as ar: + archive_path = ar.path + self.assertTrue(os.path.exists(archive_path)) + + # Extract and verify + extract_dir = os.path.join(self.temp_dir, "extract") + os.makedirs(extract_dir) + with zipfile.ZipFile(archive_path, "r") as zip_ref: + zip_ref.extractall(extract_dir) + + # Check files + self.assertTrue(os.path.exists(os.path.join(extract_dir, "keep.txt"))) + self.assertTrue( + os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt")) + ) + + # Checked ignored + self.assertFalse(os.path.exists(os.path.join(extract_dir, ".git"))) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "sub", ".git")) + ) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp")) + ) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "ignore_dir")) + ) + + def test_directory_archive_tar_filtering(self): + # Create dummy directory structure + src_dir = os.path.join(self.temp_dir, "src") + os.makedirs(src_dir) + os.makedirs(os.path.join(src_dir, ".git")) + os.makedirs(os.path.join(src_dir, "sub")) + os.makedirs(os.path.join(src_dir, "sub", ".git")) + os.makedirs(os.path.join(src_dir, "ignore_dir")) + + # Create files + self._create_file(os.path.join(src_dir, "keep.txt")) + self._create_file(os.path.join(src_dir, ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "keep2.txt")) + self._create_file(os.path.join(src_dir, "sub", ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "temp.tmp")) + self._create_file(os.path.join(src_dir, "ignore_dir", "file.txt")) + + patterns = DEFAULT_IGNORE_PATTERNS + ["*.tmp", "ignore_dir/"] + + # Archive + with DirectoryArchive(src_dir, "tar", ignore_patterns=patterns) as ar: + archive_path = ar.path + self.assertTrue(os.path.exists(archive_path)) + + # Extract and verify + extract_dir = os.path.join(self.temp_dir, "extract") + os.makedirs(extract_dir) + with tarfile.open(archive_path, "r") as tar_ref: + tar_ref.extractall(extract_dir) + + # Check files + self.assertTrue(os.path.exists(os.path.join(extract_dir, "keep.txt"))) + self.assertTrue( + os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt")) + ) + + # Checked ignored + self.assertFalse(os.path.exists(os.path.join(extract_dir, ".git"))) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "sub", ".git")) + ) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp")) + ) + self.assertFalse( + os.path.exists(os.path.join(extract_dir, "ignore_dir")) + ) + + def _create_file(self, path): + with open(path, "w") as f: + f.write("dummy content") From 3a98ee56ae1e92678d80d2283cf7d6e10156e746 Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Thu, 16 Jul 2026 20:28:49 +0000 Subject: [PATCH 2/5] Reformat Python files using black TAG=agy CONV=2a820637-152b-46a6-9a45-e29918f1e322 --- src/kaggle/api/kaggle_api_extended.py | 28 ++++--------- tests/unit/test_ignore_patterns.py | 60 +++++++-------------------- 2 files changed, 22 insertions(+), 66 deletions(-) diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index 7eb1f1a9..4918c3b5 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -429,9 +429,7 @@ def should_ignore(rel_path: str, is_dir: bool, patterns: List[str]) -> bool: import fnmatch rel_path = rel_path.replace(os.path.sep, "/") - match_path = ( - rel_path + "/" if is_dir and not rel_path.endswith("/") else rel_path - ) + match_path = rel_path + "/" if is_dir and not rel_path.endswith("/") else rel_path for pattern in patterns: pattern = pattern.replace(os.path.sep, "/") @@ -467,9 +465,7 @@ def __enter__(self): if self._ignore_patterns: self.path = self._create_archive_with_filters(archive_base_path) else: - self.path = shutil.make_archive( - archive_base_path, self._format, self._fullpath - ) + self.path = shutil.make_archive(archive_base_path, self._format, self._fullpath) _, self.name = os.path.split(self.path) return self @@ -480,17 +476,13 @@ def __exit__(self, *args): def _create_archive_with_filters(self, base_name): if self._format == "zip": archive_path = base_name + ".zip" - with zipfile.ZipFile( - archive_path, "w", zipfile.ZIP_DEFLATED - ) as zipf: + with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as zipf: for root, dirs, files in os.walk(self._fullpath): dirs[:] = [ d for d in dirs if not should_ignore( - os.path.relpath( - os.path.join(root, d), self._fullpath - ), + os.path.relpath(os.path.join(root, d), self._fullpath), True, self._ignore_patterns, ) @@ -498,9 +490,7 @@ def _create_archive_with_filters(self, base_name): for file in files: file_path = os.path.join(root, file) rel_path = os.path.relpath(file_path, self._fullpath) - if not should_ignore( - rel_path, False, self._ignore_patterns - ): + if not should_ignore(rel_path, False, self._ignore_patterns): zipf.write(file_path, rel_path) return archive_path elif self._format == "tar": @@ -511,9 +501,7 @@ def _create_archive_with_filters(self, base_name): d for d in dirs if not should_ignore( - os.path.relpath( - os.path.join(root, d), self._fullpath - ), + os.path.relpath(os.path.join(root, d), self._fullpath), True, self._ignore_patterns, ) @@ -521,9 +509,7 @@ def _create_archive_with_filters(self, base_name): for file in files: file_path = os.path.join(root, file) rel_path = os.path.relpath(file_path, self._fullpath) - if not should_ignore( - rel_path, False, self._ignore_patterns - ): + if not should_ignore(rel_path, False, self._ignore_patterns): tarf.add(file_path, rel_path) return archive_path else: diff --git a/tests/unit/test_ignore_patterns.py b/tests/unit/test_ignore_patterns.py index 06312726..41f1ed68 100644 --- a/tests/unit/test_ignore_patterns.py +++ b/tests/unit/test_ignore_patterns.py @@ -23,36 +23,24 @@ def test_should_ignore_default_patterns(self): # .git/ in sub dir should be ignored (due to */.git/) self.assertTrue(should_ignore("sub/.git", True, DEFAULT_IGNORE_PATTERNS)) - self.assertTrue( - should_ignore("sub/.git/", True, DEFAULT_IGNORE_PATTERNS) - ) - self.assertTrue( - should_ignore("sub/dir/.git", True, DEFAULT_IGNORE_PATTERNS) - ) + self.assertTrue(should_ignore("sub/.git/", True, DEFAULT_IGNORE_PATTERNS)) + self.assertTrue(should_ignore("sub/dir/.git", True, DEFAULT_IGNORE_PATTERNS)) # .git file (not dir) should NOT be ignored self.assertFalse(should_ignore(".git", False, DEFAULT_IGNORE_PATTERNS)) - self.assertFalse( - should_ignore("sub/.git", False, DEFAULT_IGNORE_PATTERNS) - ) + self.assertFalse(should_ignore("sub/.git", False, DEFAULT_IGNORE_PATTERNS)) # .cache/ at root should be ignored self.assertTrue(should_ignore(".cache", True, DEFAULT_IGNORE_PATTERNS)) # .cache/ in sub dir should NOT be ignored (no */.cache/ in defaults) - self.assertFalse( - should_ignore("sub/.cache", True, DEFAULT_IGNORE_PATTERNS) - ) + self.assertFalse(should_ignore("sub/.cache", True, DEFAULT_IGNORE_PATTERNS)) # .huggingface/ at root should be ignored - self.assertTrue( - should_ignore(".huggingface", True, DEFAULT_IGNORE_PATTERNS) - ) + self.assertTrue(should_ignore(".huggingface", True, DEFAULT_IGNORE_PATTERNS)) # .huggingface/ in sub dir should NOT be ignored - self.assertFalse( - should_ignore("sub/.huggingface", True, DEFAULT_IGNORE_PATTERNS) - ) + self.assertFalse(should_ignore("sub/.huggingface", True, DEFAULT_IGNORE_PATTERNS)) def test_should_ignore_custom_patterns(self): patterns = ["*.tmp", "ignore_dir/", "*/nested_ignore_dir/", "exact_file.txt"] @@ -70,9 +58,7 @@ def test_should_ignore_custom_patterns(self): # Directory pattern (nested) self.assertTrue(should_ignore("sub/nested_ignore_dir", True, patterns)) - self.assertTrue( - should_ignore("sub/dir/nested_ignore_dir", True, patterns) - ) + self.assertTrue(should_ignore("sub/dir/nested_ignore_dir", True, patterns)) self.assertFalse(should_ignore("nested_ignore_dir", True, patterns)) # Exact file pattern @@ -113,21 +99,13 @@ def test_directory_archive_zip_filtering(self): # Check files self.assertTrue(os.path.exists(os.path.join(extract_dir, "keep.txt"))) - self.assertTrue( - os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt")) - ) + self.assertTrue(os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt"))) # Checked ignored self.assertFalse(os.path.exists(os.path.join(extract_dir, ".git"))) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "sub", ".git")) - ) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp")) - ) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "ignore_dir")) - ) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", ".git"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "ignore_dir"))) def test_directory_archive_tar_filtering(self): # Create dummy directory structure @@ -161,21 +139,13 @@ def test_directory_archive_tar_filtering(self): # Check files self.assertTrue(os.path.exists(os.path.join(extract_dir, "keep.txt"))) - self.assertTrue( - os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt")) - ) + self.assertTrue(os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt"))) # Checked ignored self.assertFalse(os.path.exists(os.path.join(extract_dir, ".git"))) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "sub", ".git")) - ) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp")) - ) - self.assertFalse( - os.path.exists(os.path.join(extract_dir, "ignore_dir")) - ) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", ".git"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "ignore_dir"))) def _create_file(self, path): with open(path, "w") as f: From 81266cbf3e45f5b0bec66ca69a81392847193601 Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Thu, 16 Jul 2026 20:21:46 +0000 Subject: [PATCH 3/5] Implement competition ignore_patterns integration (API & CLI) - Update competition_data_update to accept and apply ignore_patterns during directory walk. - Add --ignore-patterns CLI argument to competitions data update command. TAG=agy CONV=2a820637-152b-46a6-9a45-e29918f1e322 --- src/kaggle/api/kaggle_api_extended.py | 50 ++++++++++++++++++++++++--- src/kaggle/cli.py | 7 ++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index 4918c3b5..dab4476a 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -3104,6 +3104,7 @@ def competition_data_update( rerun: bool = False, quiet: bool = False, include_hidden: bool = False, + ignore_patterns: Optional[List[str]] = None, ) -> ApiCreateCompetitionDataResponse: """Update (version) the data files for a competition you host. @@ -3129,6 +3130,7 @@ def competition_data_update( quiet (bool): Suppress per-file upload progress lines. include_hidden (bool): If True, upload hidden files and traverse hidden sub-directories. Default False. + ignore_patterns (Optional[List[str]]): Patterns of files/dirs to ignore. Returns: ApiCreateCompetitionDataResponse: url, databundle_id, @@ -3145,11 +3147,34 @@ def competition_data_update( if os.path.isfile(path): uploads.append((os.path.basename(path), path)) else: + patterns = DEFAULT_IGNORE_PATTERNS + (ignore_patterns or []) for dirpath, dirnames, filenames in os.walk(path): if not include_hidden: # Prune hidden sub-directories in place so os.walk skips them. dirnames[:] = [d for d in dirnames if not d.startswith(".")] filenames = [n for n in filenames if not n.startswith(".")] + + # Prune by ignore_patterns + pruned_dirnames = [] + for d in dirnames: + full_dir_path = os.path.join(dirpath, d) + rel_dir_path = os.path.relpath(full_dir_path, path) + if not should_ignore( + rel_dir_path, is_dir=True, patterns=patterns + ): + pruned_dirnames.append(d) + dirnames[:] = pruned_dirnames + + pruned_filenames = [] + for f in filenames: + full_file_path = os.path.join(dirpath, f) + rel_file_path = os.path.relpath(full_file_path, path) + if not should_ignore( + rel_file_path, is_dir=False, patterns=patterns + ): + pruned_filenames.append(f) + filenames = pruned_filenames + for name in filenames: full = os.path.join(dirpath, name) rel = os.path.relpath(full, path).replace(os.sep, "/") @@ -3166,7 +3191,12 @@ def competition_data_update( with ResumableUploadContext() as upload_context: for rel_name, full_path in uploads: upload_file = self._upload_file( - rel_name, full_path, ApiBlobType.DATASET, upload_context, quiet, resources=None + rel_name, + full_path, + ApiBlobType.DATASET, + upload_context, + quiet, + resources=None, ) if upload_file is not None: f = ApiCompetitionDataFile() @@ -3183,8 +3213,12 @@ def competition_data_update( request.version_notes = version_notes request.files = files if rerun: - request.competition_databundle_type = CompetitionDatabundleType.COMPETITION_DATABUNDLE_TYPE_RERUN - return kaggle.competitions.competition_api_client.create_competition_data(request) + request.competition_databundle_type = ( + CompetitionDatabundleType.COMPETITION_DATABUNDLE_TYPE_RERUN + ) + return kaggle.competitions.competition_api_client.create_competition_data( + request + ) def competition_data_update_cli( self, @@ -3195,11 +3229,14 @@ def competition_data_update_cli( rerun=False, quiet=False, include_hidden=False, + ignore_patterns=None, ): """CLI wrapper for competition_data_update.""" competition_name = competition or competition_opt if competition_name is None: - competition_name = self.get_config_value(self.CONFIG_NAME_COMPETITION) + competition_name = self.get_config_value( + self.CONFIG_NAME_COMPETITION + ) if competition_name is not None and not quiet: print("Using competition: " + competition_name) if competition_name is None: @@ -3216,8 +3253,11 @@ def competition_data_update_cli( rerun=rerun, quiet=quiet, include_hidden=include_hidden, + ignore_patterns=ignore_patterns, + ) + print( + f'New data version created for "{competition_name}": {response.url}' ) - print(f'New data version created for "{competition_name}": {response.url}') def competition_launch(self, competition_name: str, future_time: Optional[datetime] = None) -> None: """Launch a competition you host, optionally at a future UTC time. diff --git a/src/kaggle/cli.py b/src/kaggle/cli.py index 175fd981..a93342ba 100644 --- a/src/kaggle/cli.py +++ b/src/kaggle/cli.py @@ -653,6 +653,13 @@ def parse_competitions(subparsers) -> None: parser_competitions_data_update_optional.add_argument( "-q", "--quiet", dest="quiet", action="store_true", help=Help.param_quiet ) + parser_competitions_data_update_optional.add_argument( + "--ignore-patterns", + dest="ignore_patterns", + action="append", + required=False, + help="Patterns to ignore when uploading files/dirs", + ) parser_competitions_data_update._action_groups.append(parser_competitions_data_update_optional) parser_competitions_data_update.set_defaults(func=api.competition_data_update_cli) From f7f983049186c51f6872515a9fa370450fcfb1c8 Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Thu, 16 Jul 2026 20:22:27 +0000 Subject: [PATCH 4/5] Add unit and CLI parser tests for competition ignore_patterns TAG=agy CONV=2a820637-152b-46a6-9a45-e29918f1e322 --- src/kaggle/api/kaggle_api_extended.py | 7 ++++- tests/unit/test_cli_competitions.py | 26 +++++++++++++++++ tests/unit/test_competition_data_update.py | 33 ++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index dab4476a..1e647756 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -3147,7 +3147,12 @@ def competition_data_update( if os.path.isfile(path): uploads.append((os.path.basename(path), path)) else: - patterns = DEFAULT_IGNORE_PATTERNS + (ignore_patterns or []) + patterns = ( + ignore_patterns or [] + ) if include_hidden else DEFAULT_IGNORE_PATTERNS + ( + ignore_patterns or [] + ) + for dirpath, dirnames, filenames in os.walk(path): if not include_hidden: # Prune hidden sub-directories in place so os.walk skips them. diff --git a/tests/unit/test_cli_competitions.py b/tests/unit/test_cli_competitions.py index 2810c385..39051ba3 100644 --- a/tests/unit/test_cli_competitions.py +++ b/tests/unit/test_cli_competitions.py @@ -376,9 +376,35 @@ def test_competitions_data_update_succeeds(parser): assert kwargs["version_notes"] == "update msg" assert kwargs["rerun"] is True assert kwargs["include_hidden"] is True + assert kwargs.get("ignore_patterns") is None + + +def test_competitions_data_update_with_ignore_patterns_succeeds(parser): + func, kwargs = parser.dispatch( + [ + "competitions", + "data", + "update", + "my-comp", + "-p", + "/path/to/data", + "-m", + "update msg", + "--ignore-patterns", + "*.tmp", + "--ignore-patterns", + "temp/", + ] + ) + assert func.__name__ == "competition_data_update_cli" + assert kwargs["competition"] == "my-comp" + assert kwargs["path"] == "/path/to/data" + assert kwargs["version_notes"] == "update msg" + assert kwargs["ignore_patterns"] == ["*.tmp", "temp/"] def test_competitions_settings_get_succeeds(parser): + func, kwargs = parser.dispatch(["competitions", "settings", "get", "my-comp", "--json"]) assert func.__name__ == "competition_get_settings_cli" assert kwargs["competition"] == "my-comp" diff --git a/tests/unit/test_competition_data_update.py b/tests/unit/test_competition_data_update.py index d5528a5b..1b50fc65 100644 --- a/tests/unit/test_competition_data_update.py +++ b/tests/unit/test_competition_data_update.py @@ -93,6 +93,39 @@ def test_update_directory_recurses_and_preserves_paths(self, mock_client, mock_u self.assertEqual(call.args[2], ApiBlobType.DATASET) @patch.object(KaggleApi, "_upload_file") + @patch.object(KaggleApi, "build_kaggle_client") + def test_update_directory_respects_ignore_patterns( + self, mock_client, mock_upload + ): + ignore_patterns = ["test.csv", "images/cats/"] + + mock_upload.side_effect = [ + _mock_upload_file(f"tok-{i}") for i in range(10) + ] + mock_kaggle = self._patch_client(mock_client) + + self.api.competition_data_update( + competition_name="my-comp", + path=self.tmp, + version_notes="version with ignores", + ignore_patterns=ignore_patterns, + ) + + request = mock_kaggle.competitions.competition_api_client.create_competition_data.call_args[ + 0 + ][0] + names = sorted(f.name for f in request.files) + self.assertEqual( + names, + ["images/dog.png", "train.csv"], + ) + called_names = sorted( + call.args[0] for call in mock_upload.call_args_list + ) + self.assertEqual(called_names, names) + + @patch.object(KaggleApi, "_upload_file") + @patch.object(KaggleApi, "build_kaggle_client") def test_update_single_file_uploads_as_is(self, mock_client, mock_upload): archive = os.path.join(self.tmp, "bundle.zip") From 3113934255c090b82bf70e6cd54f698f478d5660 Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Thu, 16 Jul 2026 20:29:52 +0000 Subject: [PATCH 5/5] Reformat Python files using black TAG=agy CONV=2a820637-152b-46a6-9a45-e29918f1e322 --- src/kaggle/api/kaggle_api_extended.py | 30 +++++----------------- tests/unit/test_competition_data_update.py | 17 +++--------- 2 files changed, 11 insertions(+), 36 deletions(-) diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index 1e647756..3249bf88 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -3147,11 +3147,7 @@ def competition_data_update( if os.path.isfile(path): uploads.append((os.path.basename(path), path)) else: - patterns = ( - ignore_patterns or [] - ) if include_hidden else DEFAULT_IGNORE_PATTERNS + ( - ignore_patterns or [] - ) + patterns = (ignore_patterns or []) if include_hidden else DEFAULT_IGNORE_PATTERNS + (ignore_patterns or []) for dirpath, dirnames, filenames in os.walk(path): if not include_hidden: @@ -3164,9 +3160,7 @@ def competition_data_update( for d in dirnames: full_dir_path = os.path.join(dirpath, d) rel_dir_path = os.path.relpath(full_dir_path, path) - if not should_ignore( - rel_dir_path, is_dir=True, patterns=patterns - ): + if not should_ignore(rel_dir_path, is_dir=True, patterns=patterns): pruned_dirnames.append(d) dirnames[:] = pruned_dirnames @@ -3174,9 +3168,7 @@ def competition_data_update( for f in filenames: full_file_path = os.path.join(dirpath, f) rel_file_path = os.path.relpath(full_file_path, path) - if not should_ignore( - rel_file_path, is_dir=False, patterns=patterns - ): + if not should_ignore(rel_file_path, is_dir=False, patterns=patterns): pruned_filenames.append(f) filenames = pruned_filenames @@ -3218,12 +3210,8 @@ def competition_data_update( request.version_notes = version_notes request.files = files if rerun: - request.competition_databundle_type = ( - CompetitionDatabundleType.COMPETITION_DATABUNDLE_TYPE_RERUN - ) - return kaggle.competitions.competition_api_client.create_competition_data( - request - ) + request.competition_databundle_type = CompetitionDatabundleType.COMPETITION_DATABUNDLE_TYPE_RERUN + return kaggle.competitions.competition_api_client.create_competition_data(request) def competition_data_update_cli( self, @@ -3239,9 +3227,7 @@ def competition_data_update_cli( """CLI wrapper for competition_data_update.""" competition_name = competition or competition_opt if competition_name is None: - competition_name = self.get_config_value( - self.CONFIG_NAME_COMPETITION - ) + competition_name = self.get_config_value(self.CONFIG_NAME_COMPETITION) if competition_name is not None and not quiet: print("Using competition: " + competition_name) if competition_name is None: @@ -3260,9 +3246,7 @@ def competition_data_update_cli( include_hidden=include_hidden, ignore_patterns=ignore_patterns, ) - print( - f'New data version created for "{competition_name}": {response.url}' - ) + print(f'New data version created for "{competition_name}": {response.url}') def competition_launch(self, competition_name: str, future_time: Optional[datetime] = None) -> None: """Launch a competition you host, optionally at a future UTC time. diff --git a/tests/unit/test_competition_data_update.py b/tests/unit/test_competition_data_update.py index 1b50fc65..913e3cb1 100644 --- a/tests/unit/test_competition_data_update.py +++ b/tests/unit/test_competition_data_update.py @@ -94,14 +94,10 @@ def test_update_directory_recurses_and_preserves_paths(self, mock_client, mock_u @patch.object(KaggleApi, "_upload_file") @patch.object(KaggleApi, "build_kaggle_client") - def test_update_directory_respects_ignore_patterns( - self, mock_client, mock_upload - ): + def test_update_directory_respects_ignore_patterns(self, mock_client, mock_upload): ignore_patterns = ["test.csv", "images/cats/"] - mock_upload.side_effect = [ - _mock_upload_file(f"tok-{i}") for i in range(10) - ] + mock_upload.side_effect = [_mock_upload_file(f"tok-{i}") for i in range(10)] mock_kaggle = self._patch_client(mock_client) self.api.competition_data_update( @@ -111,21 +107,16 @@ def test_update_directory_respects_ignore_patterns( ignore_patterns=ignore_patterns, ) - request = mock_kaggle.competitions.competition_api_client.create_competition_data.call_args[ - 0 - ][0] + request = mock_kaggle.competitions.competition_api_client.create_competition_data.call_args[0][0] names = sorted(f.name for f in request.files) self.assertEqual( names, ["images/dog.png", "train.csv"], ) - called_names = sorted( - call.args[0] for call in mock_upload.call_args_list - ) + called_names = sorted(call.args[0] for call in mock_upload.call_args_list) self.assertEqual(called_names, names) @patch.object(KaggleApi, "_upload_file") - @patch.object(KaggleApi, "build_kaggle_client") def test_update_single_file_uploads_as_is(self, mock_client, mock_upload): archive = os.path.join(self.tmp, "bundle.zip")