diff --git a/docs/competition_creation.md b/docs/competition_creation.md index b4f77a91..be78e134 100644 --- a/docs/competition_creation.md +++ b/docs/competition_creation.md @@ -520,7 +520,7 @@ file you want in the new version. ```bash kaggle competitions data update -p -m "" \ - [--rerun] [--include-hidden] + [--rerun] [--include-hidden] [--ignore-patterns ] ``` **Arguments:** @@ -545,6 +545,7 @@ kaggle competitions data update -p -m "" \ sub-directories (names starting with `.` — e.g. `.DS_Store`, `.git/`, `.gitignore`). Skipped by default so you don't accidentally publish OS metadata or version-control detritus. +- `--ignore-patterns ` (optional): Patterns to ignore when uploading files/dirs. Can be specified multiple times. Note that default ignore patterns (like `.git/`, `.cache/`, `.huggingface/`) are bypassed when `--include-hidden` is True. **Examples:** diff --git a/docs/datasets.md b/docs/datasets.md index c753f2c5..cfc0fbb7 100644 --- a/docs/datasets.md +++ b/docs/datasets.md @@ -173,6 +173,8 @@ kaggle datasets create -p [options] * `-q, --quiet`: Suppress verbose output. * `-t, --keep-tabular`: Do not convert tabular files to CSV (default is to convert). * `-r, --dir-mode `: How to handle directories: `skip` (ignore), `zip` (compressed upload), `tar` (uncompressed upload) (default: `skip`). +* `--ignore-patterns `: Patterns of files/dirs to ignore. Can be specified multiple times. + **Example:** @@ -208,6 +210,8 @@ kaggle datasets version -p -m [options] * `-t, --keep-tabular`: Do not convert tabular files to CSV. * `-r, --dir-mode `: Directory handling mode (`skip`, `zip`, `tar`). * `-d, --delete-old-versions`: Delete old versions of this dataset. +* `--ignore-patterns `: Patterns of files/dirs to ignore. Can be specified multiple times. + **Example:** diff --git a/docs/model_variations.md b/docs/model_variations.md index 48e3ca25..06b49212 100644 --- a/docs/model_variations.md +++ b/docs/model_variations.md @@ -44,6 +44,8 @@ kaggle models variations create -p [options] * `-p, --path `: Path to the folder containing the model variation files and the `model-instance-metadata.json` file (defaults to the current directory). * `-q, --quiet`: Suppress verbose output. * `-r, --dir-mode `: How to handle directories within the upload: `skip` (ignore), `zip` (compressed upload), `tar` (uncompressed upload) (default: `skip`). +* `--ignore-patterns `: Patterns of files/dirs to ignore. Can be specified multiple times. + **Example:** diff --git a/docs/model_variations_versions.md b/docs/model_variations_versions.md index 888fbd71..10a495e9 100644 --- a/docs/model_variations_versions.md +++ b/docs/model_variations_versions.md @@ -22,6 +22,8 @@ kaggle models variations versions create -p [opt * `-n, --version-notes `: Notes describing this version. * `-q, --quiet`: Suppress verbose output. * `-r, --dir-mode `: How to handle directories within the upload: `skip` (ignore), `zip` (compressed upload), `tar` (uncompressed upload) (default: `skip`). +* `--ignore-patterns `: Patterns of files/dirs to ignore. Can be specified multiple times. + **Example:** diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index e52aace7..3edbebf9 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -416,30 +416,105 @@ def __str__(self): return self.value +DEFAULT_IGNORE_PATTERNS = [ + ".git/", + "*/.git/", + ".cache/", + ".huggingface/", +] + + +def should_ignore(rel_path: str, is_dir: bool, patterns: List[str]) -> bool: + """Helper to check if a path should be ignored based on patterns.""" + import fnmatch + + rel_path = rel_path.replace(os.path.sep, "/") + match_path = rel_path + "/" if is_dir and not rel_path.endswith("/") else rel_path + + for pattern in patterns: + pattern = pattern.replace(os.path.sep, "/") + if pattern.endswith("/"): + if not is_dir: + continue + if fnmatch.fnmatch(match_path, pattern): + return True + return False + + class DirectoryArchive(object): - """ - Context manager for handling directory archives. + """Context manager for handling directory archives with filtering. - This class provides a context manager for working with directory archives in various formats. - It manages the lifecycle of the archive, including opening and closing resources as needed. + This class provides a context manager for working with directory archives in + various formats. It manages the lifecycle of the archive, including opening + and closing resources as needed. It supports filtering files based on + ignore patterns. """ - def __init__(self, fullpath, fmt): + def __init__(self, fullpath, fmt, ignore_patterns=None): self._fullpath = fullpath self._format = fmt + self._ignore_patterns = ignore_patterns or [] self.name = None self.path = None def __enter__(self): self._temp_dir = tempfile.mkdtemp() _, dir_name = os.path.split(self._fullpath) - self.path = shutil.make_archive(os.path.join(self._temp_dir, dir_name), self._format, self._fullpath) + archive_base_path = os.path.join(self._temp_dir, dir_name) + + if self._ignore_patterns: + self.path = self._create_archive_with_filters(archive_base_path) + else: + self.path = shutil.make_archive(archive_base_path, self._format, self._fullpath) + _, self.name = os.path.split(self.path) return self def __exit__(self, *args): shutil.rmtree(self._temp_dir) + def _create_archive_with_filters(self, base_name): + if self._format == "zip": + archive_path = base_name + ".zip" + with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as zipf: + for root, dirs, files in os.walk(self._fullpath): + dirs[:] = [ + d + for d in dirs + if not should_ignore( + os.path.relpath(os.path.join(root, d), self._fullpath), + True, + self._ignore_patterns, + ) + ] + for file in files: + file_path = os.path.join(root, file) + rel_path = os.path.relpath(file_path, self._fullpath) + if not should_ignore(rel_path, False, self._ignore_patterns): + zipf.write(file_path, rel_path) + return archive_path + elif self._format == "tar": + archive_path = base_name + ".tar" + with tarfile.open(archive_path, "w") as tarf: + for root, dirs, files in os.walk(self._fullpath): + dirs[:] = [ + d + for d in dirs + if not should_ignore( + os.path.relpath(os.path.join(root, d), self._fullpath), + True, + self._ignore_patterns, + ) + ] + for file in files: + file_path = os.path.join(root, file) + rel_path = os.path.relpath(file_path, self._fullpath) + if not should_ignore(rel_path, False, self._ignore_patterns): + tarf.add(file_path, rel_path) + return archive_path + else: + raise ValueError(f"Unsupported archive format: {self._format}") + class ResumableUploadContext(object): """ @@ -3029,6 +3104,7 @@ def competition_data_update( rerun: bool = False, quiet: bool = False, include_hidden: bool = False, + ignore_patterns: Optional[List[str]] = None, ) -> ApiCreateCompetitionDataResponse: """Update (version) the data files for a competition you host. @@ -3054,6 +3130,7 @@ def competition_data_update( quiet (bool): Suppress per-file upload progress lines. include_hidden (bool): If True, upload hidden files and traverse hidden sub-directories. Default False. + ignore_patterns (Optional[List[str]]): Patterns of files/dirs to ignore. Returns: ApiCreateCompetitionDataResponse: url, databundle_id, @@ -3070,11 +3147,31 @@ def competition_data_update( if os.path.isfile(path): uploads.append((os.path.basename(path), path)) else: + patterns = (ignore_patterns or []) if include_hidden else DEFAULT_IGNORE_PATTERNS + (ignore_patterns or []) + for dirpath, dirnames, filenames in os.walk(path): if not include_hidden: # Prune hidden sub-directories in place so os.walk skips them. dirnames[:] = [d for d in dirnames if not d.startswith(".")] filenames = [n for n in filenames if not n.startswith(".")] + + # Prune by ignore_patterns + pruned_dirnames = [] + for d in dirnames: + full_dir_path = os.path.join(dirpath, d) + rel_dir_path = os.path.relpath(full_dir_path, path) + if not should_ignore(rel_dir_path, is_dir=True, patterns=patterns): + pruned_dirnames.append(d) + dirnames[:] = pruned_dirnames + + pruned_filenames = [] + for f in filenames: + full_file_path = os.path.join(dirpath, f) + rel_file_path = os.path.relpath(full_file_path, path) + if not should_ignore(rel_file_path, is_dir=False, patterns=patterns): + pruned_filenames.append(f) + filenames = pruned_filenames + for name in filenames: full = os.path.join(dirpath, name) rel = os.path.relpath(full, path).replace(os.sep, "/") @@ -3091,7 +3188,12 @@ def competition_data_update( with ResumableUploadContext() as upload_context: for rel_name, full_path in uploads: upload_file = self._upload_file( - rel_name, full_path, ApiBlobType.DATASET, upload_context, quiet, resources=None + rel_name, + full_path, + ApiBlobType.DATASET, + upload_context, + quiet, + resources=None, ) if upload_file is not None: f = ApiCompetitionDataFile() @@ -3120,6 +3222,7 @@ def competition_data_update_cli( rerun=False, quiet=False, include_hidden=False, + ignore_patterns=None, ): """CLI wrapper for competition_data_update.""" competition_name = competition or competition_opt @@ -3141,6 +3244,7 @@ def competition_data_update_cli( rerun=rerun, quiet=quiet, include_hidden=include_hidden, + ignore_patterns=ignore_patterns, ) print(f'New data version created for "{competition_name}": {response.url}') @@ -4951,6 +5055,7 @@ def dataset_create_version( convert_to_csv: bool = True, delete_old_versions: bool = False, dir_mode: str = "skip", + ignore_patterns: Optional[List[str]] = None, ) -> ApiCreateDatasetResponse: """Creates a new version of a dataset. @@ -5018,12 +5123,28 @@ def dataset_create_version( message = kaggle.datasets.dataset_api_client.create_dataset_version request.body = body with ResumableUploadContext() as upload_context: - self.upload_files(body, resources, folder, ApiBlobType.DATASET, upload_context, quiet, dir_mode) + self.upload_files( + body, + resources, + folder, + ApiBlobType.DATASET, + upload_context, + quiet, + dir_mode, + ignore_patterns, + ) response = cast(ApiCreateDatasetResponse, self.with_retry(message)(request)) return response def dataset_create_version_cli( - self, folder, version_notes, quiet=False, convert_to_csv=True, delete_old_versions=False, dir_mode="skip" + self, + folder, + version_notes, + quiet=False, + convert_to_csv=True, + delete_old_versions=False, + dir_mode="skip", + ignore_patterns=None, ): """A client wrapper for creating a new version of a dataset. @@ -5034,6 +5155,7 @@ def dataset_create_version_cli( convert_to_csv: If True, convert data to CSV on upload. delete_old_versions: If True, delete old versions of the dataset. dir_mode: What to do with directories: "skip" - ignore; "zip" - compress and upload. + ignore_patterns: Patterns of files/dirs to ignore. """ folder = folder or os.getcwd() result = self.dataset_create_version( @@ -5043,6 +5165,7 @@ def dataset_create_version_cli( convert_to_csv=convert_to_csv, delete_old_versions=delete_old_versions, dir_mode=dir_mode, + ignore_patterns=ignore_patterns, ) if result is None: @@ -5178,6 +5301,7 @@ def dataset_create_new( quiet: bool = False, convert_to_csv: bool = True, dir_mode: str = "skip", + ignore_patterns: Optional[List[str]] = None, ) -> ApiCreateDatasetResponse: """Creates a new dataset. @@ -5256,7 +5380,16 @@ def dataset_create_new( request.category_ids = keywords with ResumableUploadContext() as upload_context: - self.upload_files(request, resources, folder, ApiBlobType.DATASET, upload_context, quiet, dir_mode) + self.upload_files( + request, + resources, + folder, + ApiBlobType.DATASET, + upload_context, + quiet, + dir_mode, + ignore_patterns, + ) with self.build_kaggle_client() as kaggle: retry_request = ApiCreateDatasetRequest() @@ -5275,7 +5408,15 @@ def dataset_create_new( result.error = None return result - def dataset_create_new_cli(self, folder=None, public=False, quiet=False, convert_to_csv=True, dir_mode="skip"): + def dataset_create_new_cli( + self, + folder=None, + public=False, + quiet=False, + convert_to_csv=True, + dir_mode="skip", + ignore_patterns=None, + ): """A client wrapper for creating a new dataset. Args: @@ -5284,9 +5425,10 @@ def dataset_create_new_cli(self, folder=None, public=False, quiet=False, convert quiet: Suppress verbose output (default is False). convert_to_csv: If True, convert data to comma-separated values. dir_mode: What to do with directories: "skip" - ignore; "zip" - compress and upload. + ignore_patterns: Patterns of files/dirs to ignore. """ folder = folder or os.getcwd() - result = self.dataset_create_new(folder, public, quiet, convert_to_csv, dir_mode) + result = self.dataset_create_new(folder, public, quiet, convert_to_csv, dir_mode, ignore_patterns) if result.invalidTags: print( "The following are not valid tags and could not be added to " "the dataset: " + str(result.invalidTags) @@ -7020,7 +7162,13 @@ def model_instance_initialize_cli(self, folder): folder = folder or os.getcwd() self.model_instance_initialize(folder) - def model_instance_create(self, folder: str, quiet: bool = False, dir_mode: str = "skip") -> ApiCreateModelResponse: + def model_instance_create( + self, + folder: str, + quiet: bool = False, + dir_mode: str = "skip", + ignore_patterns: Optional[List[str]] = None, + ) -> ApiCreateModelResponse: """Creates a new model instance. Args: @@ -7090,20 +7238,30 @@ def model_instance_create(self, folder: str, quiet: bool = False, dir_mode: str request.body = body message = kaggle.models.model_api_client.create_model_instance with ResumableUploadContext() as upload_context: - self.upload_files(body, None, folder, ApiBlobType.MODEL, upload_context, quiet, dir_mode) + self.upload_files( + body, + None, + folder, + ApiBlobType.MODEL, + upload_context, + quiet, + dir_mode, + ignore_patterns, + ) response = cast(ApiCreateModelResponse, self.with_retry(message)(request)) return response - def model_instance_create_cli(self, folder, quiet=False, dir_mode="skip"): + def model_instance_create_cli(self, folder, quiet=False, dir_mode="skip", ignore_patterns=None): """A client wrapper for creating a new model instance. Args: folder: The folder from which to get the metadata file. quiet: Suppress verbose output (default is False). dir_mode: What to do with directories: "skip" - ignore; "zip" - compress and upload. + ignore_patterns: Patterns of files/dirs to ignore. """ folder = folder or os.getcwd() - result = self.model_instance_create(folder, quiet, dir_mode) + result = self.model_instance_create(folder, quiet, dir_mode, ignore_patterns) if result.id: print("Your model instance was created. Id={}. Url={}".format(result.id, result.url)) @@ -7358,7 +7516,13 @@ def model_instance_update_cli(self, folder=None): print("Model update error: " + result.error) def model_instance_version_create( - self, model_instance: str, folder: str, version_notes: str = "", quiet: bool = False, dir_mode: str = "skip" + self, + model_instance: str, + folder: str, + version_notes: str = "", + quiet: bool = False, + dir_mode: str = "skip", + ignore_patterns: Optional[List[str]] = None, ) -> ApiCreateModelResponse: """Creates a new model instance version. @@ -7385,11 +7549,28 @@ def model_instance_version_create( with self.build_kaggle_client() as kaggle: message = kaggle.models.model_api_client.create_model_instance_version with ResumableUploadContext() as upload_context: - self.upload_files(body, None, folder, ApiBlobType.MODEL, upload_context, quiet, dir_mode) + self.upload_files( + body, + None, + folder, + ApiBlobType.MODEL, + upload_context, + quiet, + dir_mode, + ignore_patterns, + ) response = cast(ApiCreateModelResponse, self.with_retry(message)(request)) return response - def model_instance_version_create_cli(self, model_instance, folder, version_notes="", quiet=False, dir_mode="skip"): + def model_instance_version_create_cli( + self, + model_instance, + folder, + version_notes="", + quiet=False, + dir_mode="skip", + ignore_patterns=None, + ): """A client wrapper for creating a new version of a model instance. Args: @@ -7399,8 +7580,11 @@ def model_instance_version_create_cli(self, model_instance, folder, version_note version_notes: The version notes to record for this new version. quiet: Suppress verbose output (default is False). dir_mode: What to do with directories: "skip" - ignore; "zip" - compress and upload. + ignore_patterns: Patterns of files/dirs to ignore. """ - result = self.model_instance_version_create(model_instance, folder, version_notes, quiet, dir_mode) + result = self.model_instance_version_create( + model_instance, folder, version_notes, quiet, dir_mode, ignore_patterns + ) if result.id != 0: print("Your model instance version was created. Url={}".format(result.url)) @@ -7938,6 +8122,7 @@ def upload_files( upload_context: ResumableUploadContext, quiet: bool = False, dir_mode: str = "skip", + ignore_patterns: Optional[List[str]] = None, ) -> None: """Uploads files in a folder. @@ -7949,10 +8134,12 @@ def upload_files( upload_context (ResumableUploadContext): The context for resumable uploads. quiet (bool): Suppress verbose output (default is False). dir_mode (str): What to do with directories: "skip" - ignore; "zip" - compress and upload. + ignore_patterns (Optional[List[str]]): Patterns of files/dirs to ignore. Returns: None: """ + patterns = DEFAULT_IGNORE_PATTERNS + (ignore_patterns or []) for file_name in os.listdir(folder): if file_name in [ self.DATASET_METADATA_FILE, @@ -7963,8 +8150,21 @@ def upload_files( self.MODEL_INSTANCE_METADATA_FILE, ]: continue + + full_path = os.path.join(folder, file_name) + is_dir = os.path.isdir(full_path) + if should_ignore(file_name, is_dir, patterns): + continue + upload_file = self._upload_file_or_folder( - folder, file_name, blob_type, upload_context, dir_mode, quiet, resources + folder, + file_name, + blob_type, + upload_context, + dir_mode, + quiet, + resources, + patterns, ) if upload_file is not None: files = request.files @@ -7988,16 +8188,29 @@ def _upload_file_or_folder( dir_mode: str, quiet: bool = False, resources: Optional[List[Dict[str, Union[str, Dict[str, List[Dict[str, str]]]]]]] = None, + ignore_patterns: Optional[List[str]] = None, ) -> Union[UploadFile, None]: full_path = os.path.join(parent_path, file_or_folder_name) upload_file = None if os.path.isfile(full_path): - upload_file = self._upload_file(file_or_folder_name, full_path, blob_type, upload_context, quiet, resources) + upload_file = self._upload_file( + file_or_folder_name, + full_path, + blob_type, + upload_context, + quiet, + resources, + ) elif os.path.isdir(full_path): if dir_mode in ["zip", "tar"]: - with DirectoryArchive(full_path, dir_mode) as archive: + with DirectoryArchive(full_path, dir_mode, ignore_patterns=ignore_patterns) as archive: upload_file = self._upload_file( - archive.name, archive.path, blob_type, upload_context, quiet, resources + archive.name, + archive.path, + blob_type, + upload_context, + quiet, + resources, ) elif not quiet: print("Skipping folder: " + file_or_folder_name + "; use '--dir-mode' to upload folders") diff --git a/src/kaggle/cli.py b/src/kaggle/cli.py index 175fd981..6daf1f7f 100644 --- a/src/kaggle/cli.py +++ b/src/kaggle/cli.py @@ -653,6 +653,13 @@ def parse_competitions(subparsers) -> None: parser_competitions_data_update_optional.add_argument( "-q", "--quiet", dest="quiet", action="store_true", help=Help.param_quiet ) + parser_competitions_data_update_optional.add_argument( + "--ignore-patterns", + dest="ignore_patterns", + action="append", + required=False, + help="Patterns to ignore when uploading files/dirs", + ) parser_competitions_data_update._action_groups.append(parser_competitions_data_update_optional) parser_competitions_data_update.set_defaults(func=api.competition_data_update_cli) @@ -996,6 +1003,13 @@ def parse_datasets(subparsers) -> None: parser_datasets_create_optional.add_argument( "-r", "--dir-mode", dest="dir_mode", choices=["skip", "zip", "tar"], default="skip", help=Help.param_dir_mode ) + parser_datasets_create_optional.add_argument( + "--ignore-patterns", + dest="ignore_patterns", + action="append", + required=False, + help="Patterns to ignore when uploading files/dirs", + ) parser_datasets_create._action_groups.append(parser_datasets_create_optional) parser_datasets_create.set_defaults(func=api.dataset_create_new_cli) @@ -1027,6 +1041,13 @@ def parse_datasets(subparsers) -> None: action="store_true", help=Help.param_delete_old_version, ) + parser_datasets_version_optional.add_argument( + "--ignore-patterns", + dest="ignore_patterns", + action="append", + required=False, + help="Patterns to ignore when uploading files/dirs", + ) parser_datasets_version._action_groups.append(parser_datasets_version_optional) parser_datasets_version.set_defaults(func=api.dataset_create_version_cli) @@ -1571,6 +1592,13 @@ def parse_model_instances(subparsers) -> None: parser_model_instances_create_optional.add_argument( "-r", "--dir-mode", dest="dir_mode", choices=["skip", "zip", "tar"], default="skip", help=Help.param_dir_mode ) + parser_model_instances_create_optional.add_argument( + "--ignore-patterns", + dest="ignore_patterns", + action="append", + required=False, + help="Patterns to ignore when uploading files/dirs", + ) parser_model_instances_create._action_groups.append(parser_model_instances_create_optional) parser_model_instances_create.set_defaults(func=api.model_instance_create_cli) @@ -1676,6 +1704,13 @@ def parse_model_instance_versions(subparsers) -> None: parser_model_instance_versions_create_optional.add_argument( "-r", "--dir-mode", dest="dir_mode", choices=["skip", "zip", "tar"], default="skip", help=Help.param_dir_mode ) + parser_model_instance_versions_create_optional.add_argument( + "--ignore-patterns", + dest="ignore_patterns", + action="append", + required=False, + help="Patterns to ignore when uploading files/dirs", + ) parser_model_instance_versions_create._action_groups.append(parser_model_instance_versions_create_optional) parser_model_instance_versions_create.set_defaults(func=api.model_instance_version_create_cli) diff --git a/tests/unit/test_cli_competitions.py b/tests/unit/test_cli_competitions.py index 2810c385..39051ba3 100644 --- a/tests/unit/test_cli_competitions.py +++ b/tests/unit/test_cli_competitions.py @@ -376,9 +376,35 @@ def test_competitions_data_update_succeeds(parser): assert kwargs["version_notes"] == "update msg" assert kwargs["rerun"] is True assert kwargs["include_hidden"] is True + assert kwargs.get("ignore_patterns") is None + + +def test_competitions_data_update_with_ignore_patterns_succeeds(parser): + func, kwargs = parser.dispatch( + [ + "competitions", + "data", + "update", + "my-comp", + "-p", + "/path/to/data", + "-m", + "update msg", + "--ignore-patterns", + "*.tmp", + "--ignore-patterns", + "temp/", + ] + ) + assert func.__name__ == "competition_data_update_cli" + assert kwargs["competition"] == "my-comp" + assert kwargs["path"] == "/path/to/data" + assert kwargs["version_notes"] == "update msg" + assert kwargs["ignore_patterns"] == ["*.tmp", "temp/"] def test_competitions_settings_get_succeeds(parser): + func, kwargs = parser.dispatch(["competitions", "settings", "get", "my-comp", "--json"]) assert func.__name__ == "competition_get_settings_cli" assert kwargs["competition"] == "my-comp" diff --git a/tests/unit/test_cli_datasets.py b/tests/unit/test_cli_datasets.py index f94ecd4b..527e4a3e 100644 --- a/tests/unit/test_cli_datasets.py +++ b/tests/unit/test_cli_datasets.py @@ -189,6 +189,22 @@ def test_datasets_create_parser_with_flags_succeeds(parser): assert kwargs["quiet"] is True assert kwargs["convert_to_csv"] is False assert kwargs["dir_mode"] == "zip" + assert kwargs.get("ignore_patterns") is None + + +def test_datasets_create_parser_with_ignore_patterns_succeeds(parser): + func, kwargs = parser.dispatch( + [ + "datasets", + "create", + "--ignore-patterns", + "*.tmp", + "--ignore-patterns", + "temp/", + ] + ) + assert func.__name__ == "dataset_create_new_cli" + assert kwargs["ignore_patterns"] == ["*.tmp", "temp/"] def test_datasets_version_parser_missing_message_fails(parser): @@ -219,6 +235,25 @@ def test_datasets_version_parser_succeeds(parser): assert kwargs["convert_to_csv"] is False assert kwargs["dir_mode"] == "tar" assert kwargs["delete_old_versions"] is True + assert kwargs.get("ignore_patterns") is None + + +def test_datasets_version_parser_with_ignore_patterns_succeeds(parser): + func, kwargs = parser.dispatch( + [ + "datasets", + "version", + "-m", + "version message", + "--ignore-patterns", + "*.tmp", + "--ignore-patterns", + "temp/", + ] + ) + assert func.__name__ == "dataset_create_version_cli" + assert kwargs["version_notes"] == "version message" + assert kwargs["ignore_patterns"] == ["*.tmp", "temp/"] def test_datasets_init_parser_default_succeeds(parser): diff --git a/tests/unit/test_cli_models.py b/tests/unit/test_cli_models.py index 05171b0a..75177279 100644 --- a/tests/unit/test_cli_models.py +++ b/tests/unit/test_cli_models.py @@ -203,6 +203,23 @@ def test_model_instances_create_parser_with_flags_succeeds(parser): assert kwargs["folder"] == "/path/to/instance" assert kwargs["quiet"] is True assert kwargs["dir_mode"] == "zip" + assert kwargs.get("ignore_patterns") is None + + +def test_model_instances_create_parser_with_ignore_patterns_succeeds(parser): + func, kwargs = parser.dispatch( + [ + "models", + "instances", + "create", + "--ignore-patterns", + "*.tmp", + "--ignore-patterns", + "temp/", + ] + ) + assert func.__name__ == "model_instance_create_cli" + assert kwargs["ignore_patterns"] == ["*.tmp", "temp/"] def test_model_instances_files_parser_missing_instance_fails(parser): @@ -346,6 +363,28 @@ def test_model_instance_versions_create_parser_succeeds(parser): assert kwargs["version_notes"] == "version notes" assert kwargs["quiet"] is True assert kwargs["dir_mode"] == "tar" + assert kwargs.get("ignore_patterns") is None + + +def test_model_instance_versions_create_parser_with_ignore_patterns_succeeds( + parser, +): + func, kwargs = parser.dispatch( + [ + "models", + "instances", + "versions", + "create", + "owner/model/framework/variation", + "--ignore-patterns", + "*.tmp", + "--ignore-patterns", + "temp/", + ] + ) + assert func.__name__ == "model_instance_version_create_cli" + assert kwargs["model_instance"] == "owner/model/framework/variation" + assert kwargs["ignore_patterns"] == ["*.tmp", "temp/"] def test_model_instance_versions_download_parser_missing_version_fails(parser): diff --git a/tests/unit/test_competition_data_update.py b/tests/unit/test_competition_data_update.py index d5528a5b..913e3cb1 100644 --- a/tests/unit/test_competition_data_update.py +++ b/tests/unit/test_competition_data_update.py @@ -92,6 +92,30 @@ def test_update_directory_recurses_and_preserves_paths(self, mock_client, mock_u for call in mock_upload.call_args_list: self.assertEqual(call.args[2], ApiBlobType.DATASET) + @patch.object(KaggleApi, "_upload_file") + @patch.object(KaggleApi, "build_kaggle_client") + def test_update_directory_respects_ignore_patterns(self, mock_client, mock_upload): + ignore_patterns = ["test.csv", "images/cats/"] + + mock_upload.side_effect = [_mock_upload_file(f"tok-{i}") for i in range(10)] + mock_kaggle = self._patch_client(mock_client) + + self.api.competition_data_update( + competition_name="my-comp", + path=self.tmp, + version_notes="version with ignores", + ignore_patterns=ignore_patterns, + ) + + request = mock_kaggle.competitions.competition_api_client.create_competition_data.call_args[0][0] + names = sorted(f.name for f in request.files) + self.assertEqual( + names, + ["images/dog.png", "train.csv"], + ) + called_names = sorted(call.args[0] for call in mock_upload.call_args_list) + self.assertEqual(called_names, names) + @patch.object(KaggleApi, "_upload_file") @patch.object(KaggleApi, "build_kaggle_client") def test_update_single_file_uploads_as_is(self, mock_client, mock_upload): diff --git a/tests/unit/test_dataset_create.py b/tests/unit/test_dataset_create.py index 759131a1..73281ebb 100644 --- a/tests/unit/test_dataset_create.py +++ b/tests/unit/test_dataset_create.py @@ -10,6 +10,7 @@ sys.path.insert(0, "../..") from kaggle.api.kaggle_api_extended import KaggleApi +from kagglesdk.blobs.types.blob_api_service import ApiBlobType class TestDatasetCreate(unittest.TestCase): @@ -174,6 +175,32 @@ def test_dataset_create_new_valid_metadata_succeeds(self, mock_client, mock_uplo self.assertFalse(request.is_private) self.assertEqual(response, mock_response) + @patch.object(KaggleApi, "dataset_status") + @patch.object(KaggleApi, "upload_files") + @patch.object(KaggleApi, "build_kaggle_client") + def test_dataset_create_new_with_ignore_patterns_succeeds(self, mock_client, mock_upload, mock_status): + mock_status.side_effect = HTTPError() + mock_kaggle = MagicMock() + mock_response = MagicMock() + mock_response.error = "" + mock_kaggle.datasets.dataset_api_client.create_dataset.return_value = mock_response + mock_client.return_value.__enter__ = MagicMock(return_value=mock_kaggle) + mock_client.return_value.__exit__ = MagicMock(return_value=False) + + metadata = self._get_valid_metadata() + ignore_patterns = ["*.tmp", "temp/"] + + with tempfile.TemporaryDirectory() as tmpdir: + self._write_metadata(tmpdir, metadata) + response = self.api.dataset_create_new(tmpdir, ignore_patterns=ignore_patterns) + + self.assertEqual(response, mock_response) + mock_upload.assert_called_once() + call_args = mock_upload.call_args[0] + self.assertEqual(call_args[2], tmpdir) + self.assertEqual(call_args[3], ApiBlobType.DATASET) + self.assertEqual(call_args[7], ignore_patterns) + @patch.object(KaggleApi, "dataset_status") @patch.object(KaggleApi, "upload_files") @patch.object(KaggleApi, "build_kaggle_client") @@ -324,6 +351,33 @@ def test_dataset_create_version_with_id_succeeds(self, mock_client, mock_upload) self.assertEqual(response, mock_response) + @patch.object(KaggleApi, "upload_files") + @patch.object(KaggleApi, "build_kaggle_client") + def test_dataset_create_version_with_ignore_patterns_succeeds(self, mock_client, mock_upload): + mock_kaggle = MagicMock() + mock_response = MagicMock() + mock_kaggle.datasets.dataset_api_client.create_dataset_version.return_value = mock_response + mock_client.return_value.__enter__ = MagicMock(return_value=mock_kaggle) + mock_client.return_value.__exit__ = MagicMock(return_value=False) + + metadata = self._get_valid_metadata() + ignore_patterns = ["*.tmp", "temp/"] + + with tempfile.TemporaryDirectory() as tmpdir: + self._write_metadata(tmpdir, metadata) + response = self.api.dataset_create_version( + tmpdir, + "version notes here", + ignore_patterns=ignore_patterns, + ) + + self.assertEqual(response, mock_response) + mock_upload.assert_called_once() + call_args = mock_upload.call_args[0] + self.assertEqual(call_args[2], tmpdir) + self.assertEqual(call_args[3], ApiBlobType.DATASET) + self.assertEqual(call_args[7], ignore_patterns) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_ignore_patterns.py b/tests/unit/test_ignore_patterns.py new file mode 100644 index 00000000..41f1ed68 --- /dev/null +++ b/tests/unit/test_ignore_patterns.py @@ -0,0 +1,152 @@ +import os +import shutil +import tempfile +import unittest +import zipfile +import tarfile + +from kaggle.api.kaggle_api_extended import should_ignore, DirectoryArchive, DEFAULT_IGNORE_PATTERNS + + +class TestIgnorePatterns(unittest.TestCase): + + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_should_ignore_default_patterns(self): + # .git/ at root should be ignored + self.assertTrue(should_ignore(".git", True, DEFAULT_IGNORE_PATTERNS)) + self.assertTrue(should_ignore(".git/", True, DEFAULT_IGNORE_PATTERNS)) + + # .git/ in sub dir should be ignored (due to */.git/) + self.assertTrue(should_ignore("sub/.git", True, DEFAULT_IGNORE_PATTERNS)) + self.assertTrue(should_ignore("sub/.git/", True, DEFAULT_IGNORE_PATTERNS)) + self.assertTrue(should_ignore("sub/dir/.git", True, DEFAULT_IGNORE_PATTERNS)) + + # .git file (not dir) should NOT be ignored + self.assertFalse(should_ignore(".git", False, DEFAULT_IGNORE_PATTERNS)) + self.assertFalse(should_ignore("sub/.git", False, DEFAULT_IGNORE_PATTERNS)) + + # .cache/ at root should be ignored + self.assertTrue(should_ignore(".cache", True, DEFAULT_IGNORE_PATTERNS)) + + # .cache/ in sub dir should NOT be ignored (no */.cache/ in defaults) + self.assertFalse(should_ignore("sub/.cache", True, DEFAULT_IGNORE_PATTERNS)) + + # .huggingface/ at root should be ignored + self.assertTrue(should_ignore(".huggingface", True, DEFAULT_IGNORE_PATTERNS)) + + # .huggingface/ in sub dir should NOT be ignored + self.assertFalse(should_ignore("sub/.huggingface", True, DEFAULT_IGNORE_PATTERNS)) + + def test_should_ignore_custom_patterns(self): + patterns = ["*.tmp", "ignore_dir/", "*/nested_ignore_dir/", "exact_file.txt"] + + # Wildcard file pattern + self.assertTrue(should_ignore("foo.tmp", False, patterns)) + self.assertTrue(should_ignore("sub/foo.tmp", False, patterns)) + self.assertFalse(should_ignore("foo.tmp.bak", False, patterns)) + + # Directory pattern (root only) + self.assertTrue(should_ignore("ignore_dir", True, patterns)) + self.assertFalse(should_ignore("sub/ignore_dir", True, patterns)) + # Directory pattern (root only) should not match file + self.assertFalse(should_ignore("ignore_dir", False, patterns)) + + # Directory pattern (nested) + self.assertTrue(should_ignore("sub/nested_ignore_dir", True, patterns)) + self.assertTrue(should_ignore("sub/dir/nested_ignore_dir", True, patterns)) + self.assertFalse(should_ignore("nested_ignore_dir", True, patterns)) + + # Exact file pattern + self.assertTrue(should_ignore("exact_file.txt", False, patterns)) + # fnmatch "exact_file.txt" will not match "sub/exact_file.txt" if we don't have wildcard + # Wait, fnmatch("sub/exact_file.txt", "exact_file.txt") is False. + self.assertFalse(should_ignore("sub/exact_file.txt", False, patterns)) + + def test_directory_archive_zip_filtering(self): + # Create dummy directory structure + src_dir = os.path.join(self.temp_dir, "src") + os.makedirs(src_dir) + os.makedirs(os.path.join(src_dir, ".git")) + os.makedirs(os.path.join(src_dir, "sub")) + os.makedirs(os.path.join(src_dir, "sub", ".git")) + os.makedirs(os.path.join(src_dir, "ignore_dir")) + + # Create files + self._create_file(os.path.join(src_dir, "keep.txt")) + self._create_file(os.path.join(src_dir, ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "keep2.txt")) + self._create_file(os.path.join(src_dir, "sub", ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "temp.tmp")) + self._create_file(os.path.join(src_dir, "ignore_dir", "file.txt")) + + patterns = DEFAULT_IGNORE_PATTERNS + ["*.tmp", "ignore_dir/"] + + # Archive + with DirectoryArchive(src_dir, "zip", ignore_patterns=patterns) as ar: + archive_path = ar.path + self.assertTrue(os.path.exists(archive_path)) + + # Extract and verify + extract_dir = os.path.join(self.temp_dir, "extract") + os.makedirs(extract_dir) + with zipfile.ZipFile(archive_path, "r") as zip_ref: + zip_ref.extractall(extract_dir) + + # Check files + self.assertTrue(os.path.exists(os.path.join(extract_dir, "keep.txt"))) + self.assertTrue(os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt"))) + + # Checked ignored + self.assertFalse(os.path.exists(os.path.join(extract_dir, ".git"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", ".git"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "ignore_dir"))) + + def test_directory_archive_tar_filtering(self): + # Create dummy directory structure + src_dir = os.path.join(self.temp_dir, "src") + os.makedirs(src_dir) + os.makedirs(os.path.join(src_dir, ".git")) + os.makedirs(os.path.join(src_dir, "sub")) + os.makedirs(os.path.join(src_dir, "sub", ".git")) + os.makedirs(os.path.join(src_dir, "ignore_dir")) + + # Create files + self._create_file(os.path.join(src_dir, "keep.txt")) + self._create_file(os.path.join(src_dir, ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "keep2.txt")) + self._create_file(os.path.join(src_dir, "sub", ".git", "config")) + self._create_file(os.path.join(src_dir, "sub", "temp.tmp")) + self._create_file(os.path.join(src_dir, "ignore_dir", "file.txt")) + + patterns = DEFAULT_IGNORE_PATTERNS + ["*.tmp", "ignore_dir/"] + + # Archive + with DirectoryArchive(src_dir, "tar", ignore_patterns=patterns) as ar: + archive_path = ar.path + self.assertTrue(os.path.exists(archive_path)) + + # Extract and verify + extract_dir = os.path.join(self.temp_dir, "extract") + os.makedirs(extract_dir) + with tarfile.open(archive_path, "r") as tar_ref: + tar_ref.extractall(extract_dir) + + # Check files + self.assertTrue(os.path.exists(os.path.join(extract_dir, "keep.txt"))) + self.assertTrue(os.path.exists(os.path.join(extract_dir, "sub", "keep2.txt"))) + + # Checked ignored + self.assertFalse(os.path.exists(os.path.join(extract_dir, ".git"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", ".git"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "sub", "temp.tmp"))) + self.assertFalse(os.path.exists(os.path.join(extract_dir, "ignore_dir"))) + + def _create_file(self, path): + with open(path, "w") as f: + f.write("dummy content") diff --git a/tests/unit/test_model_create.py b/tests/unit/test_model_create.py index 2adcbd89..361baeed 100644 --- a/tests/unit/test_model_create.py +++ b/tests/unit/test_model_create.py @@ -156,6 +156,36 @@ def test_model_instance_create_valid_metadata_succeeds(self, mock_client, mock_u self.assertEqual(body.base_model_instance, "base-instance") self.assertEqual(body.external_base_model_url, "http://example.com") + @patch.object(KaggleApi, "upload_files") + @patch.object(KaggleApi, "build_kaggle_client") + def test_model_instance_create_with_ignore_patterns_succeeds(self, mock_client, mock_upload): + mock_kaggle = MagicMock() + mock_response = ApiCreateModelResponse() + mock_kaggle.models.model_api_client.create_model_instance.return_value = mock_response + mock_client.return_value.__enter__ = MagicMock(return_value=mock_kaggle) + mock_client.return_value.__exit__ = MagicMock(return_value=False) + + metadata = self._get_valid_instance_metadata() + ignore_patterns = ["*.tmp", "temp/"] + + with tempfile.TemporaryDirectory() as tmpdir: + self._write_metadata(tmpdir, metadata) + response = self.api.model_instance_create( + tmpdir, + quiet=True, + dir_mode="zip", + ignore_patterns=ignore_patterns, + ) + + self.assertEqual(response, mock_response) + mock_upload.assert_called_once() + call_args = mock_upload.call_args[0] + self.assertEqual(call_args[2], tmpdir) + self.assertEqual(call_args[3], ApiBlobType.MODEL) + self.assertTrue(call_args[5]) + self.assertEqual(call_args[6], "zip") + self.assertEqual(call_args[7], ignore_patterns) + def test_model_instance_update_invalid_folder_fails(self): with self.assertRaises(ValueError) as context: self.api.model_instance_update("/non/existent/folder") @@ -490,5 +520,81 @@ def test_model_update_partial_fields_succeeds(self, mock_client): self.assertEqual(list(request.update_mask.paths), ["title"]) +class TestModelInstanceVersionCreate(unittest.TestCase): + """Tests for model_instance_version_create.""" + + def setUp(self): + self.api = KaggleApi.__new__(KaggleApi) + self.api.config_values = {"username": "testuser"} + self.api.already_printed_version_warning = True + + @patch.object(KaggleApi, "upload_files") + @patch.object(KaggleApi, "build_kaggle_client") + def test_model_instance_version_create_succeeds(self, mock_client, mock_upload): + mock_kaggle = MagicMock() + mock_response = ApiCreateModelResponse() + mock_kaggle.models.model_api_client.create_model_instance_version.return_value = mock_response + mock_client.return_value.__enter__ = MagicMock(return_value=mock_kaggle) + mock_client.return_value.__exit__ = MagicMock(return_value=False) + + model_instance = "testuser/test-model/keras/test-instance" + + with tempfile.TemporaryDirectory() as tmpdir: + response = self.api.model_instance_version_create( + model_instance, + tmpdir, + version_notes="test version", + quiet=True, + dir_mode="zip", + ) + + self.assertEqual(response, mock_response) + mock_upload.assert_called_once() + call_args = mock_upload.call_args[0] + self.assertEqual(call_args[2], tmpdir) + self.assertEqual(call_args[3], ApiBlobType.MODEL) + self.assertTrue(call_args[5]) + self.assertEqual(call_args[6], "zip") + self.assertIsNone(mock_upload.call_args[1].get("ignore_patterns")) + + mock_kaggle.models.model_api_client.create_model_instance_version.assert_called_once() + request = mock_kaggle.models.model_api_client.create_model_instance_version.call_args[0][0] + self.assertEqual(request.owner_slug, "testuser") + self.assertEqual(request.model_slug, "test-model") + self.assertEqual(request.instance_slug, "test-instance") + self.assertEqual(request.body.version_notes, "test version") + + @patch.object(KaggleApi, "upload_files") + @patch.object(KaggleApi, "build_kaggle_client") + def test_model_instance_version_create_with_ignore_patterns_succeeds(self, mock_client, mock_upload): + mock_kaggle = MagicMock() + mock_response = ApiCreateModelResponse() + mock_kaggle.models.model_api_client.create_model_instance_version.return_value = mock_response + mock_client.return_value.__enter__ = MagicMock(return_value=mock_kaggle) + mock_client.return_value.__exit__ = MagicMock(return_value=False) + + model_instance = "testuser/test-model/keras/test-instance" + ignore_patterns = ["*.tmp", "temp/"] + + with tempfile.TemporaryDirectory() as tmpdir: + response = self.api.model_instance_version_create( + model_instance, + tmpdir, + version_notes="test version", + quiet=True, + dir_mode="zip", + ignore_patterns=ignore_patterns, + ) + + self.assertEqual(response, mock_response) + mock_upload.assert_called_once() + call_args = mock_upload.call_args[0] + self.assertEqual(call_args[2], tmpdir) + self.assertEqual(call_args[3], ApiBlobType.MODEL) + self.assertTrue(call_args[5]) + self.assertEqual(call_args[6], "zip") + self.assertEqual(call_args[7], ignore_patterns) + + if __name__ == "__main__": unittest.main()