diff --git a/setup.py b/setup.py index ff8420800..ff352109c 100644 --- a/setup.py +++ b/setup.py @@ -126,7 +126,7 @@ ] extra_deps['hf'] = [ - 'huggingface_hub>=0.23.4,<1.4', + 'huggingface_hub>=1.2.1,<1.4', ] extra_deps['testing'] = [ diff --git a/streaming/base/storage/download.py b/streaming/base/storage/download.py index 6e437613a..13a95ffa5 100644 --- a/streaming/base/storage/download.py +++ b/streaming/base/storage/download.py @@ -502,17 +502,9 @@ def clean_up(self) -> None: def _download_file_impl(self, remote: str, local: str, timeout: float) -> None: """Implementation of the download function for a file.""" - from huggingface_hub import hf_hub_download + from huggingface_hub import hffs - _, _, _, repo_org, repo_name, path = remote.split('/', 5) - local_dirname = os.path.dirname(local) - hf_hub_download(repo_id=f'{repo_org}/{repo_name}', - filename=path, - repo_type='dataset', - local_dir=local_dirname) - - downloaded_name = os.path.join(local_dirname, path) - os.rename(downloaded_name, local) + hffs.download(remote, local) class AzureDownloader(CloudDownloader):