From dec1ac73462e7a2cd99fed3229632212f5f4060e Mon Sep 17 00:00:00 2001 From: skshetry <18718008+skshetry@users.noreply.github.com> Date: Wed, 13 Dec 2023 16:55:45 +0545 Subject: [PATCH] deps: update dvc-data to >=2.24 (#10158) --- dvc/data_cloud.py | 9 ++++++--- dvc/fs/__init__.py | 6 ++++-- dvc/output.py | 10 +++++----- dvc/repo/__init__.py | 4 ++-- dvc/repo/artifacts.py | 5 +++-- dvc/repo/imports.py | 4 ++-- dvc/repo/worktree.py | 8 +++----- dvc/stage/cache.py | 4 ++-- pyproject.toml | 2 +- 9 files changed, 28 insertions(+), 24 deletions(-) diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index be0df0ecc1..8888999dd3 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Iterable, Optional, Set, Tuple from dvc.config import NoRemoteError, RemoteConfigError -from dvc.fs.callbacks import Callback from dvc.log import logger from dvc.utils.objects import cached_property from dvc_data.hashfile.db import get_index @@ -206,11 +205,13 @@ def _push( jobs: Optional[int] = None, odb: "HashFileDB", ) -> "TransferResult": + from dvc.fs.callbacks import TqdmCallback + if odb.hash_name == "md5-dos2unix": cache = self.repo.cache.legacy else: cache = self.repo.cache.local - with Callback.as_tqdm_callback( + with TqdmCallback( desc=f"Pushing to {odb.fs.unstrip_protocol(odb.path)}", unit="file", ) as cb: @@ -265,11 +266,13 @@ def _pull( jobs: Optional[int] = None, odb: "HashFileDB", ) -> "TransferResult": + from dvc.fs.callbacks import TqdmCallback + if odb.hash_name == "md5-dos2unix": cache = self.repo.cache.legacy else: cache = self.repo.cache.local - with Callback.as_tqdm_callback( + with TqdmCallback( desc=f"Fetching from {odb.fs.unstrip_protocol(odb.path)}", unit="file", ) as cb: diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py index 1e8627efc5..3404df2178 100644 --- a/dvc/fs/__init__.py +++ b/dvc/fs/__init__.py @@ -27,7 +27,7 @@ ) from dvc_objects.fs.path import Path # noqa: F401 -from .callbacks import Callback +from .callbacks import Callback # noqa: F401 from .data import DataFileSystem # noqa: F401 from .dvc import DVCFileSystem from .git import GitFileSystem # noqa: F401 @@ -51,7 +51,9 @@ def download( ) -> int: from dvc.scm import lfs_prefetch - with Callback.as_tqdm_callback( + from .callbacks import TqdmCallback + + with TqdmCallback( desc=f"Downloading {fs.path.name(fs_path)}", unit="files", ) as cb: diff --git a/dvc/output.py b/dvc/output.py index 59f384057e..decc97e058 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -786,7 +786,7 @@ def commit(self, filter_info=None, relink=True) -> None: self.hash_name, ignore=self.dvcignore, ) - with Callback.as_tqdm_callback( + with TqdmCallback( desc=f"Committing {self} to cache", unit="file", ) as cb: @@ -826,7 +826,7 @@ def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]: save_obj = obj.filter(prefix) assert isinstance(save_obj, Tree) checkout_obj = save_obj.get_obj(self.cache, prefix) - with Callback.as_tqdm_callback( + with TqdmCallback( desc=f"Committing {self} to cache", unit="file", ) as cb: @@ -1059,7 +1059,7 @@ def transfer( upload=upload, no_progress_bar=no_progress_bar, ) - with Callback.as_tqdm_callback( + with TqdmCallback( desc=f"Transferring to {odb.fs.unstrip_protocol(odb.path)}", unit="file", ) as cb: @@ -1092,7 +1092,7 @@ def get_files_number(self, filter_info=None): def unprotect(self): if self.exists and self.use_cache: - with Callback.as_tqdm_callback( + with TqdmCallback( size=self.meta.nfiles or -1, desc=f"Unprotecting {self}" ) as callback: self.cache.unprotect(self.fs_path, callback=callback) @@ -1438,7 +1438,7 @@ def add( # noqa: C901 assert staging assert obj.hash_info - with Callback.as_tqdm_callback( + with TqdmCallback( desc=f"Adding {self} to cache", unit="file", ) as cb: diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 071e083af3..2c6f965bf5 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -449,9 +449,9 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo": return init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir) def unprotect(self, target): - from dvc_objects.fs.callbacks import Callback + from dvc.fs.callbacks import TqdmCallback - with Callback.as_tqdm_callback(desc=f"Unprotecting {target}") as callback: + with TqdmCallback(desc=f"Unprotecting {target}") as callback: return self.cache.repo.unprotect(target, callback=callback) def _ignore(self): diff --git a/dvc/repo/artifacts.py b/dvc/repo/artifacts.py index 34ee01f24e..87a62ec7e7 100644 --- a/dvc/repo/artifacts.py +++ b/dvc/repo/artifacts.py @@ -210,7 +210,8 @@ def _download_studio( ) -> Tuple[int, str]: from dvc_studio_client.model_registry import get_download_uris - from dvc.fs import Callback, HTTPFileSystem, generic, localfs + from dvc.fs import HTTPFileSystem, generic, localfs + from dvc.fs.callbacks import TqdmCallback logger.debug("Trying to download artifact '%s' via studio", name) out = out or os.getcwd() @@ -244,7 +245,7 @@ def _download_studio( ) from exc fs = HTTPFileSystem() jobs = jobs or fs.jobs - with Callback.as_tqdm_callback( + with TqdmCallback( desc=f"Downloading '{name}' from '{repo_url}'", unit="files", ) as cb: diff --git a/dvc/repo/imports.py b/dvc/repo/imports.py index 8e377fc976..c373d7ace9 100644 --- a/dvc/repo/imports.py +++ b/dvc/repo/imports.py @@ -100,10 +100,10 @@ def save_imports( Returns: Objects which were downloaded from source location. """ + from dvc.fs.callbacks import TqdmCallback from dvc.stage.exceptions import DataSourceChanged from dvc_data.index import md5, save from dvc_data.index.checkout import apply, compare - from dvc_objects.fs.callbacks import Callback downloaded: Set["HashInfo"] = set() @@ -122,7 +122,7 @@ def save_imports( if not cache.fs.exists(cache.path): os.makedirs(cache.path) with TemporaryDirectory(dir=cache.path) as tmpdir: - with Callback.as_tqdm_callback( + with TqdmCallback( desc="Downloading imports from source", unit="files", ) as cb: diff --git a/dvc/repo/worktree.py b/dvc/repo/worktree.py index 8b8d4a95a1..0235754983 100644 --- a/dvc/repo/worktree.py +++ b/dvc/repo/worktree.py @@ -3,7 +3,6 @@ from funcy import first -from dvc.fs.callbacks import Callback from dvc.log import logger from dvc.stage.exceptions import StageUpdateError @@ -247,11 +246,12 @@ def _fetch_out_changes( remote_index: Union["DataIndex", "DataIndexView"], remote: "Remote", ): + from dvc.fs.callbacks import TqdmCallback from dvc_data.index.checkout import apply, compare old, new = _get_diff_indexes(out, local_index, remote_index) - with Callback.as_tqdm_callback( + with TqdmCallback( unit="entry", desc="Comparing indexes", ) as cb: @@ -265,9 +265,7 @@ def _fetch_out_changes( ) total = len(new) - with Callback.as_tqdm_callback( - unit="file", desc=f"Updating '{out}'", disable=total == 0 - ) as cb: + with TqdmCallback(unit="file", desc=f"Updating '{out}'", disable=total == 0) as cb: cb.set_size(total) apply( diff, diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index f8807191c1..4e8501eb1c 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -231,7 +231,7 @@ def restore(self, stage, run_cache=True, pull=False, dry=False): # noqa: C901 def transfer(self, from_odb, to_odb): from dvc.fs import HTTPFileSystem, LocalFileSystem - from dvc.fs.callbacks import Callback + from dvc.fs.callbacks import TqdmCallback from_fs = from_odb.fs to_fs = to_odb.fs @@ -266,7 +266,7 @@ def transfer(self, from_odb, to_odb): src_name = from_fs.path.name(src) parent_name = from_fs.path.name(from_fs.path.parent(src)) - with Callback.as_tqdm_callback( + with TqdmCallback( desc=src_name, bytes=True, ) as cb: diff --git a/pyproject.toml b/pyproject.toml index 5c498d3518..29a29f6f99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "configobj>=5.0.6", "distro>=1.3", "dpath<3,>=2.1.0", - "dvc-data>=2.23.1,<2.24", + "dvc-data>=2.24,<2.25", "dvc-http>=2.29.0", "dvc-render>=1.0.0,<2", "dvc-studio-client>=0.17.1,<1",