diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 4740b409d..ee0ff0f13 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -75,7 +75,8 @@ def checkout( # noqa: C901 from dvc.repo.index import build_data_index from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError from dvc_data.hashfile.checkout import CheckoutError as IndexCheckoutError - from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare + from dvc_data.index.checkout import ADD, DELETE, MODIFY + from dvc_data.index.checkout import checkout as icheckout stats: Dict[str, List[str]] = { "added": [], @@ -118,24 +119,21 @@ def checkout( # noqa: C901 new = view.data["repo"] - with Callback.as_tqdm_callback( - desc="Comparing indexes", - unit="entry", - ) as cb: - diff = compare(old, new, relink=relink, delete=True, callback=cb) - with Callback.as_tqdm_callback( unit="file", desc="Checkout", ) as cb: try: - changes = apply( - diff, + changes = icheckout( + new, self.root_dir, self.fs, + old=old, callback=cb, + delete=True, prompt=prompt.confirm, update_meta=False, + relink=relink, force=force, allow_missing=allow_missing, state=self.state, diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 4e6d8615b..47311acd3 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -515,6 +515,7 @@ class BaseExecutor(ABC): targets=targets, with_deps=targets is not None, force=True, + quiet=True, allow_missing=True, recursive=kwargs.get("recursive", False), ) diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index c29a088b2..78b987276 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -36,7 +36,6 @@ def fetch( # noqa: C901, PLR0913 from fsspec.utils import tokenize from dvc.fs.callbacks import Callback - from dvc_data.index.fetch import collect from dvc_data.index.fetch import fetch as ifetch if isinstance(targets, str): @@ -77,27 +76,17 @@ def fetch( # noqa: C901, PLR0913 cache_key = ("fetch", tokenize(sorted(index_keys))) - with Callback.as_tqdm_callback( - desc="Collecting", - unit="entry", - ) as cb: - data = collect( - indexes, cache_index=self.data_index, cache_key=cache_key, callback=cb - ) - with Callback.as_tqdm_callback( desc="Fetching", unit="file", ) as cb: - try: - fetch_transferred, fetch_failed = ifetch( - data, - jobs=jobs, - callback=cb, - ) # pylint: disable=assignment-from-no-return - finally: - for fs_index in data.values(): - fs_index.close() + fetch_transferred, fetch_failed = ifetch( + indexes, + jobs=jobs, + callback=cb, + cache_index=self.data_index, + cache_key=cache_key, + ) # pylint: disable=assignment-from-no-return if fetch_transferred: # NOTE: dropping cached index to force reloading from newly saved cache diff --git a/dvc/repo/imports.py b/dvc/repo/imports.py index 62fd0663f..f14f7c996 100644 --- a/dvc/repo/imports.py +++ b/dvc/repo/imports.py @@ -100,8 +100,7 @@ def save_imports( Objects which were downloaded from source location. """ from dvc.stage.exceptions import DataSourceChanged - from dvc_data.index import md5, save - from dvc_data.index.checkout import apply, compare + from dvc_data.index import checkout, md5, save from dvc_objects.fs.callbacks import Callback downloaded: Set["HashInfo"] = set() @@ -125,8 +124,7 @@ def save_imports( desc="Downloading imports from source", unit="files", ) as cb: - diff = compare(None, data_view) - apply(diff, tmpdir, cache.fs, callback=cb, storage="remote") + checkout(data_view, tmpdir, cache.fs, callback=cb, storage="remote") md5(data_view, name=cache.hash_name) save(data_view, odb=cache, hardlink=True) diff --git a/dvc/repo/worktree.py b/dvc/repo/worktree.py index 97fdc4ef4..3f86f2ee8 100644 --- a/dvc/repo/worktree.py +++ b/dvc/repo/worktree.py @@ -112,7 +112,8 @@ def push_worktree( **kwargs: Any, ) -> int: from dvc.repo.index import build_data_index - from dvc_data.index.checkout import VersioningNotSupported, apply, compare + from dvc_data.index import checkout + from dvc_data.index.checkout import VersioningNotSupported pushed = 0 stages: Set["Stage"] = set() @@ -138,18 +139,6 @@ def push_worktree( else: diff_kwargs = {} - with Callback.as_tqdm_callback( - unit="entry", - desc=f"Comparing indexes for remote {remote_obj.name!r}", - ) as cb: - diff = compare( - old_index, - new_index, - callback=cb, - delete=remote_obj.worktree, - **diff_kwargs, - ) - total = len(new_index) with Callback.as_tqdm_callback( unit="file", @@ -158,13 +147,16 @@ def push_worktree( ) as cb: cb.set_size(total) try: - stats = apply( - diff, + stats = checkout( + new_index, remote_obj.path, remote_obj.fs, + old=old_index, + delete=remote_obj.worktree, callback=cb, latest_only=remote_obj.worktree, jobs=jobs, + **diff_kwargs, ) pushed += sum(len(changes) for changes in stats.values()) except VersioningNotSupported: @@ -324,33 +316,23 @@ def _fetch_out_changes( remote_index: Union["DataIndex", "DataIndexView"], remote: "Remote", ): - from dvc_data.index.checkout import apply, compare + from dvc_data.index import checkout old, new = _get_diff_indexes(out, local_index, remote_index) - - with Callback.as_tqdm_callback( - unit="entry", - desc="Comparing indexes", - ) as cb: - diff = compare( - old, - new, - delete=True, - meta_only=True, - meta_cmp_key=partial(_meta_checksum, remote.fs), - callback=cb, - ) - total = len(new) with Callback.as_tqdm_callback( unit="file", desc=f"Updating '{out}'", disable=total == 0 ) as cb: cb.set_size(total) - apply( - diff, + checkout( + new, out.repo.root_dir, out.fs, + old=old, + delete=True, update_meta=False, + meta_only=True, + meta_cmp_key=partial(_meta_checksum, remote.fs), storage="data", callback=cb, ) diff --git a/pyproject.toml b/pyproject.toml index ac2259da2..77e663b56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "configobj>=5.0.6", "distro>=1.3", "dpath<3,>=2.1.0", - "dvc-data>=2.0.1,<2.1.0", + "dvc-data>=1.11.0,<1.12.0", "dvc-http>=2.29.0", "dvc-render>=0.3.1,<1", "dvc-studio-client>=0.9.2,<1",