From 7041fa5f4226895b2029e64b5d3e4b52e3339823 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 25 Jun 2023 23:14:49 +0300 Subject: [PATCH] checkout: use ui.progress (#9661) * fs: callbacks: create our own TqdmCallback We have a near identical implementation in dvc-objects, but this one is meant to work with our Tqdm progress instance. dvc-objects one is useful on other levels, but dvc should have itsown that inherits from dvc-objects' Callback. * progress: add as_callback * checkout: use ui.progress --- dvc/fs/callbacks.py | 60 ++++++++++++++++++++++++++++++++++++++++---- dvc/progress.py | 9 +++++++ dvc/repo/checkout.py | 24 ++++++++++-------- 3 files changed, 78 insertions(+), 15 deletions(-) diff --git a/dvc/fs/callbacks.py b/dvc/fs/callbacks.py index 6b0bbbb99..6fb4658b8 100644 --- a/dvc/fs/callbacks.py +++ b/dvc/fs/callbacks.py @@ -2,12 +2,9 @@ from contextlib import ExitStack from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union +from dvc.progress import Tqdm from dvc.utils.objects import cached_property -from dvc_objects.fs.callbacks import ( # noqa: F401 - DEFAULT_CALLBACK, - Callback, - TqdmCallback, -) +from dvc_objects.fs.callbacks import DEFAULT_CALLBACK, Callback # noqa: F401 if TYPE_CHECKING: from rich.progress import TaskID @@ -15,6 +12,59 @@ if TYPE_CHECKING: from dvc.ui._rich_progress import RichTransferProgress +# NOTE: this is very similar to dvc-objects.fs.callbacks.TqdmCallback, +# but it works with our own Tqdm instance. +class TqdmCallback(Callback): + def __init__( + self, + size: Optional[int] = None, + value: int = 0, + progress_bar: Optional["Tqdm"] = None, + **tqdm_kwargs, + ): + tqdm_kwargs["total"] = size or -1 + self._tqdm_kwargs = tqdm_kwargs + self._progress_bar = progress_bar + self._stack = ExitStack() + super().__init__(size=size, value=value) + + @cached_property + def progress_bar(self): + progress_bar = ( + self._progress_bar + if self._progress_bar is not None + else Tqdm(**self._tqdm_kwargs) + ) + return self._stack.enter_context(progress_bar) + + def __enter__(self): + return self + + def close(self): + self._stack.close() + + def set_size(self, size): + # Tqdm tries to be smart when to refresh, + # so we try to force it to re-render. + super().set_size(size) + self.progress_bar.refresh() + + def call(self, hook_name=None, **kwargs): # noqa: ARG002 + self.progress_bar.update_to(self.value, total=self.size) + + def branch( + self, + path_1: "Union[str, BinaryIO]", + path_2: str, + kwargs: Dict[str, Any], + child: Optional[Callback] = None, + ): + child = child or TqdmCallback( + bytes=True, desc=path_1 if isinstance(path_1, str) else path_2 + ) + return super().branch(path_1, path_2, kwargs, child=child) + + class RichCallback(Callback): def __init__( self, diff --git a/dvc/progress.py b/dvc/progress.py index 057731270..9754cfbb7 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -2,12 +2,16 @@ import logging import sys from threading import RLock +from typing import TYPE_CHECKING from tqdm import tqdm from dvc.env import DVC_IGNORE_ISATTY from dvc.utils import env2bool +if TYPE_CHECKING: + from dvc.fs.callbacks import TqdmCallback + logger = logging.getLogger(__name__) tqdm.set_lock(RLock()) @@ -158,3 +162,8 @@ class Tqdm(tqdm): d["ncols_desc"] = d["ncols_info"] = 1 d["prefix"] = "" return d + + def as_callback(self) -> "TqdmCallback": + from dvc.fs.callbacks import TqdmCallback + + return TqdmCallback(progress_bar=self) diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index be2e647b2..67e4ce268 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -8,6 +8,7 @@ from dvc.exceptions import ( DvcException, NoOutputOrStageError, ) +from dvc.ui import ui from dvc.utils import relpath from . import locked @@ -106,7 +107,6 @@ def checkout( # noqa: C901 allow_missing=False, **kwargs, ): - from dvc.fs.callbacks import Callback from dvc.repo.index import build_data_index from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare @@ -142,21 +142,25 @@ def checkout( # noqa: C901 onerror=onerror, ) - with Callback.as_tqdm_callback( + with ui.progress( unit="entry", desc="Building data index", - ) as cb: + ) as pb: old = build_data_index( - view, self.root_dir, self.fs, compute_hash=True, callback=cb + view, + self.root_dir, + self.fs, + compute_hash=True, + callback=pb.as_callback(), ) new = view.data["repo"] - with Callback.as_tqdm_callback( + with ui.progress( desc="Comparing indexes", unit="entry", - ) as cb: - diff = compare(old, new, relink=relink, delete=True, callback=cb) + ) as pb: + diff = compare(old, new, relink=relink, delete=True, callback=pb.as_callback()) if not force: _check_can_delete(diff.files_delete, new, self.root_dir, self.fs) @@ -173,15 +177,15 @@ def checkout( # noqa: C901 if self.fs.path.isin_or_eq(dest_path, out_path): failed.add(out_path) - with Callback.as_tqdm_callback( + with ui.progress( unit="file", desc="Checkout", - ) as cb: + ) as pb: apply( diff, self.root_dir, self.fs, - callback=cb, + callback=pb.as_callback(), update_meta=False, onerror=checkout_onerror, state=self.state,