mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
checkout: use ui.progress (#9661)
* fs: callbacks: create our own TqdmCallback We have a near identical implementation in dvc-objects, but this one is meant to work with our Tqdm progress instance. dvc-objects one is useful on other levels, but dvc should have itsown that inherits from dvc-objects' Callback. * progress: add as_callback * checkout: use ui.progress
This commit is contained in:
+55
-5
@@ -2,12 +2,9 @@
|
||||
from contextlib import ExitStack
|
||||
from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union
|
||||
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.utils.objects import cached_property
|
||||
from dvc_objects.fs.callbacks import ( # noqa: F401
|
||||
DEFAULT_CALLBACK,
|
||||
Callback,
|
||||
TqdmCallback,
|
||||
)
|
||||
from dvc_objects.fs.callbacks import DEFAULT_CALLBACK, Callback # noqa: F401
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from rich.progress import TaskID
|
||||
@@ -15,6 +12,59 @@ if TYPE_CHECKING:
|
||||
from dvc.ui._rich_progress import RichTransferProgress
|
||||
|
||||
|
||||
# NOTE: this is very similar to dvc-objects.fs.callbacks.TqdmCallback,
|
||||
# but it works with our own Tqdm instance.
|
||||
class TqdmCallback(Callback):
|
||||
def __init__(
|
||||
self,
|
||||
size: Optional[int] = None,
|
||||
value: int = 0,
|
||||
progress_bar: Optional["Tqdm"] = None,
|
||||
**tqdm_kwargs,
|
||||
):
|
||||
tqdm_kwargs["total"] = size or -1
|
||||
self._tqdm_kwargs = tqdm_kwargs
|
||||
self._progress_bar = progress_bar
|
||||
self._stack = ExitStack()
|
||||
super().__init__(size=size, value=value)
|
||||
|
||||
@cached_property
|
||||
def progress_bar(self):
|
||||
progress_bar = (
|
||||
self._progress_bar
|
||||
if self._progress_bar is not None
|
||||
else Tqdm(**self._tqdm_kwargs)
|
||||
)
|
||||
return self._stack.enter_context(progress_bar)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def close(self):
|
||||
self._stack.close()
|
||||
|
||||
def set_size(self, size):
|
||||
# Tqdm tries to be smart when to refresh,
|
||||
# so we try to force it to re-render.
|
||||
super().set_size(size)
|
||||
self.progress_bar.refresh()
|
||||
|
||||
def call(self, hook_name=None, **kwargs): # noqa: ARG002
|
||||
self.progress_bar.update_to(self.value, total=self.size)
|
||||
|
||||
def branch(
|
||||
self,
|
||||
path_1: "Union[str, BinaryIO]",
|
||||
path_2: str,
|
||||
kwargs: Dict[str, Any],
|
||||
child: Optional[Callback] = None,
|
||||
):
|
||||
child = child or TqdmCallback(
|
||||
bytes=True, desc=path_1 if isinstance(path_1, str) else path_2
|
||||
)
|
||||
return super().branch(path_1, path_2, kwargs, child=child)
|
||||
|
||||
|
||||
class RichCallback(Callback):
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -2,12 +2,16 @@
|
||||
import logging
|
||||
import sys
|
||||
from threading import RLock
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
from dvc.env import DVC_IGNORE_ISATTY
|
||||
from dvc.utils import env2bool
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs.callbacks import TqdmCallback
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
tqdm.set_lock(RLock())
|
||||
|
||||
@@ -158,3 +162,8 @@ class Tqdm(tqdm):
|
||||
d["ncols_desc"] = d["ncols_info"] = 1
|
||||
d["prefix"] = ""
|
||||
return d
|
||||
|
||||
def as_callback(self) -> "TqdmCallback":
|
||||
from dvc.fs.callbacks import TqdmCallback
|
||||
|
||||
return TqdmCallback(progress_bar=self)
|
||||
|
||||
+14
-10
@@ -8,6 +8,7 @@ from dvc.exceptions import (
|
||||
DvcException,
|
||||
NoOutputOrStageError,
|
||||
)
|
||||
from dvc.ui import ui
|
||||
from dvc.utils import relpath
|
||||
|
||||
from . import locked
|
||||
@@ -106,7 +107,6 @@ def checkout( # noqa: C901
|
||||
allow_missing=False,
|
||||
**kwargs,
|
||||
):
|
||||
from dvc.fs.callbacks import Callback
|
||||
from dvc.repo.index import build_data_index
|
||||
from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
|
||||
from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare
|
||||
@@ -142,21 +142,25 @@ def checkout( # noqa: C901
|
||||
onerror=onerror,
|
||||
)
|
||||
|
||||
with Callback.as_tqdm_callback(
|
||||
with ui.progress(
|
||||
unit="entry",
|
||||
desc="Building data index",
|
||||
) as cb:
|
||||
) as pb:
|
||||
old = build_data_index(
|
||||
view, self.root_dir, self.fs, compute_hash=True, callback=cb
|
||||
view,
|
||||
self.root_dir,
|
||||
self.fs,
|
||||
compute_hash=True,
|
||||
callback=pb.as_callback(),
|
||||
)
|
||||
|
||||
new = view.data["repo"]
|
||||
|
||||
with Callback.as_tqdm_callback(
|
||||
with ui.progress(
|
||||
desc="Comparing indexes",
|
||||
unit="entry",
|
||||
) as cb:
|
||||
diff = compare(old, new, relink=relink, delete=True, callback=cb)
|
||||
) as pb:
|
||||
diff = compare(old, new, relink=relink, delete=True, callback=pb.as_callback())
|
||||
|
||||
if not force:
|
||||
_check_can_delete(diff.files_delete, new, self.root_dir, self.fs)
|
||||
@@ -173,15 +177,15 @@ def checkout( # noqa: C901
|
||||
if self.fs.path.isin_or_eq(dest_path, out_path):
|
||||
failed.add(out_path)
|
||||
|
||||
with Callback.as_tqdm_callback(
|
||||
with ui.progress(
|
||||
unit="file",
|
||||
desc="Checkout",
|
||||
) as cb:
|
||||
) as pb:
|
||||
apply(
|
||||
diff,
|
||||
self.root_dir,
|
||||
self.fs,
|
||||
callback=cb,
|
||||
callback=pb.as_callback(),
|
||||
update_meta=False,
|
||||
onerror=checkout_onerror,
|
||||
state=self.state,
|
||||
|
||||
Reference in New Issue
Block a user