Merge pull request #2846 from casperdcl/add

UI progress cleanup

dvc/cli.py

@@ -3,37 +3,39 @@ import argparse
 import logging
 import sys
 
-import dvc.command.add as add
-import dvc.command.cache as cache
-import dvc.command.checkout as checkout
-import dvc.command.commit as commit
-import dvc.command.config as config
-import dvc.command.daemon as daemon
-import dvc.command.data_sync as data_sync
-import dvc.command.destroy as destroy
-import dvc.command.diff as diff
-import dvc.command.gc as gc
-import dvc.command.get as get
-import dvc.command.get_url as get_url
-import dvc.command.imp as imp
-import dvc.command.imp_url as imp_url
-import dvc.command.init as init
-import dvc.command.install as install
-import dvc.command.lock as lock
-import dvc.command.metrics as metrics
-import dvc.command.move as move
-import dvc.command.pipeline as pipeline
-import dvc.command.remote as remote
-import dvc.command.remove as remove
-import dvc.command.repro as repro
-import dvc.command.root as root
-import dvc.command.run as run
-import dvc.command.tag as tag
-import dvc.command.unprotect as unprotect
-import dvc.command.update as update
-import dvc.command.version as version
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import DvcParserError
+from .command import (
+    add,
+    cache,
+    checkout,
+    commit,
+    config,
+    daemon,
+    data_sync,
+    destroy,
+    diff,
+    gc,
+    get,
+    get_url,
+    imp,
+    imp_url,
+    init,
+    install,
+    lock,
+    metrics,
+    move,
+    pipeline,
+    remote,
+    remove,
+    repro,
+    root,
+    run,
+    tag,
+    unprotect,
+    update,
+    version,
+)
+from .command.base import fix_subparsers
+from .exceptions import DvcParserError
 
 
 logger = logging.getLogger(__name__)

dvc/output/base.py

@@ -405,7 +405,7 @@ class OutputBase(object):
 
         if self.stage.is_repo_import:
             cache = NamedCache()
-            dep, = self.stage.deps
+            (dep,) = self.stage.deps
             cache.external[dep.repo_pair].add(dep.def_path)
             return cache
 

dvc/progress.py

@@ -4,7 +4,6 @@ import logging
 import sys
 from threading import RLock
 
-from funcy import merge
 from tqdm import tqdm
 
 from dvc.utils import env2bool
@@ -31,7 +30,10 @@ class Tqdm(tqdm):
     )
     BAR_FMT_NOTOTAL = (
         "{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
-        " [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"
+        " [{elapsed}, {rate_fmt:>11}{postfix}]"
+    )
+    BYTES_DEFAULTS = dict(
+        unit="B", unit_scale=True, unit_divisor=1024, miniters=1
     )
 
     def __init__(
@@ -44,6 +46,7 @@ class Tqdm(tqdm):
         bar_format=None,
         bytes=False,  # pylint: disable=W0622
         file=None,
+        total=None,
         **kwargs
     ):
         """
@@ -58,12 +61,10 @@ class Tqdm(tqdm):
         kwargs  : anything accepted by `tqdm.tqdm()`
         """
         kwargs = kwargs.copy()
-        kwargs.setdefault("unit_scale", True)
         if bytes:
-            bytes_defaults = dict(
-                unit="B", unit_scale=True, unit_divisor=1024, miniters=1
-            )
-            kwargs = merge(bytes_defaults, kwargs)
+            kwargs = {**self.BYTES_DEFAULTS, **kwargs}
+        else:
+            kwargs.setdefault("unit_scale", total > 999 if total else True)
         if file is None:
             file = sys.stderr
         self.desc_persist = desc
@@ -84,6 +85,7 @@ class Tqdm(tqdm):
             desc=desc,
             bar_format="!",
             lock_args=(False,),
+            total=total,
             **kwargs
         )
         if bar_format is None:
@@ -114,6 +116,10 @@ class Tqdm(tqdm):
     def close(self):
         if self.desc_persist is not None:
             self.set_description_str(self.desc_persist, refresh=False)
+        # unknown/zero ETA
+        self.bar_format = self.bar_format.replace("<{remaining}", "")
+        # remove completed bar
+        self.bar_format = self.bar_format.replace("|{bar:10}|", " ")
         super().close()
 
     @property

dvc/remote/base.py

@@ -493,7 +493,9 @@ class RemoteBASE(object):
         cache_info = self.checksum_to_path_info(checksum)
         dir_info = self.get_dir_cache(checksum)
 
-        for entry in dir_info:
+        for entry in Tqdm(
+            dir_info, desc="Saving " + path_info.name, unit="file"
+        ):
             entry_info = path_info / entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
             self._save_file(entry_info, entry_checksum, save_link=False)

dvc/repo/__init__.py

@@ -205,7 +205,7 @@ class Repo(object):
             return [(stage, None) for stage in self.stages]
 
         try:
-            out, = self.find_outs_by_path(target, strict=False)
+            (out,) = self.find_outs_by_path(target, strict=False)
             filter_info = PathInfo(os.path.abspath(target))
             return [(out.stage, filter_info)]
         except OutputNotFoundError:
@@ -419,7 +419,7 @@ class Repo(object):
 
     def find_out_by_relpath(self, relpath):
         path = os.path.join(self.root_dir, relpath)
-        out, = self.find_outs_by_path(path)
+        (out,) = self.find_outs_by_path(path)
         return out
 
     def is_dvc_internal(self, path):

dvc/repo/add.py

@@ -4,11 +4,12 @@ import os
 import colorama
 
 from . import locked
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-from dvc.progress import Tqdm
-from dvc.repo.scm_context import scm_context
-from dvc.stage import Stage
-from dvc.utils import LARGE_DIR_SIZE
+from ..exceptions import RecursiveAddingWhileUsingFilename
+from ..output.base import OutputDoesNotExistError
+from ..progress import Tqdm
+from ..repo.scm_context import scm_context
+from ..stage import Stage
+from ..utils import LARGE_DIR_SIZE
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +24,12 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
         targets = [targets]
 
     stages_list = []
-    with Tqdm(total=len(targets), desc="Add", unit="file", leave=True) as pbar:
+    num_targets = len(targets)
+    with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar:
+        if num_targets == 1:
+            # clear unneeded top-level progress bar for single target
+            pbar.bar_format = "Adding..."
+            pbar.refresh()
         for target in targets:
             sub_targets = _find_all_targets(repo, target, recursive)
             pbar.total += len(sub_targets) - 1
@@ -45,17 +51,29 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
 
             repo.check_modified_graph(stages)
 
-            for stage in stages:
-                stage.save()
+            with Tqdm(
+                total=len(stages),
+                desc="Processing",
+                unit="file",
+                disable=True if len(stages) == 1 else None,
+            ) as pbar_stages:
+                for stage in stages:
+                    try:
+                        stage.save()
+                    except OutputDoesNotExistError:
+                        pbar.n -= 1
+                        raise
 
-                if not no_commit:
-                    stage.commit()
+                    if not no_commit:
+                        stage.commit()
 
-                stage.dump()
+                    stage.dump()
+                    pbar_stages.update()
 
             stages_list += stages
-        # remove filled bar bit of progress, leaving stats
-        pbar.bar_format = pbar.BAR_FMT_DEFAULT.replace("|{bar:10}|", " ")
+
+        if num_targets == 1:  # restore bar format for stats
+            pbar.bar_format = pbar.BAR_FMT_DEFAULT
 
     return stages_list
 
@@ -64,7 +82,12 @@ def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
             fname
-            for fname in repo.tree.walk_files(target)
+            for fname in Tqdm(
+                repo.tree.walk_files(target),
+                desc="Searching " + target,
+                bar_format=Tqdm.BAR_FMT_NOTOTAL,
+                unit="file",
+            )
             if not repo.is_dvc_internal(fname)
             if not Stage.is_stage_file(fname)
             if not repo.scm.belongs_to_scm(fname)
@@ -76,7 +99,12 @@ def _find_all_targets(repo, target, recursive):
 def _create_stages(repo, targets, fname, pbar=None):
     stages = []
 
-    for out in targets:
+    for out in Tqdm(
+        targets,
+        desc="Creating DVC-files",
+        disable=True if len(targets) < LARGE_DIR_SIZE else None,
+        unit="file",
+    ):
         stage = Stage.create(repo, outs=[out], add=True, fname=fname)
 
         if not stage:

dvc/repo/get_url.py

@@ -14,6 +14,6 @@ def get_url(url, out=None):
 
     out = os.path.abspath(out)
 
-    dep, = dependency.loads_from(None, [url])
-    out, = output.loads_from(None, [out], use_cache=False)
+    (dep,) = dependency.loads_from(None, [url])
+    (out,) = output.loads_from(None, [out], use_cache=False)
     dep.download(out)