From c9342754ed2b090f71bc7d97e5f307527f47992d Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 25 Aug 2019 01:18:57 +0100 Subject: [PATCH 01/15] fixed with progress - fixes #2434 --- dvc/progress.py | 15 ++++++++++++--- dvc/remote/base.py | 4 +++- dvc/remote/local/__init__.py | 6 +++--- dvc/remote/ssh/__init__.py | 6 ++++-- dvc/repo/checkout.py | 2 +- setup.py | 2 +- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index 04d8d953d..c4ce7c6a2 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -40,6 +40,10 @@ class Tqdm(tqdm): bytes=False, # pylint: disable=W0622 desc_truncate=None, leave=None, + bar_format=( + "{percentage:3.0f}%|{bar:10}|{desc} {bar:-10b}{n}/{total}" + " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]" + ), **kwargs ): """ @@ -49,6 +53,7 @@ class Tqdm(tqdm): kwargs : anything accepted by `tqdm.tqdm()` """ kwargs = deepcopy(kwargs) + kwargs.setdefault("unit_scale", True) if bytes: for k, v in dict( unit="B", unit_scale=True, unit_divisor=1024, miniters=1 @@ -62,14 +67,18 @@ class Tqdm(tqdm): >= logging.CRITICAL ) super(Tqdm, self).__init__( - iterable=iterable, disable=disable, leave=leave, **kwargs + iterable=iterable, + disable=disable, + leave=leave, + bar_format=bar_format, + **kwargs ) def update_desc(self, desc, n=1, truncate=True): """ - Calls `set_description(truncate(desc))` and `update(n)` + Calls `set_description_str(truncate(desc))` and `update(n)` """ - self.set_description( + self.set_description_str( self.truncate(desc) if truncate else desc, refresh=False ) self.update(n) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 67b174c44..544dea2ed 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -638,7 +638,9 @@ class RemoteBASE(object): if not self.no_traverse: return list(set(checksums) & set(self.all())) - with Tqdm(total=len(checksums), unit="md5") as pbar: + with Tqdm( + desc="Querying remote cache", total=len(checksums), unit="md5" + ) as pbar: def exists_with_progress(path_info): ret = self.exists(path_info) diff --git a/dvc/remote/local/__init__.py b/dvc/remote/local/__init__.py index 8381e3511..fdbdc865b 100644 --- a/dvc/remote/local/__init__.py +++ b/dvc/remote/local/__init__.py @@ -254,7 +254,9 @@ class RemoteLOCAL(RemoteBASE): def cache_exists(self, checksums, jobs=None): return [ checksum - for checksum in Tqdm(checksums, unit="md5") + for checksum in Tqdm( + checksums, unit="md5", desc="Querying local cache" + ) if not self.changed_cache_file(checksum) ] @@ -319,7 +321,6 @@ class RemoteLOCAL(RemoteBASE): ret = self._group(checksum_infos, show_checksums=show_checksums) or {} md5s = list(ret) - logger.info("Collecting information from local cache...") local_exists = self.cache_exists(md5s, jobs=jobs) # This is a performance optimization. We can safely assume that, @@ -329,7 +330,6 @@ class RemoteLOCAL(RemoteBASE): if download and sorted(local_exists) == sorted(md5s): remote_exists = local_exists else: - logger.info("Collecting information from remote cache...") remote_exists = list(remote.cache_exists(md5s, jobs=jobs)) self._fill_statuses(ret, local_exists, remote_exists) diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index d5df6a75e..4c48c6b02 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -267,7 +267,9 @@ class RemoteSSH(RemoteBASE): if not self.no_traverse: return list(set(checksums) & set(self.all())) - with Tqdm(total=len(checksums), unit="md5") as pbar: + with Tqdm( + desc="Querying remote cache", total=len(checksums), unit="md5" + ) as pbar: def exists_with_progress(chunks): return self.batch_exists(chunks, callback=pbar.update_desc) @@ -278,5 +280,5 @@ class RemoteSSH(RemoteBASE): results = executor.map(exists_with_progress, chunks) in_remote = itertools.chain.from_iterable(results) ret = list(itertools.compress(checksums, in_remote)) - pbar.update_desc("", 0) # clear path name description + pbar.set_description_str("Querying remote cache") return ret diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index e335763a2..077c18f5c 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -51,4 +51,4 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False): ) stage.checkout(force=force, progress_callback=pbar.update_desc) - pbar.update_desc("Checkout", 0) # clear path name description + pbar.set_description_str("Checkout") diff --git a/setup.py b/setup.py index 94c3b5153..b69333af5 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ install_requires = [ "funcy>=1.12", "pathspec>=0.5.9", "shortuuid>=0.5.0", - "tqdm>=4.34.0", + "tqdm>=4.35.0", "win-unicode-console>=0.5; sys_platform == 'win32'", ] From 06f920d138faf90e1f9b774a0cb81a4f015df07e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 26 Aug 2019 01:25:50 +0100 Subject: [PATCH 02/15] fix unknown total progress - depends on https://github.com/tqdm/tqdm/pull/800 --- dvc/progress.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index c4ce7c6a2..e921677db 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -32,6 +32,11 @@ class Tqdm(tqdm): """ maximum-compatibility tqdm-based progressbars """ + BAR_FMT_DEFAULT = ( + "{percentage:3.0f}%|{bar:10}|{desc} {bar:-10b}{n}/{total}" + " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]" + ) + BAR_FMT_NOTOTAL = "{desc} {bar:b}{n} [{elapsed}11}{postfix}]" def __init__( self, @@ -40,10 +45,7 @@ class Tqdm(tqdm): bytes=False, # pylint: disable=W0622 desc_truncate=None, leave=None, - bar_format=( - "{percentage:3.0f}%|{bar:10}|{desc} {bar:-10b}{n}/{total}" - " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]" - ), + bar_format=None, **kwargs ): """ @@ -66,6 +68,11 @@ class Tqdm(tqdm): logging.getLogger(__name__).getEffectiveLevel() >= logging.CRITICAL ) + if bar_format is None: + if kwargs.get('total', getattr(iterable, '__len__', None)) is None: + bar_format = self.BAR_FMT_NOTOTAL + else: + bar_format = self.BAR_FMT_DEFAULT super(Tqdm, self).__init__( iterable=iterable, disable=disable, From c8cb9ecef328a0bc68d591c9168b519b458d4a89 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 27 Aug 2019 09:56:32 +0100 Subject: [PATCH 03/15] minor logic tidy sacrifice accuracy for readability --- dvc/progress.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index e921677db..7fd60969b 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -32,11 +32,14 @@ class Tqdm(tqdm): """ maximum-compatibility tqdm-based progressbars """ + BAR_FMT_DEFAULT = ( "{percentage:3.0f}%|{bar:10}|{desc} {bar:-10b}{n}/{total}" " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]" ) - BAR_FMT_NOTOTAL = "{desc} {bar:b}{n} [{elapsed}11}{postfix}]" + BAR_FMT_NOTOTAL = ( + "{desc} {bar:b}{n} [{elapsed}11}{postfix}]" + ) def __init__( self, @@ -69,10 +72,10 @@ class Tqdm(tqdm): >= logging.CRITICAL ) if bar_format is None: - if kwargs.get('total', getattr(iterable, '__len__', None)) is None: - bar_format = self.BAR_FMT_NOTOTAL - else: + if kwargs.get("total", hasattr(iterable, "__len__")): bar_format = self.BAR_FMT_DEFAULT + else: + bar_format = self.BAR_FMT_NOTOTAL super(Tqdm, self).__init__( iterable=iterable, disable=disable, From 7c6d68ad5af1610fa0114e0751fa2d6d30d6593a Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 27 Aug 2019 10:57:18 +0100 Subject: [PATCH 04/15] decrease verbosity, default progress leave=False --- dvc/progress.py | 15 +++++++++++---- dvc/remote/local/__init__.py | 6 ++++-- dvc/remote/ssh/__init__.py | 1 - dvc/repo/checkout.py | 8 +++++++- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index 7fd60969b..a638d1bf8 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -5,6 +5,8 @@ from tqdm import tqdm from copy import deepcopy from concurrent.futures import ThreadPoolExecutor +logger = logging.getLogger(__name__) + class TqdmThreadPoolExecutor(ThreadPoolExecutor): """ @@ -49,12 +51,18 @@ class Tqdm(tqdm): desc_truncate=None, leave=None, bar_format=None, + level=logging.ERROR, + level_leave=logging.DEBUG, **kwargs ): """ bytes : shortcut for `unit='B', unit_scale=True, unit_divisor=1024, miniters=1` desc_truncate : like `desc` but will truncate to 10 chars + level : effective logging level for determining `disable`; + used only if `disable` is unspecified + level_leave : effective logging level for determining `leave`; + used only if `leave` is unspecified kwargs : anything accepted by `tqdm.tqdm()` """ kwargs = deepcopy(kwargs) @@ -67,10 +75,9 @@ class Tqdm(tqdm): if desc_truncate is not None: kwargs.setdefault("desc", self.truncate(desc_truncate)) if disable is None: - disable = ( - logging.getLogger(__name__).getEffectiveLevel() - >= logging.CRITICAL - ) + disable = logger.getEffectiveLevel() > level + if leave is None: + leave = logger.getEffectiveLevel() <= level_leave if bar_format is None: if kwargs.get("total", hasattr(iterable, "__len__")): bar_format = self.BAR_FMT_DEFAULT diff --git a/dvc/remote/local/__init__.py b/dvc/remote/local/__init__.py index fdbdc865b..d97fb0159 100644 --- a/dvc/remote/local/__init__.py +++ b/dvc/remote/local/__init__.py @@ -315,12 +315,13 @@ class RemoteLOCAL(RemoteBASE): show_checksums=False, download=False, ): - logger.info( + logger.debug( "Preparing to collect status from {}".format(remote.path_info) ) ret = self._group(checksum_infos, show_checksums=show_checksums) or {} md5s = list(ret) + logger.debug("Collecting information from local cache...") local_exists = self.cache_exists(md5s, jobs=jobs) # This is a performance optimization. We can safely assume that, @@ -330,6 +331,7 @@ class RemoteLOCAL(RemoteBASE): if download and sorted(local_exists) == sorted(md5s): remote_exists = local_exists else: + logger.debug("Collecting information from remote cache...") remote_exists = list(remote.cache_exists(md5s, jobs=jobs)) self._fill_statuses(ret, local_exists, remote_exists) @@ -377,7 +379,7 @@ class RemoteLOCAL(RemoteBASE): show_checksums=False, download=False, ): - logger.info( + logger.debug( "Preparing to {} '{}'".format( "download data from" if download else "upload data to", remote.path_info, diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index 4c48c6b02..8ccb29e8e 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -280,5 +280,4 @@ class RemoteSSH(RemoteBASE): results = executor.map(exists_with_progress, chunks) in_remote = itertools.chain.from_iterable(results) ret = list(itertools.compress(checksums, in_remote)) - pbar.set_description_str("Querying remote cache") return ret diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 077c18f5c..c7b967710 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -38,8 +38,14 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False): with self.state: _cleanup_unused_links(self, all_stages) total = get_all_files_numbers(stages) + if total == 0: + logger.info("Nothing to do") with Tqdm( - total=total, unit="file", desc="Checkout", disable=total == 0 + total=total, + unit="file", + desc="Checkout", + disable=total == 0, + level_leave=logging.INFO, ) as pbar: for stage in stages: if stage.locked: From de97d3ddac7f560d58ec89823a596cdcd1a45b9d Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 27 Aug 2019 11:08:27 +0100 Subject: [PATCH 05/15] persistent desc --- dvc/progress.py | 20 +++++++++++++++----- dvc/repo/checkout.py | 1 - 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index a638d1bf8..13c05ed36 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -47,18 +47,20 @@ class Tqdm(tqdm): self, iterable=None, disable=None, - bytes=False, # pylint: disable=W0622 + level=logging.ERROR, + desc=None, desc_truncate=None, leave=None, - bar_format=None, - level=logging.ERROR, level_leave=logging.DEBUG, + bar_format=None, + bytes=False, # pylint: disable=W0622 **kwargs ): """ bytes : shortcut for `unit='B', unit_scale=True, unit_divisor=1024, miniters=1` - desc_truncate : like `desc` but will truncate to 10 chars + desc : persists after `close()` + desc_truncate : like `desc` but will `truncate()` and not persist level : effective logging level for determining `disable`; used only if `disable` is unspecified level_leave : effective logging level for determining `leave`; @@ -72,8 +74,10 @@ class Tqdm(tqdm): unit="B", unit_scale=True, unit_divisor=1024, miniters=1 ).items(): kwargs.setdefault(k, v) + if desc is not None: + self.desc_persist = desc if desc_truncate is not None: - kwargs.setdefault("desc", self.truncate(desc_truncate)) + desc = self.truncate(desc_truncate) if disable is None: disable = logger.getEffectiveLevel() > level if leave is None: @@ -87,6 +91,7 @@ class Tqdm(tqdm): iterable=iterable, disable=disable, leave=leave, + desc=desc, bar_format=bar_format, **kwargs ) @@ -105,6 +110,11 @@ class Tqdm(tqdm): self.total = total # pylint: disable=W0613,W0201 self.update(current - self.n) + def close(self): + if hasattr(self, "desc_persist"): + self.set_description_str(self.desc_persist, refresh=False) + super(Tqdm, self).close() + @classmethod def truncate(cls, s, max_len=25, end=True, fill="..."): """ diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index c7b967710..26057c15c 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -57,4 +57,3 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False): ) stage.checkout(force=force, progress_callback=pbar.update_desc) - pbar.set_description_str("Checkout") From 043119b747d52e73d5e4b34c418960e27b74d9db Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 27 Aug 2019 13:25:43 +0100 Subject: [PATCH 06/15] misc minor tidy as per review --- dvc/progress.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index 13c05ed36..e0a00944c 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -2,7 +2,6 @@ from __future__ import print_function import logging from tqdm import tqdm -from copy import deepcopy from concurrent.futures import ThreadPoolExecutor logger = logging.getLogger(__name__) @@ -67,15 +66,14 @@ class Tqdm(tqdm): used only if `leave` is unspecified kwargs : anything accepted by `tqdm.tqdm()` """ - kwargs = deepcopy(kwargs) + kwargs = kwargs.copy() kwargs.setdefault("unit_scale", True) if bytes: for k, v in dict( unit="B", unit_scale=True, unit_divisor=1024, miniters=1 ).items(): kwargs.setdefault(k, v) - if desc is not None: - self.desc_persist = desc + self.desc_persist = desc if desc_truncate is not None: desc = self.truncate(desc_truncate) if disable is None: @@ -111,7 +109,7 @@ class Tqdm(tqdm): self.update(current - self.n) def close(self): - if hasattr(self, "desc_persist"): + if self.desc_persist: self.set_description_str(self.desc_persist, refresh=False) super(Tqdm, self).close() From 86605b6fa6d98237d5f069c468cdfb84601d0ef4 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 27 Aug 2019 13:41:12 +0100 Subject: [PATCH 07/15] use funcy --- dvc/progress.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index e0a00944c..ffef29255 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -3,6 +3,7 @@ from __future__ import print_function import logging from tqdm import tqdm from concurrent.futures import ThreadPoolExecutor +from funcy import merge logger = logging.getLogger(__name__) @@ -69,10 +70,10 @@ class Tqdm(tqdm): kwargs = kwargs.copy() kwargs.setdefault("unit_scale", True) if bytes: - for k, v in dict( - unit="B", unit_scale=True, unit_divisor=1024, miniters=1 - ).items(): - kwargs.setdefault(k, v) + kwargs = merge( + dict(unit="B", unit_scale=True, unit_divisor=1024, miniters=1), + kwargs, + ) self.desc_persist = desc if desc_truncate is not None: desc = self.truncate(desc_truncate) From 7171712fa998820d0fa3b9bb19ce7a9bf378011f Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 27 Aug 2019 13:41:37 +0100 Subject: [PATCH 08/15] delay bar_format --- dvc/progress.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index ffef29255..c0a67b323 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -81,19 +81,21 @@ class Tqdm(tqdm): disable = logger.getEffectiveLevel() > level if leave is None: leave = logger.getEffectiveLevel() <= level_leave - if bar_format is None: - if kwargs.get("total", hasattr(iterable, "__len__")): - bar_format = self.BAR_FMT_DEFAULT - else: - bar_format = self.BAR_FMT_NOTOTAL super(Tqdm, self).__init__( iterable=iterable, disable=disable, leave=leave, desc=desc, - bar_format=bar_format, + bar_format="", **kwargs ) + if bar_format is None: + self.bar_format = ( + self.BAR_FMT_DEFAULT if len(self) else self.BAR_FMT_NOTOTAL + ) + else: + self.bar_format = bar_format + self.refresh() def update_desc(self, desc, n=1, truncate=True): """ From 03a5c6df5e1d446a8270d02ebb9b72a1bf0cbe4e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 27 Aug 2019 14:00:04 +0100 Subject: [PATCH 09/15] progress default leave=False as per reviews --- dvc/progress.py | 7 +------ dvc/repo/checkout.py | 2 +- dvc/utils/__init__.py | 6 +++--- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index c0a67b323..64c021653 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -50,8 +50,7 @@ class Tqdm(tqdm): level=logging.ERROR, desc=None, desc_truncate=None, - leave=None, - level_leave=logging.DEBUG, + leave=False, bar_format=None, bytes=False, # pylint: disable=W0622 **kwargs @@ -63,8 +62,6 @@ class Tqdm(tqdm): desc_truncate : like `desc` but will `truncate()` and not persist level : effective logging level for determining `disable`; used only if `disable` is unspecified - level_leave : effective logging level for determining `leave`; - used only if `leave` is unspecified kwargs : anything accepted by `tqdm.tqdm()` """ kwargs = kwargs.copy() @@ -79,8 +76,6 @@ class Tqdm(tqdm): desc = self.truncate(desc_truncate) if disable is None: disable = logger.getEffectiveLevel() > level - if leave is None: - leave = logger.getEffectiveLevel() <= level_leave super(Tqdm, self).__init__( iterable=iterable, disable=disable, diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 26057c15c..40f4b123b 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -45,7 +45,7 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False): unit="file", desc="Checkout", disable=total == 0, - level_leave=logging.INFO, + leave=logger.getEffectiveLevel() < logging.CRITICAL, ) as pbar: for stage in stages: if stage.locked: diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 050f65366..9878a8c2e 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -51,16 +51,16 @@ def file_md5(fname): hash_md5 = hashlib.md5() binary = not istextfile(fname) size = os.path.getsize(fname) - bar = False + no_progress_bar = True if size >= LARGE_FILE_SIZE: - bar = True + no_progress_bar = False msg = "Computing md5 for a large file {}. This is only done once." logger.info(msg.format(relpath(fname))) name = relpath(fname) with Tqdm( desc_truncate=name, - disable=not bar, + disable=no_progress_bar, total=size, bytes=True, leave=False, From f336afa8611b2b08420edfbe537e6c393595bed5 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 27 Aug 2019 20:23:55 +0100 Subject: [PATCH 10/15] remove Tqdm.truncate => add {ncols_desc} --- dvc/progress.py | 43 ++++++++++++++---------------------- dvc/remote/azure.py | 8 ++----- dvc/remote/http.py | 2 +- dvc/remote/oss.py | 8 ++----- dvc/remote/s3.py | 10 ++------- dvc/remote/ssh/connection.py | 4 ++-- dvc/utils/__init__.py | 7 ++---- 7 files changed, 28 insertions(+), 54 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index 64c021653..a2331fc93 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -36,11 +36,13 @@ class Tqdm(tqdm): """ BAR_FMT_DEFAULT = ( - "{percentage:3.0f}%|{bar:10}|{desc} {bar:-10b}{n}/{total}" + "{percentage:3.0f}%|{bar:10}|" + "{desc:{ncols_desc}.{ncols_desc}}{n}/{total}" " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]" ) BAR_FMT_NOTOTAL = ( - "{desc} {bar:b}{n} [{elapsed}11}{postfix}]" + "{desc:{ncols_desc}.{ncols_desc}}{n}" + " [{elapsed}11}{postfix}]" ) def __init__( @@ -49,7 +51,6 @@ class Tqdm(tqdm): disable=None, level=logging.ERROR, desc=None, - desc_truncate=None, leave=False, bar_format=None, bytes=False, # pylint: disable=W0622 @@ -59,7 +60,6 @@ class Tqdm(tqdm): bytes : shortcut for `unit='B', unit_scale=True, unit_divisor=1024, miniters=1` desc : persists after `close()` - desc_truncate : like `desc` but will `truncate()` and not persist level : effective logging level for determining `disable`; used only if `disable` is unspecified kwargs : anything accepted by `tqdm.tqdm()` @@ -72,8 +72,6 @@ class Tqdm(tqdm): kwargs, ) self.desc_persist = desc - if desc_truncate is not None: - desc = self.truncate(desc_truncate) if disable is None: disable = logger.getEffectiveLevel() > level super(Tqdm, self).__init__( @@ -81,7 +79,7 @@ class Tqdm(tqdm): disable=disable, leave=leave, desc=desc, - bar_format="", + bar_format="!", **kwargs ) if bar_format is None: @@ -92,13 +90,11 @@ class Tqdm(tqdm): self.bar_format = bar_format self.refresh() - def update_desc(self, desc, n=1, truncate=True): + def update_desc(self, desc, n=1): """ - Calls `set_description_str(truncate(desc))` and `update(n)` + Calls `set_description_str(desc)` and `update(n)` """ - self.set_description_str( - self.truncate(desc) if truncate else desc, refresh=False - ) + self.set_description_str(desc, refresh=False) self.update(n) def update_to(self, current, total=None): @@ -107,20 +103,15 @@ class Tqdm(tqdm): self.update(current - self.n) def close(self): - if self.desc_persist: + if self.desc_persist is not None: self.set_description_str(self.desc_persist, refresh=False) super(Tqdm, self).close() - @classmethod - def truncate(cls, s, max_len=25, end=True, fill="..."): - """ - Guarantee len(output) < max_lenself. - >>> truncate("hello", 4) - '...o' - """ - if len(s) <= max_len: - return s - if len(fill) > max_len: - return fill[-max_len:] if end else fill[:max_len] - i = max_len - len(fill) - return (fill + s[-i:]) if end else (s[:i] + fill) + @property + def format_dict(self): + """inject `ncols_desc` to fill the display width (`ncols`)""" + d = super(Tqdm, self).format_dict + ncols = d.get("ncols", 80) + ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1 + d["ncols_desc"] = max(ncols_desc, 0) + return d diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 2527b66ba..e891c345d 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -115,9 +115,7 @@ class RemoteAZURE(RemoteBASE): def _upload( self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs ): - with Tqdm( - desc_truncate=name, disable=no_progress_bar, bytes=True - ) as pbar: + with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar: self.blob_service.create_blob_from_path( to_info.bucket, to_info.path, @@ -128,9 +126,7 @@ class RemoteAZURE(RemoteBASE): def _download( self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs ): - with Tqdm( - desc_truncate=name, disable=no_progress_bar, bytes=True - ) as pbar: + with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar: self.blob_service.get_blob_to_path( from_info.bucket, from_info.path, diff --git a/dvc/remote/http.py b/dvc/remote/http.py index 6f430b0e7..c12c6e940 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -38,7 +38,7 @@ class RemoteHTTP(RemoteBASE): total=None if no_progress_bar else self._content_length(from_info), leave=False, bytes=True, - desc_truncate=from_info.url if name is None else name, + desc=from_info.url if name is None else name, disable=no_progress_bar, ) as pbar: with open(to_file, "wb") as fd: diff --git a/dvc/remote/oss.py b/dvc/remote/oss.py index 57e900087..ca6d5a4e0 100644 --- a/dvc/remote/oss.py +++ b/dvc/remote/oss.py @@ -107,9 +107,7 @@ class RemoteOSS(RemoteBASE): def _upload( self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs ): - with Tqdm( - desc_truncate=name, disable=no_progress_bar, bytes=True - ) as pbar: + with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar: self.oss_service.put_object_from_file( to_info.path, from_file, progress_callback=pbar.update_to ) @@ -117,9 +115,7 @@ class RemoteOSS(RemoteBASE): def _download( self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs ): - with Tqdm( - desc_truncate=name, disable=no_progress_bar, bytes=True - ) as pbar: + with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar: self.oss_service.get_object_to_file( from_info.path, to_file, progress_callback=pbar.update_to ) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 938a9a2ae..7de02f152 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -208,10 +208,7 @@ class RemoteS3(RemoteBASE): def _upload(self, from_file, to_info, name=None, no_progress_bar=False): total = os.path.getsize(from_file) with Tqdm( - disable=no_progress_bar, - total=total, - bytes=True, - desc_truncate=name, + disable=no_progress_bar, total=total, bytes=True, desc=name ) as pbar: self.s3.upload_file( from_file, @@ -229,10 +226,7 @@ class RemoteS3(RemoteBASE): Bucket=from_info.bucket, Key=from_info.path )["ContentLength"] with Tqdm( - disable=no_progress_bar, - total=total, - bytes=True, - desc_truncate=name, + disable=no_progress_bar, total=total, bytes=True, desc=name ) as pbar: self.s3.download_file( from_info.bucket, from_info.path, to_file, Callback=pbar.update diff --git a/dvc/remote/ssh/connection.py b/dvc/remote/ssh/connection.py index fbb0c2b7b..43b380ada 100644 --- a/dvc/remote/ssh/connection.py +++ b/dvc/remote/ssh/connection.py @@ -169,7 +169,7 @@ class SSHConnection: def download(self, src, dest, no_progress_bar=False, progress_title=None): with Tqdm( - desc_truncate=progress_title or os.path.basename(src), + desc=progress_title or os.path.basename(src), disable=no_progress_bar, bytes=True, ) as pbar: @@ -186,7 +186,7 @@ class SSHConnection: progress_title = posixpath.basename(dest) with Tqdm( - desc_truncate=progress_title, disable=no_progress_bar, bytes=True + desc=progress_title, disable=no_progress_bar, bytes=True ) as pbar: self.sftp.put(src, tmp_file, callback=pbar.update_to) diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 9878a8c2e..59e943707 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -59,7 +59,7 @@ def file_md5(fname): name = relpath(fname) with Tqdm( - desc_truncate=name, + desc=name, disable=no_progress_bar, total=size, bytes=True, @@ -132,10 +132,7 @@ def copyfile(src, dest, no_progress_bar=False, name=None): System.reflink(src, dest) except DvcException: with Tqdm( - desc_truncate=name, - disable=no_progress_bar, - total=total, - bytes=True, + desc=name, disable=no_progress_bar, total=total, bytes=True ) as pbar: with open(src, "rb") as fsrc, open(dest, "wb+") as fdest: while True: From bc452bb68901bdad08d352027c3eb12578a7dba3 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 27 Aug 2019 20:30:43 +0100 Subject: [PATCH 11/15] minor test fix --- dvc/progress.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index a2331fc93..93334c46b 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -83,9 +83,10 @@ class Tqdm(tqdm): **kwargs ) if bar_format is None: - self.bar_format = ( - self.BAR_FMT_DEFAULT if len(self) else self.BAR_FMT_NOTOTAL - ) + if self.__len__(): + self.bar_format = self.BAR_FMT_DEFAULT + else: + self.bar_format = self.BAR_FMT_NOTOTAL else: self.bar_format = bar_format self.refresh() @@ -111,7 +112,7 @@ class Tqdm(tqdm): def format_dict(self): """inject `ncols_desc` to fill the display width (`ncols`)""" d = super(Tqdm, self).format_dict - ncols = d.get("ncols", 80) + ncols = d["ncols"] or 80 ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1 d["ncols_desc"] = max(ncols_desc, 0) return d From 5413576e274be6000267df06bfa04bda57e5d13a Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 28 Aug 2019 17:22:33 +0100 Subject: [PATCH 12/15] decrease final verbosity further; minor tidy --- dvc/progress.py | 6 +++--- dvc/repo/checkout.py | 6 +----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/dvc/progress.py b/dvc/progress.py index 93334c46b..df36726b8 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -67,10 +67,10 @@ class Tqdm(tqdm): kwargs = kwargs.copy() kwargs.setdefault("unit_scale", True) if bytes: - kwargs = merge( - dict(unit="B", unit_scale=True, unit_divisor=1024, miniters=1), - kwargs, + bytes_defaults = dict( + unit="B", unit_scale=True, unit_divisor=1024, miniters=1 ) + kwargs = merge(bytes_defaults, kwargs) self.desc_persist = desc if disable is None: disable = logger.getEffectiveLevel() > level diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 40f4b123b..de7e77cea 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -41,11 +41,7 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False): if total == 0: logger.info("Nothing to do") with Tqdm( - total=total, - unit="file", - desc="Checkout", - disable=total == 0, - leave=logger.getEffectiveLevel() < logging.CRITICAL, + total=total, unit="file", desc="Checkout", disable=total == 0 ) as pbar: for stage in stages: if stage.locked: From 5f81352f901bbc5b0dc0292aea472973a79e2d3e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 28 Aug 2019 17:44:12 +0100 Subject: [PATCH 13/15] add cache_dir to progress output --- dvc/remote/base.py | 17 ++++++++++++++--- dvc/remote/local/__init__.py | 17 +++++++++-------- dvc/remote/ssh/__init__.py | 7 +++++-- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 544dea2ed..2fbf79885 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -140,6 +140,14 @@ class RemoteBASE(object): def cache(self): return getattr(self.repo.cache, self.scheme) + @property + def cache_dir(self): + return ( + getattr(self.path_info, "fspath", self.path_info.url) + if self.path_info + else None + ) + def get_file_checksum(self, path_info): raise NotImplementedError @@ -612,7 +620,7 @@ class RemoteBASE(object): return self._changed_dir_cache(checksum) return self.changed_cache_file(checksum) - def cache_exists(self, checksums, jobs=None): + def cache_exists(self, checksums, jobs=None, name=None): """Check if the given checksums are stored in the remote. There are two ways of performing this check: @@ -630,7 +638,7 @@ class RemoteBASE(object): take much shorter time to just retrieve everything they have under a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can check if particular file exists much quicker, use their own - implementation of cache_exists (see http, local). + implementation of cache_exists (see ssh, local). Returns: A list with checksums that were found in the remote @@ -639,7 +647,10 @@ class RemoteBASE(object): return list(set(checksums) & set(self.all())) with Tqdm( - desc="Querying remote cache", total=len(checksums), unit="md5" + desc="Querying " + + ("cache in " + name if name else "remote cache"), + total=len(checksums), + unit="md5", ) as pbar: def exists_with_progress(path_info): diff --git a/dvc/remote/local/__init__.py b/dvc/remote/local/__init__.py index d97fb0159..5bba17b6a 100644 --- a/dvc/remote/local/__init__.py +++ b/dvc/remote/local/__init__.py @@ -97,10 +97,6 @@ class RemoteLOCAL(RemoteBASE): def state(self): return self.repo.state - @property - def cache_dir(self): - return self.path_info.fspath if self.path_info else None - @classmethod def supported(cls, config): return True @@ -251,11 +247,14 @@ class RemoteLOCAL(RemoteBASE): move(from_info, to_info, mode=mode) - def cache_exists(self, checksums, jobs=None): + def cache_exists(self, checksums, jobs=None, name=None): return [ checksum for checksum in Tqdm( - checksums, unit="md5", desc="Querying local cache" + checksums, + unit="md5", + desc="Querying " + + ("cache in " + name if name else "local cache"), ) if not self.changed_cache_file(checksum) ] @@ -322,7 +321,7 @@ class RemoteLOCAL(RemoteBASE): md5s = list(ret) logger.debug("Collecting information from local cache...") - local_exists = self.cache_exists(md5s, jobs=jobs) + local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir) # This is a performance optimization. We can safely assume that, # if the resources that we want to fetch are already cached, @@ -332,7 +331,9 @@ class RemoteLOCAL(RemoteBASE): remote_exists = local_exists else: logger.debug("Collecting information from remote cache...") - remote_exists = list(remote.cache_exists(md5s, jobs=jobs)) + remote_exists = list( + remote.cache_exists(md5s, jobs=jobs, name=remote.cache_dir) + ) self._fill_statuses(ret, local_exists, remote_exists) diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index 8ccb29e8e..5c243f471 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -258,7 +258,7 @@ class RemoteSSH(RemoteBASE): return results - def cache_exists(self, checksums, jobs=None): + def cache_exists(self, checksums, jobs=None, name=None): """This is older implementation used in remote/base.py We are reusing it in RemoteSSH, because SSH's batch_exists proved to be faster than current approach (relying on exists(path_info)) applied in @@ -268,7 +268,10 @@ class RemoteSSH(RemoteBASE): return list(set(checksums) & set(self.all())) with Tqdm( - desc="Querying remote cache", total=len(checksums), unit="md5" + desc="Querying " + + ("cache in " + name if name else "remote cache"), + total=len(checksums), + unit="md5", ) as pbar: def exists_with_progress(chunks): From 7be68d9448d2f1dcbaa4a1ca0ab980a2b62db841 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 28 Aug 2019 17:58:53 +0100 Subject: [PATCH 14/15] replace md5 => file --- dvc/remote/base.py | 6 +++--- dvc/remote/local/__init__.py | 2 +- dvc/remote/ssh/__init__.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 2fbf79885..9efd0180e 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -650,16 +650,16 @@ class RemoteBASE(object): desc="Querying " + ("cache in " + name if name else "remote cache"), total=len(checksums), - unit="md5", + unit="file", ) as pbar: def exists_with_progress(path_info): ret = self.exists(path_info) - pbar.update() + pbar.update_desc(str(path_info)) return ret with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor: - path_infos = [self.checksum_to_path_info(x) for x in checksums] + path_infos = map(self.checksum_to_path_info, checksums) in_remote = executor.map(exists_with_progress, path_infos) ret = list(itertools.compress(checksums, in_remote)) return ret diff --git a/dvc/remote/local/__init__.py b/dvc/remote/local/__init__.py index 5bba17b6a..0a0a4b84d 100644 --- a/dvc/remote/local/__init__.py +++ b/dvc/remote/local/__init__.py @@ -252,7 +252,7 @@ class RemoteLOCAL(RemoteBASE): checksum for checksum in Tqdm( checksums, - unit="md5", + unit="file", desc="Querying " + ("cache in " + name if name else "local cache"), ) diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index 5c243f471..74280d7bb 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -271,7 +271,7 @@ class RemoteSSH(RemoteBASE): desc="Querying " + ("cache in " + name if name else "remote cache"), total=len(checksums), - unit="md5", + unit="file", ) as pbar: def exists_with_progress(chunks): From 6c3029deded95ce2dc2ef82f239e3db54090e719 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 28 Aug 2019 18:02:50 +0100 Subject: [PATCH 15/15] revert cache_dir move partially reverts 5f81352f --- dvc/remote/base.py | 8 -------- dvc/remote/local/__init__.py | 8 +++++++- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 9efd0180e..5f77d8636 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -140,14 +140,6 @@ class RemoteBASE(object): def cache(self): return getattr(self.repo.cache, self.scheme) - @property - def cache_dir(self): - return ( - getattr(self.path_info, "fspath", self.path_info.url) - if self.path_info - else None - ) - def get_file_checksum(self, path_info): raise NotImplementedError diff --git a/dvc/remote/local/__init__.py b/dvc/remote/local/__init__.py index 0a0a4b84d..168c1f9c0 100644 --- a/dvc/remote/local/__init__.py +++ b/dvc/remote/local/__init__.py @@ -97,6 +97,10 @@ class RemoteLOCAL(RemoteBASE): def state(self): return self.repo.state + @property + def cache_dir(self): + return self.path_info.fspath if self.path_info else None + @classmethod def supported(cls, config): return True @@ -332,7 +336,9 @@ class RemoteLOCAL(RemoteBASE): else: logger.debug("Collecting information from remote cache...") remote_exists = list( - remote.cache_exists(md5s, jobs=jobs, name=remote.cache_dir) + remote.cache_exists( + md5s, jobs=jobs, name=str(remote.path_info) + ) ) self._fill_statuses(ret, local_exists, remote_exists)