decrease verbosity, default progress leave=False

This commit is contained in:
Casper da Costa-Luis
2019-08-27 10:57:18 +01:00
parent c8cb9ecef3
commit 7c6d68ad5a
4 changed files with 22 additions and 8 deletions
+11 -4
View File
@@ -5,6 +5,8 @@ from tqdm import tqdm
from copy import deepcopy
from concurrent.futures import ThreadPoolExecutor
logger = logging.getLogger(__name__)
class TqdmThreadPoolExecutor(ThreadPoolExecutor):
"""
@@ -49,12 +51,18 @@ class Tqdm(tqdm):
desc_truncate=None,
leave=None,
bar_format=None,
level=logging.ERROR,
level_leave=logging.DEBUG,
**kwargs
):
"""
bytes : shortcut for
`unit='B', unit_scale=True, unit_divisor=1024, miniters=1`
desc_truncate : like `desc` but will truncate to 10 chars
level : effective logging level for determining `disable`;
used only if `disable` is unspecified
level_leave : effective logging level for determining `leave`;
used only if `leave` is unspecified
kwargs : anything accepted by `tqdm.tqdm()`
"""
kwargs = deepcopy(kwargs)
@@ -67,10 +75,9 @@ class Tqdm(tqdm):
if desc_truncate is not None:
kwargs.setdefault("desc", self.truncate(desc_truncate))
if disable is None:
disable = (
logging.getLogger(__name__).getEffectiveLevel()
>= logging.CRITICAL
)
disable = logger.getEffectiveLevel() > level
if leave is None:
leave = logger.getEffectiveLevel() <= level_leave
if bar_format is None:
if kwargs.get("total", hasattr(iterable, "__len__")):
bar_format = self.BAR_FMT_DEFAULT
+4 -2
View File
@@ -315,12 +315,13 @@ class RemoteLOCAL(RemoteBASE):
show_checksums=False,
download=False,
):
logger.info(
logger.debug(
"Preparing to collect status from {}".format(remote.path_info)
)
ret = self._group(checksum_infos, show_checksums=show_checksums) or {}
md5s = list(ret)
logger.debug("Collecting information from local cache...")
local_exists = self.cache_exists(md5s, jobs=jobs)
# This is a performance optimization. We can safely assume that,
@@ -330,6 +331,7 @@ class RemoteLOCAL(RemoteBASE):
if download and sorted(local_exists) == sorted(md5s):
remote_exists = local_exists
else:
logger.debug("Collecting information from remote cache...")
remote_exists = list(remote.cache_exists(md5s, jobs=jobs))
self._fill_statuses(ret, local_exists, remote_exists)
@@ -377,7 +379,7 @@ class RemoteLOCAL(RemoteBASE):
show_checksums=False,
download=False,
):
logger.info(
logger.debug(
"Preparing to {} '{}'".format(
"download data from" if download else "upload data to",
remote.path_info,
-1
View File
@@ -280,5 +280,4 @@ class RemoteSSH(RemoteBASE):
results = executor.map(exists_with_progress, chunks)
in_remote = itertools.chain.from_iterable(results)
ret = list(itertools.compress(checksums, in_remote))
pbar.set_description_str("Querying remote cache")
return ret
+7 -1
View File
@@ -38,8 +38,14 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False):
with self.state:
_cleanup_unused_links(self, all_stages)
total = get_all_files_numbers(stages)
if total == 0:
logger.info("Nothing to do")
with Tqdm(
total=total, unit="file", desc="Checkout", disable=total == 0
total=total,
unit="file",
desc="Checkout",
disable=total == 0,
level_leave=logging.INFO,
) as pbar:
for stage in stages:
if stage.locked: