From bdd51c854b9d6ac91f690acb1d57566834fbb991 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 4 Mar 2018 15:11:21 +0200 Subject: [PATCH] cloud: don't print progress bar when collecting dir Signed-off-by: Ruslan Kuprieiev --- dvc/cloud/aws.py | 13 ++++++++++--- dvc/cloud/base.py | 4 ++-- dvc/cloud/gcp.py | 12 +++++++----- dvc/cloud/local.py | 2 +- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/dvc/cloud/aws.py b/dvc/cloud/aws.py index ca2585b07..7e354f140 100644 --- a/dvc/cloud/aws.py +++ b/dvc/cloud/aws.py @@ -130,7 +130,7 @@ class DataCloudAWS(DataCloudBase): """ return fname + '.download' - def _pull_key(self, key, fname): + def _pull_key(self, key, fname, no_progress_bar=False): Logger.debug("Pulling key '{}' from bucket '{}' to file '{}'".format(key.name, key.bucket.name, fname)) @@ -147,17 +147,24 @@ class DataCloudAWS(DataCloudBase): key.name, fname)) + if no_progress_bar: + cb = None + else: + cb = create_cb(name) + res_h = ResumableDownloadHandler(tracker_file_name=self._download_tracker(tmp_file), num_retries=10) try: - key.get_contents_to_filename(tmp_file, cb=create_cb(name), res_download_handler=res_h) + key.get_contents_to_filename(tmp_file, cb=cb, res_download_handler=res_h) except Exception as exc: Logger.error('Failed to download "{}": {}'.format(key.name, exc)) return None os.rename(tmp_file, fname) - progress.finish_target(name) + if not no_progress_bar: + progress.finish_target(name) + Logger.debug('Downloading completed') return fname diff --git a/dvc/cloud/base.py b/dvc/cloud/base.py index e280b1e8d..9e983753d 100644 --- a/dvc/cloud/base.py +++ b/dvc/cloud/base.py @@ -109,7 +109,7 @@ class DataCloudBase(object): Logger.debug("File '{}' does not exist in the cloud".format(path)) return ret tmp = os.path.join(tempfile.mkdtemp(), os.path.basename(path)) - self._pull_key(key, tmp) + self._pull_key(key, tmp, no_progress_bar=True) dir_path = tmp for relpath, md5 in Output.get_dir_cache(dir_path).items(): @@ -138,7 +138,7 @@ class DataCloudBase(object): if e.errno != os.errno.EEXIST: raise - def _pull_key(self, key, path): + def _pull_key(self, key, path, no_progress_bar=False): """ Cloud-specific method of pulling keys """ pass diff --git a/dvc/cloud/gcp.py b/dvc/cloud/gcp.py index ec6720401..1a8ae4917 100644 --- a/dvc/cloud/gcp.py +++ b/dvc/cloud/gcp.py @@ -45,7 +45,7 @@ class DataCloudGCP(DataCloudBase): return False - def _pull_key(self, key, path): + def _pull_key(self, key, path, no_progress_bar=False): self._makedirs(path) name = os.path.relpath(path, self._cloud_settings.cache.cache_dir) @@ -57,9 +57,10 @@ class DataCloudGCP(DataCloudBase): Logger.debug('Downloading cache file from gc "{}/{}"'.format(key.bucket.name, key.name)) - # percent_cb is not available for download_to_filename, so - # lets at least update progress at keypoints(start, finish) - progress.update_target(name, 0, None) + if not no_progress_bar: + # percent_cb is not available for download_to_filename, so + # lets at least update progress at keypoints(start, finish) + progress.update_target(name, 0, None) try: key.download_to_filename(tmp_file) @@ -69,7 +70,8 @@ class DataCloudGCP(DataCloudBase): os.rename(tmp_file, path) - progress.finish_target(name) + if not no_progress_bar: + progress.finish_target(name) Logger.debug('Downloading completed') diff --git a/dvc/cloud/local.py b/dvc/cloud/local.py index 714611778..5a606a7bb 100644 --- a/dvc/cloud/local.py +++ b/dvc/cloud/local.py @@ -47,7 +47,7 @@ class DataCloudLOCAL(DataCloudBase): copyfile(path, key.path) return path - def _pull_key(self, key, path): + def _pull_key(self, key, path, no_progress_bar=False): self._makedirs(path) tmp_file = self.tmp_file(path)