cloud: don't print progress bar when collecting dir

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-03-04 15:11:21 +02:00
parent dbc6e1ac88
commit bdd51c854b
4 changed files with 20 additions and 11 deletions
+10 -3
View File
@@ -130,7 +130,7 @@ class DataCloudAWS(DataCloudBase):
"""
return fname + '.download'
def _pull_key(self, key, fname):
def _pull_key(self, key, fname, no_progress_bar=False):
Logger.debug("Pulling key '{}' from bucket '{}' to file '{}'".format(key.name,
key.bucket.name,
fname))
@@ -147,17 +147,24 @@ class DataCloudAWS(DataCloudBase):
key.name,
fname))
if no_progress_bar:
cb = None
else:
cb = create_cb(name)
res_h = ResumableDownloadHandler(tracker_file_name=self._download_tracker(tmp_file),
num_retries=10)
try:
key.get_contents_to_filename(tmp_file, cb=create_cb(name), res_download_handler=res_h)
key.get_contents_to_filename(tmp_file, cb=cb, res_download_handler=res_h)
except Exception as exc:
Logger.error('Failed to download "{}": {}'.format(key.name, exc))
return None
os.rename(tmp_file, fname)
progress.finish_target(name)
if not no_progress_bar:
progress.finish_target(name)
Logger.debug('Downloading completed')
return fname
+2 -2
View File
@@ -109,7 +109,7 @@ class DataCloudBase(object):
Logger.debug("File '{}' does not exist in the cloud".format(path))
return ret
tmp = os.path.join(tempfile.mkdtemp(), os.path.basename(path))
self._pull_key(key, tmp)
self._pull_key(key, tmp, no_progress_bar=True)
dir_path = tmp
for relpath, md5 in Output.get_dir_cache(dir_path).items():
@@ -138,7 +138,7 @@ class DataCloudBase(object):
if e.errno != os.errno.EEXIST:
raise
def _pull_key(self, key, path):
def _pull_key(self, key, path, no_progress_bar=False):
""" Cloud-specific method of pulling keys """
pass
+7 -5
View File
@@ -45,7 +45,7 @@ class DataCloudGCP(DataCloudBase):
return False
def _pull_key(self, key, path):
def _pull_key(self, key, path, no_progress_bar=False):
self._makedirs(path)
name = os.path.relpath(path, self._cloud_settings.cache.cache_dir)
@@ -57,9 +57,10 @@ class DataCloudGCP(DataCloudBase):
Logger.debug('Downloading cache file from gc "{}/{}"'.format(key.bucket.name, key.name))
# percent_cb is not available for download_to_filename, so
# lets at least update progress at keypoints(start, finish)
progress.update_target(name, 0, None)
if not no_progress_bar:
# percent_cb is not available for download_to_filename, so
# lets at least update progress at keypoints(start, finish)
progress.update_target(name, 0, None)
try:
key.download_to_filename(tmp_file)
@@ -69,7 +70,8 @@ class DataCloudGCP(DataCloudBase):
os.rename(tmp_file, path)
progress.finish_target(name)
if not no_progress_bar:
progress.finish_target(name)
Logger.debug('Downloading completed')
+1 -1
View File
@@ -47,7 +47,7 @@ class DataCloudLOCAL(DataCloudBase):
copyfile(path, key.path)
return path
def _pull_key(self, key, path):
def _pull_key(self, key, path, no_progress_bar=False):
self._makedirs(path)
tmp_file = self.tmp_file(path)