mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-17 07:18:26 +01:00
cloud: don't print progress bar when collecting dir
Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
+10
-3
@@ -130,7 +130,7 @@ class DataCloudAWS(DataCloudBase):
|
||||
"""
|
||||
return fname + '.download'
|
||||
|
||||
def _pull_key(self, key, fname):
|
||||
def _pull_key(self, key, fname, no_progress_bar=False):
|
||||
Logger.debug("Pulling key '{}' from bucket '{}' to file '{}'".format(key.name,
|
||||
key.bucket.name,
|
||||
fname))
|
||||
@@ -147,17 +147,24 @@ class DataCloudAWS(DataCloudBase):
|
||||
key.name,
|
||||
fname))
|
||||
|
||||
if no_progress_bar:
|
||||
cb = None
|
||||
else:
|
||||
cb = create_cb(name)
|
||||
|
||||
res_h = ResumableDownloadHandler(tracker_file_name=self._download_tracker(tmp_file),
|
||||
num_retries=10)
|
||||
try:
|
||||
key.get_contents_to_filename(tmp_file, cb=create_cb(name), res_download_handler=res_h)
|
||||
key.get_contents_to_filename(tmp_file, cb=cb, res_download_handler=res_h)
|
||||
except Exception as exc:
|
||||
Logger.error('Failed to download "{}": {}'.format(key.name, exc))
|
||||
return None
|
||||
|
||||
os.rename(tmp_file, fname)
|
||||
|
||||
progress.finish_target(name)
|
||||
if not no_progress_bar:
|
||||
progress.finish_target(name)
|
||||
|
||||
Logger.debug('Downloading completed')
|
||||
|
||||
return fname
|
||||
|
||||
+2
-2
@@ -109,7 +109,7 @@ class DataCloudBase(object):
|
||||
Logger.debug("File '{}' does not exist in the cloud".format(path))
|
||||
return ret
|
||||
tmp = os.path.join(tempfile.mkdtemp(), os.path.basename(path))
|
||||
self._pull_key(key, tmp)
|
||||
self._pull_key(key, tmp, no_progress_bar=True)
|
||||
dir_path = tmp
|
||||
|
||||
for relpath, md5 in Output.get_dir_cache(dir_path).items():
|
||||
@@ -138,7 +138,7 @@ class DataCloudBase(object):
|
||||
if e.errno != os.errno.EEXIST:
|
||||
raise
|
||||
|
||||
def _pull_key(self, key, path):
|
||||
def _pull_key(self, key, path, no_progress_bar=False):
|
||||
""" Cloud-specific method of pulling keys """
|
||||
pass
|
||||
|
||||
|
||||
+7
-5
@@ -45,7 +45,7 @@ class DataCloudGCP(DataCloudBase):
|
||||
|
||||
return False
|
||||
|
||||
def _pull_key(self, key, path):
|
||||
def _pull_key(self, key, path, no_progress_bar=False):
|
||||
self._makedirs(path)
|
||||
|
||||
name = os.path.relpath(path, self._cloud_settings.cache.cache_dir)
|
||||
@@ -57,9 +57,10 @@ class DataCloudGCP(DataCloudBase):
|
||||
|
||||
Logger.debug('Downloading cache file from gc "{}/{}"'.format(key.bucket.name, key.name))
|
||||
|
||||
# percent_cb is not available for download_to_filename, so
|
||||
# lets at least update progress at keypoints(start, finish)
|
||||
progress.update_target(name, 0, None)
|
||||
if not no_progress_bar:
|
||||
# percent_cb is not available for download_to_filename, so
|
||||
# lets at least update progress at keypoints(start, finish)
|
||||
progress.update_target(name, 0, None)
|
||||
|
||||
try:
|
||||
key.download_to_filename(tmp_file)
|
||||
@@ -69,7 +70,8 @@ class DataCloudGCP(DataCloudBase):
|
||||
|
||||
os.rename(tmp_file, path)
|
||||
|
||||
progress.finish_target(name)
|
||||
if not no_progress_bar:
|
||||
progress.finish_target(name)
|
||||
|
||||
Logger.debug('Downloading completed')
|
||||
|
||||
|
||||
+1
-1
@@ -47,7 +47,7 @@ class DataCloudLOCAL(DataCloudBase):
|
||||
copyfile(path, key.path)
|
||||
return path
|
||||
|
||||
def _pull_key(self, key, path):
|
||||
def _pull_key(self, key, path, no_progress_bar=False):
|
||||
self._makedirs(path)
|
||||
|
||||
tmp_file = self.tmp_file(path)
|
||||
|
||||
Reference in New Issue
Block a user