diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 77bca4694..6813c55e0 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -30,6 +30,7 @@ from dvc.utils.compat import str from dvc.utils.compat import urlparse from dvc.utils.fs import move from dvc.utils.http import open_url +from pathlib import Path logger = logging.getLogger(__name__) @@ -549,15 +550,50 @@ class RemoteBASE(object): makedirs(to_info.parent, exist_ok=True, mode=dir_mode) + print(from_info) + if self.isdir(from_info): - # need to create dir here - # also pass the individual file name (file and to) - for file_info in self.walk_files(from_info): - self.single_file_download(file_info, to_info, name, no_progress_bar, file_mode) + makedirs(to_info, exist_ok=True, mode=dir_mode) + # TEMP limits to test + limit = 5 + current = 0 + # / TEMP limits to test + for file_from_info in self.walk_files(from_info): + # TEMP limits to test + current += 1 + if current == limit: + exit() + # / TEMP limits to test + + file_to_info = Path( + str(to_info) + + "/" + + str(file_from_info)[len(str(from_info)) + 1 :] + ) + + self.single_file_download( + file_from_info, + file_to_info, + name, + no_progress_bar, + file_mode, + dir_mode, + ) else: - self.single_file_download(from_info, to_info, name, no_progress_bar, file_mode) + self.single_file_download( + from_info, to_info, name, no_progress_bar, file_mode, dir_mode + ) + + def single_file_download( + self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode + ): + # Make sure full path exist + # print(to_info) + for parent in reversed(to_info.parents): + if not parent.exists(): + # print('making dir: {}'.format(parent)) + makedirs(parent, exist_ok=True, mode=dir_mode) - def single_file_download(self, from_info, to_info, name, no_progress_bar, file_mode): logger.debug("Downloading '{}' to '{}'".format(from_info, to_info)) name = name or to_info.name @@ -568,7 +604,7 @@ class RemoteBASE(object): from_info, tmp_file, name=name, no_progress_bar=no_progress_bar ) except Exception: - msg = "failed to download '{}' to '{}'" + msg = "failed to doooooownload '{}' to '{}'" logger.exception(msg.format(from_info, to_info)) return 1 # 1 fail @@ -576,7 +612,6 @@ class RemoteBASE(object): return 0 - def open(self, path_info, mode="r", encoding=None): if hasattr(self, "_generate_download_url"): get_url = partial(self._generate_download_url, path_info)