From 30304c3454edbcf14d46a807e94b80617d19ad93 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Tue, 2 Jul 2019 18:48:44 +0700 Subject: [PATCH 1/2] remote: always ignore ENOENT in ssh.rmdir() --- dvc/remote/ssh/connection.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dvc/remote/ssh/connection.py b/dvc/remote/ssh/connection.py index 5bdf58f97..76d14a90c 100644 --- a/dvc/remote/ssh/connection.py +++ b/dvc/remote/ssh/connection.py @@ -173,11 +173,13 @@ class SSHConnection: for root, dirs, files in self.walk(path, topdown=False): for fname in files: path = posixpath.join(root, fname) - self._remove_file(path) + with ignore_file_not_found(): + self._remove_file(path) for dname in dirs: path = posixpath.join(root, dname) - self._sftp.rmdir(dname) + with ignore_file_not_found(): + self._sftp.rmdir(dname) with ignore_file_not_found(): self._sftp.rmdir(path) From 835a04a1864752a1450e745f855cb388b634d914 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Tue, 2 Jul 2019 18:49:19 +0700 Subject: [PATCH 2/2] remote: fix ssh dir outputs --- dvc/path_info.py | 10 ++++++++++ dvc/remote/base.py | 12 +++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/dvc/path_info.py b/dvc/path_info.py index 99f6d915c..2d140f885 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -192,6 +192,16 @@ class URLInfo(object): scheme=self.scheme, netloc=self.netloc, path=self._path.parent ) + def relative_to(self, other): + if isinstance(other, str): + other_str = other + other = URLInfo(other) + if self.scheme != other.scheme or self.netloc != other.netloc: + raise ValueError( + "'{}' does not start with '{}'".format(self, other_str) + ) + return self._path.relative_to(other._path) + def isin(self, other): if isinstance(other, basestring): other = self.__class__(other) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index fa8eb2998..58cf47995 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -137,17 +137,19 @@ class RemoteBASE(object): dir_info = [] for root, dirs, files in self.walk(path_info): + root_info = path_info / root + if len(files) > LARGE_DIR_SIZE: msg = ( "Computing md5 for a large directory {}. " "This is only done once." ) - title = str(self.path_cls(root)) + title = str(root_info) logger.info(msg.format(title)) files = progress(files, name=title) for fname in files: - file_info = self.path_cls(root) / fname + file_info = root_info / fname relative_path = file_info.relative_to(path_info) dir_info.append( { @@ -207,7 +209,7 @@ class RemoteBASE(object): fobj = tempfile.NamedTemporaryFile(delete=False) path = fobj.name - to_info = self.path_cls(path) + to_info = PathInfo(path) self.cache.download([path_info], [to_info], no_progress_bar=True) try: @@ -226,7 +228,7 @@ class RemoteBASE(object): for info in d: # NOTE: here is a BUG, see comment to .as_posix() below - relative_path = self.path_cls.from_posix(info[self.PARAM_RELPATH]) + relative_path = PathInfo.from_posix(info[self.PARAM_RELPATH]) info[self.PARAM_RELPATH] = relative_path.fspath return d @@ -720,7 +722,7 @@ class RemoteBASE(object): def _remove_redundant_files(self, path_info, dir_info, force): existing_files = set( - self.path_cls(root) / fname + path_info / root / fname for root, _, files in self.walk(path_info) for fname in files )