From 10dd5351760c23e2e72bcddb30132e4fe8496c91 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 11 Aug 2018 05:25:25 +0300 Subject: [PATCH 1/6] travis: cache pip Signed-off-by: Ruslan Kuprieiev --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 33de22eec..acf7b15d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: python +cache: pip sudo: required dist: trusty group: deprecated-2017Q4 From b2615548189d47bbef64e296957efa12d9ae0678 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 11 Aug 2018 01:01:15 +0300 Subject: [PATCH 2/6] dvc: remote: format "No Remote repository" error Fixes #988 Signed-off-by: Ruslan Kuprieiev --- dvc/data_cloud.py | 18 ++++++++++++------ dvc/project.py | 3 ++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index ccf83de02..771c6a3e9 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -83,15 +83,18 @@ class DataCloud(object): cloud = cloud_type(self.project, cloud_config) return cloud - def _get_cloud(self, remote): + def _get_cloud(self, remote, cmd): if remote: return self._init_remote(remote) if self._cloud: return self._cloud - msg = "No remote repository specified. Setup default repository " \ - "with 'dvc config core.remote ' or use '-r '." + msg = "No remote repository specified. Setup default " \ + "repository with\n"\ + " dvc config core.remote \n" \ + "or use:\n" \ + " dvc {} -r ".format(cmd) raise ConfigError(msg) def push(self, targets, jobs=1, remote=None, show_checksums=False): @@ -100,7 +103,8 @@ class DataCloud(object): """ return self.project.cache.local.push(targets, jobs=jobs, - remote=self._get_cloud(remote), + remote=self._get_cloud(remote, + 'push'), show_checksums=show_checksums) def pull(self, targets, jobs=1, remote=None, show_checksums=False): @@ -109,14 +113,16 @@ class DataCloud(object): """ return self.project.cache.local.pull(targets, jobs=jobs, - remote=self._get_cloud(remote), + remote=self._get_cloud(remote, + 'pull'), show_checksums=show_checksums) def status(self, targets, jobs=1, remote=None, show_checksums=False): """ Check status of data items in a cloud-agnostic way. """ + cloud = self._get_cloud(remote, 'status') return self.project.cache.local.status(targets, jobs=jobs, - remote=self._get_cloud(remote), + remote=cloud, show_checksums=show_checksums) diff --git a/dvc/project.py b/dvc/project.py index 02d833df0..590f9a227 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -383,7 +383,8 @@ class Project(object): self._do_gc('azure', self.cache.azure.gc, clist) if cloud: - self._do_gc('remote', self.cloud._get_cloud(remote).gc, clist) + self._do_gc('remote', self.cloud._get_cloud(remote, + 'gc -c').gc, clist) def push(self, target=None, From 6776f6e21c3212a478556eeaa04c52013d092910 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 11 Aug 2018 02:23:44 +0300 Subject: [PATCH 3/6] dvc: print more info on checkout Fixes #987 Signed-off-by: Ruslan Kuprieiev --- dvc/output/local.py | 4 ++-- dvc/remote/local.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dvc/output/local.py b/dvc/output/local.py index 4f978418e..49fec124b 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -69,8 +69,8 @@ class OutputLOCAL(DependencyLOCAL): return if not self.changed(): - msg = u'Data file \'{}\' didn\'t change, skipping checkout.' - self.project.logger.debug(msg.format(self.rel_path)) + msg = u'Data file \'{}\' didn\'t change.' + self.project.logger.info(msg.format(self.rel_path)) return self.project.cache.local.checkout(self.path_info, self.info) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 392bfa44c..fb7ebe93d 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -104,6 +104,10 @@ class RemoteLOCAL(RemoteBase): while i > 0: try: self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path) + msg = "Created '{}': {} -> {}" + Logger.info(msg.format(self.cache_types[0], + os.path.relpath(cache), + os.path.relpath(path))) return except Exception as exc: msg = 'Cache type \'{}\' is not supported: {}' @@ -238,7 +242,7 @@ class RemoteLOCAL(RemoteBase): return if os.path.exists(path): - msg = u'Data \'{}\' exists. Removing before checkout' + msg = u'Data \'{}\' exists. Removing before checkout.' Logger.warn(msg.format(os.path.relpath(path))) remove(path) From 292702473bb80c93c0faae9118e51c3737c3f6bc Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 11 Aug 2018 02:27:04 +0300 Subject: [PATCH 4/6] remote: local: rename changed() -> changed_cache() Signed-off-by: Ruslan Kuprieiev --- dvc/remote/local.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index fb7ebe93d..c928205cd 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -82,7 +82,7 @@ class RemoteLOCAL(RemoteBase): relpath = os.path.relpath(path, self.cache_dir) return os.path.dirname(relpath) + os.path.basename(relpath) - def changed(self, md5): + def changed_cache(self, md5): cache = self.get(md5) if self.state.changed(cache, md5=md5): if os.path.exists(cache): @@ -176,7 +176,7 @@ class RemoteLOCAL(RemoteBase): dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH)) md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX - if self.changed(md5): + if self.changed_cache(md5): self.dump_dir_cache(md5, dir_info) return (md5, dir_info) @@ -235,7 +235,7 @@ class RemoteLOCAL(RemoteBase): Logger.warn(msg.format(os.path.relpath(path))) return - if self.changed(md5): + if self.changed_cache(md5): msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.' Logger.warn(msg.format(md5, os.path.relpath(path))) remove(path) @@ -281,7 +281,7 @@ class RemoteLOCAL(RemoteBase): cache = self.get(md5) - if self.changed(md5): + if self.changed_cache(md5): self._move(path, cache) else: remove(path) @@ -301,7 +301,7 @@ class RemoteLOCAL(RemoteBase): p = os.path.join(path, relpath) c = self.get(m) - if self.changed(m): + if self.changed_cache(m): self._move(p, c) else: remove(p) @@ -502,7 +502,7 @@ class RemoteLOCAL(RemoteBase): progress.update_target(title, 90, 100) - local_exists = [not self.changed(md5) for md5 in md5s] + local_exists = [not self.changed_cache(md5) for md5 in md5s] progress.finish_target(title) @@ -529,7 +529,7 @@ class RemoteLOCAL(RemoteBase): names = [] # NOTE: filter files that are not corrupted for md5, name in grouped: - if self.changed(md5): + if self.changed_cache(md5): md5s.append(md5) names.append(name) @@ -598,7 +598,7 @@ class RemoteLOCAL(RemoteBase): # NOTE: verifying that our cache is not corrupted def func(info): - return not self.changed(info[self.PARAM_MD5]) + return not self.changed_cache(info[self.PARAM_MD5]) checksum_infos = list(filter(func, checksum_infos)) progress.update_target(title, 20, 100) From 3814e0da4e5281a0773156e6687cdc62c3e0ec1a Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 11 Aug 2018 02:34:52 +0300 Subject: [PATCH 5/6] remote: add changed() method Signed-off-by: Ruslan Kuprieiev --- dvc/output/gs.py | 2 +- dvc/output/hdfs.py | 2 +- dvc/output/local.py | 2 +- dvc/output/s3.py | 2 +- dvc/output/ssh.py | 2 +- dvc/remote/azure.py | 3 +++ dvc/remote/base.py | 3 +++ dvc/remote/gs.py | 3 +++ dvc/remote/hdfs.py | 3 +++ dvc/remote/local.py | 3 +++ dvc/remote/s3.py | 3 +++ dvc/remote/ssh.py | 3 +++ 12 files changed, 26 insertions(+), 5 deletions(-) diff --git a/dvc/output/gs.py b/dvc/output/gs.py index 82e53b154..e85330999 100644 --- a/dvc/output/gs.py +++ b/dvc/output/gs.py @@ -30,7 +30,7 @@ class OutputGS(DependencyGS): return True if self.use_cache \ - and self.info != self.project.cache.gs.save_info(self.path_info): + and self.project.cache.gs.changed(self.path_info, self.info): return True return False diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index ec81f23a0..6091bcde4 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -30,7 +30,7 @@ class OutputHDFS(DependencyHDFS): return True if self.use_cache \ - and self.info != self.project.cache.hdfs.save_info(self.path_info): + and self.project.cache.hdfs.save_info(self.path_info, self.info): return True return False diff --git a/dvc/output/local.py b/dvc/output/local.py index 49fec124b..0149fd5fb 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -62,7 +62,7 @@ class OutputLOCAL(DependencyLOCAL): if not self.use_cache: return super(OutputLOCAL, self).changed() - return self.info != self.project.cache.local.save_info(self.path_info) + return self.project.cache.local.changed(self.path_info, self.info) def checkout(self): if not self.use_cache: diff --git a/dvc/output/s3.py b/dvc/output/s3.py index 620ea7d7c..e3e2babb2 100644 --- a/dvc/output/s3.py +++ b/dvc/output/s3.py @@ -30,7 +30,7 @@ class OutputS3(DependencyS3): return True if self.use_cache \ - and self.info != self.project.cache.s3.save_info(self.path_info): + and self.project.cache.s3.changed(self.path_info, self.info): return True return False diff --git a/dvc/output/ssh.py b/dvc/output/ssh.py index 8fa04c2ab..4b4f6c298 100644 --- a/dvc/output/ssh.py +++ b/dvc/output/ssh.py @@ -30,7 +30,7 @@ class OutputSSH(DependencySSH): return True if self.use_cache \ - and self.info != self.project.cache.ssh.save_info(self.path_info): + and self.project.cache.ssh.changed(self.path_info, self.info): return True return False diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 8c23ee408..4c6692a31 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -92,6 +92,9 @@ class RemoteAzure(RemoteBase): # # return {self.PARAM_ETAG: etag} # +# def changed(self, path_info, checksum_info): +# return checksum_info != self.save_info(path_info) +# # def _copy_blob(self, to_bucket, to_key, from_bucket, from_key): # source = self.blob_service.make_blob_url(from_bucket, from_key) # diff --git a/dvc/remote/base.py b/dvc/remote/base.py index d9f56dc28..cadf91de4 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -81,6 +81,9 @@ class RemoteBase(object): def save_info(self, path_info): raise NotImplementedError + def changed(self, path_info, checksum_info): + raise NotImplementedError + def save(self, path_info): raise NotImplementedError diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index b5a85cc33..c165056bb 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -59,6 +59,9 @@ class RemoteGS(RemoteBase): return {self.PARAM_ETAG: self.get_etag(path_info['bucket'], path_info['key'])} + def changed(self, path_info, checksum_info): + return checksum_info != self.save_info(path_info) + def _copy(self, from_info, to_info, gs=None): gs = gs if gs else self.gs diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 306065f8c..7b1e01e61 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -72,6 +72,9 @@ class RemoteHDFS(RemoteBase): return {self.PARAM_CHECKSUM: self.checksum(path_info)} + def changed(self, path_info, checksum_info): + return checksum_info != self.save_info(path_info) + def save(self, path_info): if path_info['scheme'] != 'hdfs': raise NotImplementedError diff --git a/dvc/remote/local.py b/dvc/remote/local.py index c928205cd..45bb5a9a9 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -333,6 +333,9 @@ class RemoteLOCAL(RemoteBase): return {self.PARAM_MD5: self.state.update(path_info['path'])} + def changed(self, path_info, checksum_info): + return checksum_info != self.save_info(path_info) + def remove(self, path_info): if path_info['scheme'] != 'local': raise NotImplementedError diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 14a78836c..b7c2628f4 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -93,6 +93,9 @@ class RemoteS3(RemoteBase): return {self.PARAM_ETAG: self.get_etag(path_info['bucket'], path_info['key'])} + def changed(self, path_info, checksum_info): + return checksum_info != self.save_info(path_info) + def _copy(self, from_info, to_info, s3=None): s3 = s3 if s3 else self.s3 diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index c54a312cb..ebc47586f 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -148,6 +148,9 @@ class RemoteSSH(RemoteBase): return {self.PARAM_MD5: self.md5(path_info)} + def changed(self, path_info, checksum_info): + return checksum_info != self.save_info(path_info) + def save(self, path_info): if path_info['scheme'] != 'ssh': raise NotImplementedError From 3ec0a5c4ae852a2c1fdaad97f32e20189bccc2d3 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 11 Aug 2018 05:15:29 +0300 Subject: [PATCH 6/6] remote: add additional checks and msgs for checkout Signed-off-by: Ruslan Kuprieiev --- dvc/output/hdfs.py | 2 +- dvc/output/local.py | 5 ----- dvc/remote/gs.py | 47 ++++++++++++++++++++++++++++++++++++++++ dvc/remote/hdfs.py | 49 ++++++++++++++++++++++++++++++++++++++++++ dvc/remote/local.py | 15 +++++++++++++ dvc/remote/s3.py | 46 +++++++++++++++++++++++++++++++++++++++ dvc/remote/ssh.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 210 insertions(+), 6 deletions(-) diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index 6091bcde4..19cca3585 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -30,7 +30,7 @@ class OutputHDFS(DependencyHDFS): return True if self.use_cache \ - and self.project.cache.hdfs.save_info(self.path_info, self.info): + and self.project.cache.hdfs.changed(self.path_info, self.info): return True return False diff --git a/dvc/output/local.py b/dvc/output/local.py index 0149fd5fb..f23958530 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -68,11 +68,6 @@ class OutputLOCAL(DependencyLOCAL): if not self.use_cache: return - if not self.changed(): - msg = u'Data file \'{}\' didn\'t change.' - self.project.logger.info(msg.format(self.rel_path)) - return - self.project.cache.local.checkout(self.path_info, self.info) def _verify_metric(self): diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index c165056bb..67426aa7e 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -59,7 +59,35 @@ class RemoteGS(RemoteBase): return {self.PARAM_ETAG: self.get_etag(path_info['bucket'], path_info['key'])} + @staticmethod + def to_string(path_info): + return "{}://{}/{}".format(path_info['scheme'], + path_info['bucket'], + path_info['key']) + + def changed_cache(self, etag): + key = posixpath.join(self.prefix, etag[0:2], etag[2:]) + cache = {'scheme': 'gs', 'bucket': self.bucket, 'key': key} + + if {self.PARAM_ETAG: etag} != self.save_info(cache): + if self.exists([cache])[0]: + msg = 'Corrupted cache file {}' + Logger.warn(msg.format(self.to_string(cache))) + self.remove(cache) + return True + return False + def changed(self, path_info, checksum_info): + if not self.exists([path_info])[0]: + return True + + etag = checksum_info.get(self.PARAM_ETAG, None) + if etag is None: + return True + + if self.changed_cache(etag): + return True + return checksum_info != self.save_info(path_info) def _copy(self, from_info, to_info, gs=None): @@ -95,6 +123,25 @@ class RemoteGS(RemoteBase): if not etag: return + if not self.changed(path_info, checksum_info): + msg = "Data '{}' didn't change." + Logger.info(msg.format(self.to_string(path_info))) + return + + if self.changed_cache(etag): + msg = "Cache '{}' not found. File '{}' won't be created." + Logger.warn(msg.format(etag, self.to_string(path_info))) + return + + if self.exists([path_info])[0]: + msg = "Data '{}' exists. Removing before checkout." + Logger.warn(msg.format(self.to_string(path_info))) + self.remove(path_info) + return + + msg = "Checking out '{}' with cache '{}'." + Logger.info(msg.format(self.to_string(path_info), etag)) + key = posixpath.join(self.prefix, etag[0:2], etag[2:]) from_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key} diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 7b1e01e61..306806b8a 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -72,7 +72,37 @@ class RemoteHDFS(RemoteBase): return {self.PARAM_CHECKSUM: self.checksum(path_info)} + @staticmethod + def to_string(path_info): + return "{}://{}".format(path_info['scheme'], + path_info['url']) + + def changed_cache(self, checksum): + cache = {} + cache['scheme'] = 'hdfs' + cache['user'] = self.user + cache['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:]) + + if {self.PARAM_CHECKSUM: checksum} != self.save_info(cache): + if self.exists([cache])[0]: + msg = 'Corrupted cache file {}' + Logger.warn(msg.format(self.to_string(cache))) + self.remove(cache) + return True + + return False + def changed(self, path_info, checksum_info): + if not self.exists([path_info])[0]: + return True + + checksum = checksum_info.get(self.PARAM_CHECKSUM, None) + if checksum is None: + return True + + if self.changed_cache(self, checksum): + return True + return checksum_info != self.save_info(path_info) def save(self, path_info): @@ -99,6 +129,25 @@ class RemoteHDFS(RemoteBase): if not checksum: return + if not self.changed(path_info, checksum_info): + msg = "Data '{}' didn't change." + Logger.info(msg.format(self.to_string(path_info))) + return + + if self.changed_cache(checksum): + msg = "Cache '{}' not found. File '{}' won't be created." + Logger.warn(msg.format(checksum, self.to_string(path_info))) + return + + if self.exists([path_info])[0]: + msg = "Data '{}' exists. Removing before checkout." + Logger.warn(msg.format(self.to_string(path_info))) + self.remove(path_info) + return + + msg = "Checking out '{}' with cache '{}'." + Logger.info(msg.format(self.to_string(path_info), checksum)) + src = path_info.copy() src['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:]) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 45bb5a9a9..5cde2a3ea 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -235,6 +235,11 @@ class RemoteLOCAL(RemoteBase): Logger.warn(msg.format(os.path.relpath(path))) return + if not self.changed(path_info, checksum_info): + msg = "Data '{}' didn't change." + Logger.info(msg.format(os.path.relpath(path))) + return + if self.changed_cache(md5): msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.' Logger.warn(msg.format(md5, os.path.relpath(path))) @@ -334,6 +339,16 @@ class RemoteLOCAL(RemoteBase): return {self.PARAM_MD5: self.state.update(path_info['path'])} def changed(self, path_info, checksum_info): + if not self.exists([path_info])[0]: + return True + + md5 = checksum_info.get(self.PARAM_MD5, None) + if md5 is None: + return True + + if self.changed_cache(md5): + return True + return checksum_info != self.save_info(path_info) def remove(self, path_info): diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index b7c2628f4..5e07abc88 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -94,6 +94,16 @@ class RemoteS3(RemoteBase): path_info['key'])} def changed(self, path_info, checksum_info): + if not self.exists([path_info])[0]: + return True + + etag = checksum_info.get(self.PARAM_ETAG, None) + if etag is None: + return True + + if self.changed_cache(etag): + return True + return checksum_info != self.save_info(path_info) def _copy(self, from_info, to_info, s3=None): @@ -115,6 +125,23 @@ class RemoteS3(RemoteBase): return {self.PARAM_ETAG: etag} + @staticmethod + def to_string(path_info): + return "s3://{}/{}".format(path_info['bucket'], path_info['key']) + + def changed_cache(self, etag): + key = posixpath.join(self.prefix, etag[0:2], etag[2:]) + cache = {'scheme': 's3', 'bucket': self.bucket, 'key': key} + + if {self.PARAM_ETAG: etag} != self.save_info(cache): + if self.exists([cache])[0]: + msg = 'Corrupted cache file {}' + Logger.warn(msg.format(self.to_string(cache))) + self.remove(cache) + return True + + return False + def checkout(self, path_info, checksum_info): if path_info['scheme'] != 's3': raise NotImplementedError @@ -123,6 +150,25 @@ class RemoteS3(RemoteBase): if not etag: return + if not self.changed(path_info, checksum_info): + msg = "Data '{}' didn't change." + Logger.info(msg.format(self.to_string(path_info))) + return + + if self.changed_cache(etag): + msg = "Cache '{}' not found. File '{}' won't be created." + Logger.warn(msg.format(etag, self.to_string(path_info))) + return + + if self.exists([path_info])[0]: + msg = "Data '{}' exists. Removing before checkout." + Logger.warn(msg.format(self.to_string(path_info))) + self.remove(path_info) + return + + msg = "Checking out '{}' with cache '{}'." + Logger.info(msg.format(self.to_string(path_info), etag)) + key = posixpath.join(self.prefix, etag[0:2], etag[2:]) from_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key} diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index ebc47586f..10236918d 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -148,7 +148,40 @@ class RemoteSSH(RemoteBase): return {self.PARAM_MD5: self.md5(path_info)} + @staticmethod + def to_string(path_info): + return "{}://{}@{}:{}".format(path_info['scheme'], + path_info['user'], + path_info['host'], + path_info['path']) + + def changed_cache(self, md5): + cache = {} + cache['scheme'] = 'ssh' + cache['host'] = self.host + cache['user'] = self.user + cache['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:]) + + if {self.PARAM_MD5: md5} != self.save_info(cache): + if self.exists([cache])[0]: + msg = 'Corrupted cache file {}' + Logger.warn(msg.format(self.to_string(cache))) + self.remove(cache) + return True + + return False + def changed(self, path_info, checksum_info): + if not self.exists([path_info])[0]: + return True + + md5 = checksum_info.get(self.PARAM_MD5, None) + if md5 is None: + return True + + if self.changed_code(md5): + return True + return checksum_info != self.save_info(path_info) def save(self, path_info): @@ -171,6 +204,25 @@ class RemoteSSH(RemoteBase): if not md5: return + if not self.changed(path_info, checksum_info): + msg = "Data '{}' didn't change." + Logger.info(msg.format(self.to_string(path_info))) + return + + if self.changed_cache(md5): + msg = "Cache '{}' not found. File '{}' won't be created." + Logger.warn(msg.format(md5, self.to_string(path_info))) + return + + if self.exists([path_info])[0]: + msg = "Data '{}' exists. Removing before checkout." + Logger.warn(msg.format(self.to_string(path_info))) + self.remove(path_info) + return + + msg = "Checking out '{}' with cache '{}'." + Logger.info(msg.format(self.to_string(path_info), md5)) + src = path_info.copy() src['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])