From 3814e0da4e5281a0773156e6687cdc62c3e0ec1a Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 11 Aug 2018 02:34:52 +0300 Subject: [PATCH] remote: add changed() method Signed-off-by: Ruslan Kuprieiev --- dvc/output/gs.py | 2 +- dvc/output/hdfs.py | 2 +- dvc/output/local.py | 2 +- dvc/output/s3.py | 2 +- dvc/output/ssh.py | 2 +- dvc/remote/azure.py | 3 +++ dvc/remote/base.py | 3 +++ dvc/remote/gs.py | 3 +++ dvc/remote/hdfs.py | 3 +++ dvc/remote/local.py | 3 +++ dvc/remote/s3.py | 3 +++ dvc/remote/ssh.py | 3 +++ 12 files changed, 26 insertions(+), 5 deletions(-) diff --git a/dvc/output/gs.py b/dvc/output/gs.py index 82e53b154..e85330999 100644 --- a/dvc/output/gs.py +++ b/dvc/output/gs.py @@ -30,7 +30,7 @@ class OutputGS(DependencyGS): return True if self.use_cache \ - and self.info != self.project.cache.gs.save_info(self.path_info): + and self.project.cache.gs.changed(self.path_info, self.info): return True return False diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index ec81f23a0..6091bcde4 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -30,7 +30,7 @@ class OutputHDFS(DependencyHDFS): return True if self.use_cache \ - and self.info != self.project.cache.hdfs.save_info(self.path_info): + and self.project.cache.hdfs.save_info(self.path_info, self.info): return True return False diff --git a/dvc/output/local.py b/dvc/output/local.py index 49fec124b..0149fd5fb 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -62,7 +62,7 @@ class OutputLOCAL(DependencyLOCAL): if not self.use_cache: return super(OutputLOCAL, self).changed() - return self.info != self.project.cache.local.save_info(self.path_info) + return self.project.cache.local.changed(self.path_info, self.info) def checkout(self): if not self.use_cache: diff --git a/dvc/output/s3.py b/dvc/output/s3.py index 620ea7d7c..e3e2babb2 100644 --- a/dvc/output/s3.py +++ b/dvc/output/s3.py @@ -30,7 +30,7 @@ class OutputS3(DependencyS3): return True if self.use_cache \ - and self.info != self.project.cache.s3.save_info(self.path_info): + and self.project.cache.s3.changed(self.path_info, self.info): return True return False diff --git a/dvc/output/ssh.py b/dvc/output/ssh.py index 8fa04c2ab..4b4f6c298 100644 --- a/dvc/output/ssh.py +++ b/dvc/output/ssh.py @@ -30,7 +30,7 @@ class OutputSSH(DependencySSH): return True if self.use_cache \ - and self.info != self.project.cache.ssh.save_info(self.path_info): + and self.project.cache.ssh.changed(self.path_info, self.info): return True return False diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 8c23ee408..4c6692a31 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -92,6 +92,9 @@ class RemoteAzure(RemoteBase): # # return {self.PARAM_ETAG: etag} # +# def changed(self, path_info, checksum_info): +# return checksum_info != self.save_info(path_info) +# # def _copy_blob(self, to_bucket, to_key, from_bucket, from_key): # source = self.blob_service.make_blob_url(from_bucket, from_key) # diff --git a/dvc/remote/base.py b/dvc/remote/base.py index d9f56dc28..cadf91de4 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -81,6 +81,9 @@ class RemoteBase(object): def save_info(self, path_info): raise NotImplementedError + def changed(self, path_info, checksum_info): + raise NotImplementedError + def save(self, path_info): raise NotImplementedError diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index b5a85cc33..c165056bb 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -59,6 +59,9 @@ class RemoteGS(RemoteBase): return {self.PARAM_ETAG: self.get_etag(path_info['bucket'], path_info['key'])} + def changed(self, path_info, checksum_info): + return checksum_info != self.save_info(path_info) + def _copy(self, from_info, to_info, gs=None): gs = gs if gs else self.gs diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 306065f8c..7b1e01e61 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -72,6 +72,9 @@ class RemoteHDFS(RemoteBase): return {self.PARAM_CHECKSUM: self.checksum(path_info)} + def changed(self, path_info, checksum_info): + return checksum_info != self.save_info(path_info) + def save(self, path_info): if path_info['scheme'] != 'hdfs': raise NotImplementedError diff --git a/dvc/remote/local.py b/dvc/remote/local.py index c928205cd..45bb5a9a9 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -333,6 +333,9 @@ class RemoteLOCAL(RemoteBase): return {self.PARAM_MD5: self.state.update(path_info['path'])} + def changed(self, path_info, checksum_info): + return checksum_info != self.save_info(path_info) + def remove(self, path_info): if path_info['scheme'] != 'local': raise NotImplementedError diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 14a78836c..b7c2628f4 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -93,6 +93,9 @@ class RemoteS3(RemoteBase): return {self.PARAM_ETAG: self.get_etag(path_info['bucket'], path_info['key'])} + def changed(self, path_info, checksum_info): + return checksum_info != self.save_info(path_info) + def _copy(self, from_info, to_info, s3=None): s3 = s3 if s3 else self.s3 diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index c54a312cb..ebc47586f 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -148,6 +148,9 @@ class RemoteSSH(RemoteBase): return {self.PARAM_MD5: self.md5(path_info)} + def changed(self, path_info, checksum_info): + return checksum_info != self.save_info(path_info) + def save(self, path_info): if path_info['scheme'] != 'ssh': raise NotImplementedError