Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1638 from efiop/master

Browse Source 
ssh: external-outs: check if cache exists before copying
This commit is contained in:
Ruslan Kuprieiev
2019-02-19 21:43:13 +02:00
committed by GitHub
parent 1b703b8bc4 e5107b1da3
commit 2cf4e99d75
6 changed files with 45 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/remote/base.py
+19 -1
View File
@@ -173,7 +173,21 @@ class RemoteBase(object):
return False
def save(self, path_info, checksum_info):
raise RemoteActionNotImplemented("save", self.scheme)
if path_info["scheme"] != self.scheme:
raise RemoteActionNotImplemented(
"save {} -> {}".format(path_info["scheme"], self.scheme),
self.scheme,
)
checksum = checksum_info[self.PARAM_CHECKSUM]
if not self.changed_cache(checksum):
return
to_info = self.checksum_to_path_info(checksum)
logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
self.copy(path_info, to_info)
def download(
self,
@@ -265,6 +279,10 @@ class RemoteBase(object):
def changed_cache(self, checksum):
cache = self.checksum_to_path_info(checksum)
expected = {self.PARAM_CHECKSUM: checksum}
if not self.exists(cache):
return True
actual = self.save_info(cache)
logger.debug(
dvc/remote/gs.py
-12
View File
@@ -87,18 +87,6 @@ class RemoteGS(RemoteBase):
blob, self.gs.bucket(to_info["bucket"]), new_name=to_info["path"]
)
def save(self, path_info):
if path_info["scheme"] != "gs":
raise NotImplementedError
md5 = self.get_md5(path_info["bucket"], path_info["path"])
path = self.checksum_to_path(md5)
to_info = {"scheme": "gs", "bucket": self.bucket, "path": path}
self.copy(path_info, to_info)
return {self.PARAM_CHECKSUM: md5}
def remove(self, path_info):
if path_info["scheme"] != "gs":
raise NotImplementedError
dvc/remote/hdfs.py
-12
View File
@@ -88,18 +88,6 @@ class RemoteHDFS(RemoteBase):
return {self.PARAM_CHECKSUM: self.checksum(path_info)}
def save(self, path_info, checksum_info):
if path_info["scheme"] != "hdfs":
raise NotImplementedError
assert path_info.get("path")
checksum = checksum_info[self.PARAM_CHECKSUM]
dest = path_info.copy()
dest["path"] = self.checksum_to_path(checksum)
self.copy(path_info, dest)
def remove(self, path_info):
if path_info["scheme"] != "hdfs":
raise NotImplementedError
dvc/remote/s3.py
-10
View File
@@ -109,16 +109,6 @@ class RemoteS3(RemoteBase):
source = {"Bucket": from_info["bucket"], "Key": from_info["path"]}
self.s3.copy(source, to_info["bucket"], to_info["path"])
def save(self, path_info, checksum_info):
if path_info["scheme"] != "s3":
raise NotImplementedError
etag = checksum_info[self.PARAM_CHECKSUM]
path = self.checksum_to_path(etag)
to_info = {"scheme": "s3", "bucket": self.bucket, "path": path}
self.copy(path_info, to_info)
def remove(self, path_info):
if path_info["scheme"] != "s3":
raise NotImplementedError
dvc/remote/ssh/__init__.py
-10
View File
@@ -115,16 +115,6 @@ class RemoteSSH(RemoteBase):
return {self.PARAM_CHECKSUM: self.md5(path_info)}
def save(self, path_info, checksum_info):
if path_info["scheme"] != self.scheme:
raise NotImplementedError
md5 = checksum_info[self.PARAM_CHECKSUM]
dest = path_info.copy()
dest["path"] = self.checksum_to_path(md5)
self.copy(path_info, dest)
def remove(self, path_info):
if path_info["scheme"] != self.scheme:
raise NotImplementedError
tests/unit/remote/base.py
+26
View File
@@ -33,3 +33,29 @@ class TestCmdError(TestCase):
):
with self.assertRaises(RemoteCmdError):
remote_class(repo, config).remove("file")
class TestCacheExists(TestCase):
def test(self):
for remote_class in REMOTES:
config = {
"url": remote_class.scheme + "://example/prefix",
"connection_string": "1234567",
}
remote = remote_class(None, config)
with mock.patch.object(remote, "changed_cache", return_value=True):
with mock.patch.object(remote, "copy") as cp:
remote.save(
remote.path_info, {remote.PARAM_CHECKSUM: "1234567890"}
)
cp.assert_called_once()
with mock.patch.object(
remote, "changed_cache", return_value=False
):
with mock.patch.object(remote, "copy") as cp:
remote.save(
remote.path_info, {remote.PARAM_CHECKSUM: "1234567890"}
)
cp.assert_not_called()