remote: RFC: remote.save()

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/output/base.py

@@ -137,11 +137,10 @@ class OutputBase(object):
         return {}
 
     def save(self):
-        if not self.use_cache:
-            self.info = self.remote.save_info(self.path_info)
-        else:
-            self.info = getattr(self.repo.cache, self.scheme).save(
-                self.path_info
+        self.info = self.remote.save_info(self.path_info)
+        if self.use_cache:
+            getattr(self.repo.cache, self.scheme).save(
+                self.path_info, self.info
             )
 
     def dumpd(self):

dvc/output/local.py

@@ -117,4 +117,5 @@ class OutputLOCAL(OutputBase):
             if self.use_cache:
                 self.repo.scm.ignore(self.path)
 
-        self.info = self.repo.cache.local.save(self.path_info)
+        self.info = self.remote.save_info(self.path_info)
+        self.project.cache.local.save(self.path_info, self.info)

dvc/remote/base.py

@@ -172,7 +172,7 @@ class RemoteBase(object):
         logger.debug("'{}' hasn't changed.".format(path_info))
         return False
 
-    def save(self, path_info):
+    def save(self, path_info, checksum_info):
         raise RemoteActionNotImplemented("save", self.scheme)
 
     def download(

dvc/remote/hdfs.py

@@ -88,20 +88,18 @@ class RemoteHDFS(RemoteBase):
 
         return {self.PARAM_CHECKSUM: self.checksum(path_info)}
 
-    def save(self, path_info):
+    def save(self, path_info, checksum_info):
         if path_info["scheme"] != "hdfs":
             raise NotImplementedError
 
         assert path_info.get("path")
 
-        checksum = self.checksum(path_info)
+        checksum = checksum_info[self.PARAM_CHECKSUM]
         dest = path_info.copy()
         dest["path"] = self.checksum_to_path(checksum)
 
         self.copy(path_info, dest)
 
-        return {self.PARAM_CHECKSUM: checksum}
-
     def remove(self, path_info):
         if path_info["scheme"] != "hdfs":
             raise NotImplementedError

dvc/remote/local.py

@@ -370,9 +370,7 @@ class RemoteLOCAL(RemoteBase):
         move(inp, tmp)
         move(tmp, outp)
 
-    def _save_file(self, path_info):
-        path = path_info["path"]
-        md5 = self.state.update(path)
+    def _save_file(self, path, md5):
         assert md5 is not None
 
         cache = self.get(md5)
@@ -393,9 +391,8 @@ class RemoteLOCAL(RemoteBase):
 
         return {self.PARAM_CHECKSUM: md5}
 
-    def _save_dir(self, path_info):
-        path = path_info["path"]
-        md5, dir_info = self.state.update_info(path)
+    def _save_dir(self, path, md5):
+        dir_info = self.load_dir_cache(md5)
         dir_relpath = os.path.relpath(path)
         dir_size = len(dir_info)
         bar = dir_size > LARGE_DIR_SIZE
@@ -429,9 +426,7 @@ class RemoteLOCAL(RemoteBase):
         if bar:
             progress.finish_target(dir_relpath)
 
-        return {self.PARAM_CHECKSUM: md5}
-
-    def save(self, path_info):
+    def save(self, path_info, checksum_info):
         if path_info["scheme"] != "local":
             raise NotImplementedError
 
@@ -442,10 +437,11 @@ class RemoteLOCAL(RemoteBase):
             msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir))
         )
 
+        md5 = checksum_info[self.PARAM_CHECKSUM]
         if os.path.isdir(path):
-            return self._save_dir(path_info)
+            self._save_dir(path, md5)
         else:
-            return self._save_file(path_info)
+            self._save_file(path, md5)
 
     def save_info(self, path_info):
         if path_info["scheme"] != "local":

dvc/remote/s3.py

@@ -109,18 +109,16 @@ class RemoteS3(RemoteBase):
         source = {"Bucket": from_info["bucket"], "Key": from_info["path"]}
         self.s3.copy(source, to_info["bucket"], to_info["path"])
 
-    def save(self, path_info):
+    def save(self, path_info, checksum_info):
         if path_info["scheme"] != "s3":
             raise NotImplementedError
 
-        etag = self.get_etag(path_info["bucket"], path_info["path"])
+        etag = checksum_info[self.PARAM_CHECKSUM]
         path = self.checksum_to_path(etag)
         to_info = {"scheme": "s3", "bucket": self.bucket, "path": path}
 
         self.copy(path_info, to_info)
 
-        return {self.PARAM_CHECKSUM: etag}
-
     def remove(self, path_info):
         if path_info["scheme"] != "s3":
             raise NotImplementedError

dvc/remote/ssh.py

@@ -244,18 +244,16 @@ class RemoteSSH(RemoteBase):
 
         return {self.PARAM_CHECKSUM: self.md5(path_info)}
 
-    def save(self, path_info):
+    def save(self, path_info, checksum_info):
         if path_info["scheme"] != "ssh":
             raise NotImplementedError
 
-        md5 = self.md5(path_info)
+        md5 = checksum_info[self.PARAM_CHECKSUM]
         dest = path_info.copy()
         dest["path"] = self.checksum_to_path(md5)
 
         self.copy(path_info, dest)
 
-        return {self.PARAM_CHECKSUM: md5}
-
     def remove(self, path_info):
         if path_info["scheme"] != "ssh":
             raise NotImplementedError