state: add cache state

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/path/data_item.py

@@ -34,6 +34,7 @@ class NotInGitDirError(DataDirError):
 class DataItem(object):
     STATE_FILE_SUFFIX = '.state'
     LOCAL_STATE_FILE_SUFFIX = '.local_state'
+    CACHE_STATE_FILE_SUFFIX = '.cache_state'
     CACHE_FILE_SEP = '_'
 
     def __init__(self, data_file, git, config, cache_file=None):
@@ -88,6 +89,11 @@ class DataItem(object):
         local_state_file = os.path.join(self.state_dir, self.data.dvc + self.LOCAL_STATE_FILE_SUFFIX)
         return Path(local_state_file, self._git)
 
+    @cached_property
+    def cache_state(self):
+        cache_state_file = os.path.join(self.state_dir, self.data.dvc + self.CACHE_STATE_FILE_SUFFIX)
+        return Path(cache_state_file, self._git)
+
     @cached_property
     def cache_dir(self):
         return os.path.join(self._git.git_dir_abs, self._config.cache_dir)
@@ -99,8 +105,8 @@ class DataItem(object):
         if self._cache_file:
             file_name = os.path.relpath(os.path.realpath(self._cache_file), cache_dir)
         else:
-            from dvc.state_file import StateFile
-            file_name = StateFile.load(self, self._git).md5
+            from dvc.state_file import CacheStateFile
+            file_name = CacheStateFile.load(self).md5
 
         cache_file = os.path.join(cache_dir, file_name)
         return Path(cache_file, self._git)
@@ -117,6 +123,9 @@ class DataItem(object):
             System.hardlink(self.data.relative, self.cache.relative)
         os.chmod(self.data.relative, stat.S_IREAD)
 
+        from dvc.state_file import CacheStateFile
+        cache_state = CacheStateFile(self).save()
+
         from dvc.state_file import LocalStateFile
         local_state = LocalStateFile(self).save()
         self._git.modify_gitignore([self.local_state.relative])

dvc/state_file.py

@@ -8,7 +8,7 @@ from dvc import utils
 from dvc.exceptions import DvcException
 from dvc.path.data_item import DataDirError
 from dvc.system import System
-
+from dvc.data_cloud import file_md5
 
 class StateFileError(DvcException):
     def __init__(self, msg):
@@ -40,7 +40,6 @@ class StateFile(object):
     PARAM_SHELL = "Shell"
     PARAM_TARGET_METRICS = 'TargetMetrics'
     TARGET_METRICS_SINGLE_METRIC = 'SingleMetric'
-    PARAM_MD5 = 'Md5'
 
     def __init__(self,
                  command,
@@ -55,8 +54,7 @@ class StateFile(object):
                  stderr=None,
                  cwd=None,
                  shell=False,
-                 target_metrics={},
-                 md5=None):
+                 target_metrics={}):
         self.data_item = data_item
 
         self.settings = settings
@@ -66,10 +64,6 @@ class StateFile(object):
         self.code_dependencies = code_dependencies
         self.shell = shell
 
-        self.md5 = md5
-        if not md5 and data_item:
-            self.md5 = os.path.basename(data_item.cache.relative)
-
         if command not in self.ACCEPTED_COMMANDS:
             raise StateFileError('Args error: unknown command %s' % command)
         self.command = command
@@ -176,8 +170,7 @@ class StateFile(object):
                          StateFile.decode_path(json.get(StateFile.PARAM_STDERR)),
                          StateFile.decode_path(json.get(StateFile.PARAM_CWD)),
                          json.get(StateFile.PARAM_SHELL, False),
-                         json.get(StateFile.PARAM_TARGET_METRICS, {}),
-                         json.get(StateFile.PARAM_MD5, ''))
+                         json.get(StateFile.PARAM_TARGET_METRICS, {}))
 
     @staticmethod
     def load(data_item, settings):
@@ -209,7 +202,6 @@ class StateFile(object):
             self.PARAM_STDERR:              self.encode_path(self.stderr),
             self.PARAM_SHELL:               self.shell,
             self.PARAM_TARGET_METRICS:      self.target_metrics,
-            self.PARAM_MD5:                 self.md5
         }
 
         if self.locked:
@@ -242,6 +234,44 @@ class StateFile(object):
 
         return os.path.relpath(pwd, self.settings.git.git_dir_abs)
 
+
+class CacheStateFile(object):
+    MAGIC = 'DVC-Cache-State'
+    VERSION = '0.1'
+
+    PARAM_MD5 = 'Md5'
+
+    def __init__(self, data_item, md5=None):
+        self.data_item = data_item
+        self.md5 = md5
+
+        if not md5:
+            self.md5 = file_md5(data_item.data.relative)[0]
+
+    @staticmethod
+    def load_json(json):
+        return CacheStateFile(None,
+                              json.get(CacheStateFile.PARAM_MD5, None))
+
+    @staticmethod
+    def load(data_item):
+        with open(data_item.cache_state.relative, 'r') as fd:
+            data = json.load(fd)
+            return CacheStateFile.load_json(data)
+
+    def save(self):
+        res = {
+            self.PARAM_MD5 : self.md5,
+        }
+
+        file_dir = os.path.dirname(self.data_item.cache_state.relative)
+        if not os.path.isdir(file_dir):
+            os.makedirs(file_dir)
+
+        with open(self.data_item.cache_state.relative, 'w') as fd:
+            json.dump(res, fd, indent=2, sort_keys=True)
+
+
 class LocalStateFile(object):
     MAGIC = 'DVC-Local-State'
     VERSION = '0.1'