state: add local_state

First two parameters are data and cache timestamps, that are saved
when dvc modifies them and are used to cheaply detect if something
has changed. These .local_state files are ignored by git.

Fixes #279

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/path/data_item.py

@@ -33,6 +33,7 @@ class NotInGitDirError(DataDirError):
 
 class DataItem(object):
     STATE_FILE_SUFFIX = '.state'
+    LOCAL_STATE_FILE_SUFFIX = '.local_state'
     CACHE_FILE_SEP = '_'
 
     def __init__(self, data_file, git, config, cache_file=None):
@@ -69,16 +70,28 @@ class DataItem(object):
     def data(self):
         return self._data
 
+    @cached_property
+    def state_dir(self):
+        return os.path.join(self._git.git_dir_abs, self._config.state_dir)
+
     @cached_property
     def state(self):
-        state_dir = os.path.join(self._git.git_dir_abs, ConfigI.STATE_DIR)
-        state_file = os.path.join(state_dir, self.data.dvc + self.STATE_FILE_SUFFIX)
+        state_file = os.path.join(self.state_dir, self.data.dvc + self.STATE_FILE_SUFFIX)
         return Path(state_file, self._git)
 
     @cached_property
     def cache_dir_abs(self):
         return os.path.join(self._git.git_dir_abs, ConfigI.CACHE_DIR)
 
+    @cached_property
+    def local_state(self):
+        local_state_file = os.path.join(self.state_dir, self.data.dvc + self.LOCAL_STATE_FILE_SUFFIX)
+        return Path(local_state_file, self._git)
+
+    @cached_property
+    def cache_dir(self):
+        return os.path.join(self._git.git_dir_abs, self._config.cache_dir)
+
     @cached_property
     def cache(self):
         cache_dir = self.cache_dir_abs
@@ -103,3 +116,7 @@ class DataItem(object):
         if not os.path.isfile(self.cache.relative):
             System.hardlink(self.data.relative, self.cache.relative)
         os.chmod(self.data.relative, stat.S_IREAD)
+
+        from dvc.state_file import LocalStateFile
+        local_state = LocalStateFile(self).save()
+        self._git.modify_gitignore([self.local_state.relative])

dvc/path/factory.py

@@ -61,7 +61,6 @@ class PathFactory(object):
                 path = os.path.join(root, fname)
 
                 if not fname.endswith(DataItem.STATE_FILE_SUFFIX):
-                    Logger.warn('Found file \'{}\' without state suffix inside state dir'.format(path))
                     continue
 
                 states.append(path)

dvc/state_file.py

@@ -241,3 +241,45 @@ class StateFile(object):
             raise StateFileError('the file cannot be created outside of a git repository')
 
         return os.path.relpath(pwd, self.settings.git.git_dir_abs)
+
+class LocalStateFile(object):
+    MAGIC = 'DVC-Local-State'
+    VERSION = '0.1'
+
+    PARAM_DATA_TIMESTAMP = 'DataTimestamp'
+    PARAM_CACHE_TIMESTAMP = 'CacheTimestamp'
+
+    def __init__(self, data_item, data_timestamp=None, cache_timestamp=None):
+        self.data_item = data_item
+        self.data_timestamp = data_timestamp
+        self.cache_timestamp = cache_timestamp
+
+        if not data_timestamp:
+            self.data_timestamp = os.path.getmtime(self.data_item.data.relative)
+        if not cache_timestamp:
+            self.cache_timestamp = os.path.getmtime(self.data_item.cache.relative)
+
+    @staticmethod
+    def load_json(json):
+        return LocalStateFile(None,
+                              json.get(LocalStateFile.PARAM_DATA_TIMESTAMP, None),
+                              json.get(LocalStateFile.PARAM_CACHE_TIMESTAMP, None))
+
+    @staticmethod
+    def load(data_item):
+        with open(data_item.local_state.relative, 'r') as fd:
+            data = json.load(fd)
+            return LocalStateFile.load_json(data)
+
+    def save(self):
+        res = {
+            self.PARAM_DATA_TIMESTAMP    : self.data_timestamp,
+            self.PARAM_CACHE_TIMESTAMP   : self.cache_timestamp
+        }
+
+        file_dir = os.path.dirname(self.data_item.local_state.relative)
+        if not os.path.isdir(file_dir):
+            os.makedirs(file_dir)
+
+        with open(self.data_item.local_state.relative, 'w') as fd:
+            json.dump(res, fd, indent=2, sort_keys=True)