state: save collected dir to cache

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/project.py

@@ -52,8 +52,8 @@ class Project(object):
         self.lock = Lock(self.dvc_dir)
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
-        self.state = State(self.dvc_dir)
-        self.link_state = LinkState(self.root_dir, self.dvc_dir)
+        self.state = State(self)
+        self.link_state = LinkState(self)
         self.logger = Logger(self.config._config[Config.SECTION_CORE].get(Config.SECTION_CORE_LOGLEVEL, None))
         self.cache = Cache(self)
         self.cloud = DataCloud(self, config=self.config._config)

dvc/remote/local.py

@@ -111,8 +111,11 @@ class RemoteLOCAL(RemoteBase):
                 if typ == types[-1]:
                     raise DvcException(msg, cause=exc)
 
-    @staticmethod
-    def load_dir_cache(path):
+    def load_dir_cache(self, md5):
+        path = self.get(md5)
+
+        assert self.is_dir_cache(path)
+
         try:
             with open(path, 'r') as fd:
                 d = json.load(fd)
@@ -128,6 +131,19 @@ class RemoteLOCAL(RemoteBase):
 
         return d
 
+    def dump_dir_cache(self, md5, dir_info):
+        path = self.get(md5)
+        dname = os.path.dirname(path)
+
+        assert self.is_dir_cache(path)
+        assert isinstance(dir_info, list)
+
+        if not os.path.isdir(dname):
+            os.makedirs(dname)
+
+        with open(path, 'w+') as fd:
+            json.dump(dir_info, fd, sort_keys=True)
+
     @staticmethod
     def is_dir_cache(cache):
         return cache.endswith(State.MD5_DIR_SUFFIX)
@@ -159,7 +175,7 @@ class RemoteLOCAL(RemoteBase):
         if not os.path.exists(path):
             os.makedirs(path)
 
-        for entry in self.load_dir_cache(cache):
+        for entry in self.load_dir_cache(md5):
             md5 = entry[self.PARAM_MD5]
             relpath = entry[self.PARAM_RELPATH]
             p = os.path.join(path, relpath)
@@ -188,8 +204,6 @@ class RemoteLOCAL(RemoteBase):
     def _save_dir(self, path_info):
         path = path_info['path']
         md5, dir_info = self.state.update_info(path)
-        cache = self.get(md5)
-        dname = os.path.dirname(cache)
 
         for entry in dir_info:
             relpath = entry[State.PARAM_RELPATH]
@@ -197,15 +211,6 @@ class RemoteLOCAL(RemoteBase):
 
             self._save_file({'scheme': 'local', 'path': p})
 
-        if not os.path.isdir(dname):
-            os.makedirs(dname)
-
-        Logger.debug(u'Saving directory \'{}\' to \'{}\''.format(os.path.relpath(path),
-                                                                 os.path.relpath(cache)))
-
-        with open(cache, 'w+') as fd:
-            json.dump(dir_info, fd, sort_keys=True)
-
         return {self.PARAM_MD5: md5}
 
     def save(self, path_info):
@@ -316,7 +321,7 @@ class RemoteLOCAL(RemoteBase):
                 continue
             if not os.path.exists(cache):
                 missing.append(info)
-            collected.extend(self.load_dir_cache(cache))
+            collected.extend(self.load_dir_cache(md5))
         collected.extend(checksum_infos)
         return collected, missing
 

dvc/state.py

@@ -48,20 +48,21 @@ class State(object):
     PARAM_MD5 = 'md5'
     MD5_DIR_SUFFIX = '.dir'
 
-    def __init__(self, dvc_dir):
-        self.dvc_dir = dvc_dir
+    def __init__(self, project):
+        self.project = project
+        self.dvc_dir = project.dvc_dir
         self._lock = threading.Lock()
-        if dvc_dir:
-            self.state_file = os.path.join(dvc_dir, self.STATE_FILE)
-            self._lock_file = Lock(dvc_dir, self.STATE_LOCK_FILE)
+        if self.dvc_dir:
+            self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
+            self._lock_file = Lock(self.dvc_dir, self.STATE_LOCK_FILE)
         else:
             self.state_file = None
             self._lock_file = threading.Lock()
         self._db = self.load()
 
     @staticmethod
-    def init(dvc_dir):
-        return State(dvc_dir)
+    def init(project):
+        return State(project)
 
     def _collect_dir(self, dname):
         dir_info = []
@@ -75,12 +76,14 @@ class State(object):
                 dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5})
 
         md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
+        if self.project.cache.local.changed(md5):
+            self.project.cache.local.dump_dir_cache(md5, dir_info)
 
         return (md5, dir_info)
 
     def _collect(self, path):
         if os.path.isdir(path):
-            return collect_dir
+            return self._collect_dir(path)
         else:
             return (file_md5(path)[0], None)
 
@@ -121,17 +124,17 @@ class State(object):
 
     def _do_update(self, path, dump=True):
         if not os.path.exists(path):
-            return None
+            return (None, None)
 
         mtime = self.mtime(path)
         inode = self.inode(path)
 
-        entry = self._get(inode, mtime)
-        if entry:
-            return entry
+        md5 = self._get(inode, mtime)
+        if md5:
+            return (md5, None)
 
         md5, info = self._collect(path)
-        state = StateEntry(md5, mtime, info)
+        state = StateEntry(md5, mtime)
         d = state.dumpd()
 
         with self._lock:
@@ -144,10 +147,13 @@ class State(object):
         return (md5, info)
 
     def update(self, path, dump=True):
-        return self._do_update(self, path, dump=True)[0]
+        return self._do_update(path, dump=dump)[0]
 
     def update_info(self, path, dump=True):
-        return self._do_update(self, path, dump=True)
+        md5, info = self._do_update(path, dump=dump)
+        if not info:
+            info = self.project.cache.local.load_dir_cache(md5)
+        return (md5, info)
 
     def _get(self, inode, mtime):
         with self._lock:
@@ -159,7 +165,7 @@ class State(object):
 
         state = StateEntry.loadd(d)
         if mtime == state.mtime:
-            return (state.md5, state.info)
+            return state.md5
 
         return None
 
@@ -195,9 +201,9 @@ class LinkState(State):
     STATE_FILE = 'link.state'
     STATE_LOCK_FILE = STATE_FILE + '.lock'
 
-    def __init__(self, root_dir, dvc_dir):
-        super(LinkState, self).__init__(dvc_dir)
-        self.root_dir = root_dir
+    def __init__(self, project):
+        super(LinkState, self).__init__(project)
+        self.root_dir = project.root_dir
 
     def update(self, path, dump=True):
         if not os.path.exists(path):

tests/test_cache.py

@@ -41,11 +41,11 @@ class TestCacheLoadBadDirCache(TestDvc):
         self.assertEqual(len(ret), 0)
 
     def test(self):
-        fname = 'not-json'
+        fname = self.dvc.cache.local.get('123')
         self.create(fname, '<clearly>not,json')
         self._do_test(RemoteLOCAL.load_dir_cache(fname))
 
-        fname = 'not-list'
+        fname = self.dvc.cache.local.get('234')
         self.create(fname, '{"a": "b"}')
         self._do_test(RemoteLOCAL.load_dir_cache(fname))
 

tests/test_state.py

@@ -16,7 +16,7 @@ class TestState(TestDvc):
         mtime = os.path.getmtime(path)
         inode = System.inode(path)
 
-        state = State(self.dvc.dvc_dir)
+        state = State(self.dvc)
 
         entry_md5 = state.update(path)
         self.assertEqual(entry_md5, md5)
@@ -41,6 +41,6 @@ class TestState(TestDvc):
 
         # Reload state db to make sure that it stays the same
         old_db = state._db
-        state = State(self.dvc.dvc_dir)
+        state = State(self.dvc)
         new_db = state._db
         self.assertEqual(old_db, new_db)