diff --git a/dvc/project.py b/dvc/project.py index 5321f20f3..b45ca1aee 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -52,8 +52,8 @@ class Project(object): self.lock = Lock(self.dvc_dir) # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. - self.state = State(self.dvc_dir) - self.link_state = LinkState(self.root_dir, self.dvc_dir) + self.state = State(self) + self.link_state = LinkState(self) self.logger = Logger(self.config._config[Config.SECTION_CORE].get(Config.SECTION_CORE_LOGLEVEL, None)) self.cache = Cache(self) self.cloud = DataCloud(self, config=self.config._config) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 2cc9edad0..c1e643b9e 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -111,8 +111,11 @@ class RemoteLOCAL(RemoteBase): if typ == types[-1]: raise DvcException(msg, cause=exc) - @staticmethod - def load_dir_cache(path): + def load_dir_cache(self, md5): + path = self.get(md5) + + assert self.is_dir_cache(path) + try: with open(path, 'r') as fd: d = json.load(fd) @@ -128,6 +131,19 @@ class RemoteLOCAL(RemoteBase): return d + def dump_dir_cache(self, md5, dir_info): + path = self.get(md5) + dname = os.path.dirname(path) + + assert self.is_dir_cache(path) + assert isinstance(dir_info, list) + + if not os.path.isdir(dname): + os.makedirs(dname) + + with open(path, 'w+') as fd: + json.dump(dir_info, fd, sort_keys=True) + @staticmethod def is_dir_cache(cache): return cache.endswith(State.MD5_DIR_SUFFIX) @@ -159,7 +175,7 @@ class RemoteLOCAL(RemoteBase): if not os.path.exists(path): os.makedirs(path) - for entry in self.load_dir_cache(cache): + for entry in self.load_dir_cache(md5): md5 = entry[self.PARAM_MD5] relpath = entry[self.PARAM_RELPATH] p = os.path.join(path, relpath) @@ -188,8 +204,6 @@ class RemoteLOCAL(RemoteBase): def _save_dir(self, path_info): path = path_info['path'] md5, dir_info = self.state.update_info(path) - cache = self.get(md5) - dname = os.path.dirname(cache) for entry in dir_info: relpath = entry[State.PARAM_RELPATH] @@ -197,15 +211,6 @@ class RemoteLOCAL(RemoteBase): self._save_file({'scheme': 'local', 'path': p}) - if not os.path.isdir(dname): - os.makedirs(dname) - - Logger.debug(u'Saving directory \'{}\' to \'{}\''.format(os.path.relpath(path), - os.path.relpath(cache))) - - with open(cache, 'w+') as fd: - json.dump(dir_info, fd, sort_keys=True) - return {self.PARAM_MD5: md5} def save(self, path_info): @@ -316,7 +321,7 @@ class RemoteLOCAL(RemoteBase): continue if not os.path.exists(cache): missing.append(info) - collected.extend(self.load_dir_cache(cache)) + collected.extend(self.load_dir_cache(md5)) collected.extend(checksum_infos) return collected, missing diff --git a/dvc/state.py b/dvc/state.py index 35f6dded8..0d20a3f51 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -48,20 +48,21 @@ class State(object): PARAM_MD5 = 'md5' MD5_DIR_SUFFIX = '.dir' - def __init__(self, dvc_dir): - self.dvc_dir = dvc_dir + def __init__(self, project): + self.project = project + self.dvc_dir = project.dvc_dir self._lock = threading.Lock() - if dvc_dir: - self.state_file = os.path.join(dvc_dir, self.STATE_FILE) - self._lock_file = Lock(dvc_dir, self.STATE_LOCK_FILE) + if self.dvc_dir: + self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE) + self._lock_file = Lock(self.dvc_dir, self.STATE_LOCK_FILE) else: self.state_file = None self._lock_file = threading.Lock() self._db = self.load() @staticmethod - def init(dvc_dir): - return State(dvc_dir) + def init(project): + return State(project) def _collect_dir(self, dname): dir_info = [] @@ -75,12 +76,14 @@ class State(object): dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5}) md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX + if self.project.cache.local.changed(md5): + self.project.cache.local.dump_dir_cache(md5, dir_info) return (md5, dir_info) def _collect(self, path): if os.path.isdir(path): - return collect_dir + return self._collect_dir(path) else: return (file_md5(path)[0], None) @@ -121,17 +124,17 @@ class State(object): def _do_update(self, path, dump=True): if not os.path.exists(path): - return None + return (None, None) mtime = self.mtime(path) inode = self.inode(path) - entry = self._get(inode, mtime) - if entry: - return entry + md5 = self._get(inode, mtime) + if md5: + return (md5, None) md5, info = self._collect(path) - state = StateEntry(md5, mtime, info) + state = StateEntry(md5, mtime) d = state.dumpd() with self._lock: @@ -144,10 +147,13 @@ class State(object): return (md5, info) def update(self, path, dump=True): - return self._do_update(self, path, dump=True)[0] + return self._do_update(path, dump=dump)[0] def update_info(self, path, dump=True): - return self._do_update(self, path, dump=True) + md5, info = self._do_update(path, dump=dump) + if not info: + info = self.project.cache.local.load_dir_cache(md5) + return (md5, info) def _get(self, inode, mtime): with self._lock: @@ -159,7 +165,7 @@ class State(object): state = StateEntry.loadd(d) if mtime == state.mtime: - return (state.md5, state.info) + return state.md5 return None @@ -195,9 +201,9 @@ class LinkState(State): STATE_FILE = 'link.state' STATE_LOCK_FILE = STATE_FILE + '.lock' - def __init__(self, root_dir, dvc_dir): - super(LinkState, self).__init__(dvc_dir) - self.root_dir = root_dir + def __init__(self, project): + super(LinkState, self).__init__(project) + self.root_dir = project.root_dir def update(self, path, dump=True): if not os.path.exists(path): diff --git a/tests/test_cache.py b/tests/test_cache.py index 37963f2ce..d8627b5d4 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -41,11 +41,11 @@ class TestCacheLoadBadDirCache(TestDvc): self.assertEqual(len(ret), 0) def test(self): - fname = 'not-json' + fname = self.dvc.cache.local.get('123') self.create(fname, 'not,json') self._do_test(RemoteLOCAL.load_dir_cache(fname)) - fname = 'not-list' + fname = self.dvc.cache.local.get('234') self.create(fname, '{"a": "b"}') self._do_test(RemoteLOCAL.load_dir_cache(fname)) diff --git a/tests/test_state.py b/tests/test_state.py index 10ec64005..7ea8af24a 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -16,7 +16,7 @@ class TestState(TestDvc): mtime = os.path.getmtime(path) inode = System.inode(path) - state = State(self.dvc.dvc_dir) + state = State(self.dvc) entry_md5 = state.update(path) self.assertEqual(entry_md5, md5) @@ -41,6 +41,6 @@ class TestState(TestDvc): # Reload state db to make sure that it stays the same old_db = state._db - state = State(self.dvc.dvc_dir) + state = State(self.dvc) new_db = state._db self.assertEqual(old_db, new_db)