diff --git a/dvc/output.py b/dvc/output.py index e43ce00c4..08ebf8d13 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -86,7 +86,7 @@ class Dependency(object): if not os.path.isfile(self.path) and not os.path.isdir(self.path): raise CmdOutputIsNotFileOrDirError(self.rel_path) - self.md5 = self.project.state.update(self.path).md5 + self.md5 = self.project.state.update(self.path) def dumpd(self, cwd): return { @@ -157,8 +157,8 @@ class Output(Dependency): return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list] def _changed_cache(self, cache): - state = self.project.state.update(cache) - if state.md5 != self.project.cache.path_to_md5(cache): + md5 = self.project.state.update(cache) + if md5 != self.project.cache.path_to_md5(cache): self.project.logger.debug('Corrupted cache file {}'.format(cache)) self.project._remove_cache_file(cache) return True @@ -303,8 +303,10 @@ class Output(Dependency): path = os.path.join(root, fname) relpath = os.path.relpath(path, self.path) - state = self.project.state.update(path) - dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: state.md5}) + md5 = self.project.state.update(path, dump=False) + dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5}) + + self.project.state.dump() return dir_info diff --git a/dvc/state.py b/dvc/state.py index 1e1babf4a..5d61b91c8 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -1,5 +1,5 @@ import os -import tinydb +import json from checksumdir import dirhash from dvc.system import System @@ -11,30 +11,25 @@ from dvc.exceptions import DvcException class StateEntry(object): PARAM_MTIME = 'mtime' PARAM_MD5 = 'md5' - PARAM_INODE = 'inode' - def __init__(self, md5, mtime, inode): + def __init__(self, md5, mtime): self.mtime = mtime self.md5 = md5 - self.inode = inode - def update(self, md5, mtime, inode): + def update(self, md5, mtime): self.mtime = mtime self.md5 = md5 - self.inode = inode @staticmethod def loadd(d): mtime = d[StateEntry.PARAM_MTIME] md5 = d[StateEntry.PARAM_MD5] - inode = d[StateEntry.PARAM_INODE] - return StateEntry(md5, mtime, inode) + return StateEntry(md5, mtime) def dumpd(self): return { self.PARAM_MD5: self.md5, self.PARAM_MTIME: self.mtime, - self.PARAM_INODE: self.inode } @@ -49,8 +44,7 @@ class State(object): self.root_dir = root_dir self.dvc_dir = dvc_dir self.state_file = os.path.join(dvc_dir, self.STATE_FILE) - self._db = tinydb.TinyDB(self.state_file) - self._q = tinydb.Query() + self._db = {} @staticmethod def init(root_dir, dvc_dir): @@ -63,38 +57,42 @@ class State(object): return file_md5(path)[0] def changed(self, path, md5): - state = self.update(path) - return state.md5 != md5 + return self.update(path) != md5 - def update(self, path): + def load(self): + with open(self.state_file, 'r') as fd: + self._db = json.load(fd) + + def dump(self): + with open(self.state_file, 'w+') as fd: + json.dump(self._db, fd) + + def update(self, path, dump=True): mtime = os.path.getmtime(path) inode = System.inode(path) - state = self._get(inode, mtime) - if state: - return state + md5 = self._get(inode, mtime) + if md5: + return md5 md5 = self.compute_md5(path) - state = StateEntry(md5, mtime, inode) + state = StateEntry(md5, mtime) d = state.dumpd() - if self._db.contains(self._q.inode == inode): - self._db.update(d, self._q.inode == inode) - else: - self._db.insert(d) + self._db[inode] = d - return state + if dump: + self.dump() + + return md5 def _get(self, inode, mtime): - d_list = self._db.search(self._q.inode == inode) - if not len(d_list): + d = self._db.get(inode, None) + if not d: return None - if len(d_list) > 1: - raise StateDuplicateError() - - state = StateEntry.loadd(d_list[0]) - if mtime == state.mtime and inode == state.inode: - return state + state = StateEntry.loadd(d) + if mtime == state.mtime: + return state.md5 return None diff --git a/requirements.txt b/requirements.txt index 626e8bf77..e8dcbf4bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ colorama>=0.3.9 configobj>=5.0.6 networkx>=1.11 pyyaml>=3.12 -tinydb>=3.7.0 gitpython>=2.1.8 ntfsutils>=0.1.3 checksumdir>=1.1.4 diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh index b84e58568..87551efb3 100644 --- a/scripts/ci/install.sh +++ b/scripts/ci/install.sh @@ -3,14 +3,6 @@ set -x set -e - -if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - wget https://s3-us-west-2.amazonaws.com/dataversioncontrol/graphviz-2.38.0.pkg - sudo installer -pkg graphviz-2.38.0.pkg -target / -else - sudo apt-get install graphviz -fi - pip install --upgrade pip pip install -r requirements.txt pip install -r test-requirements.txt diff --git a/setup.py b/setup.py index 3a12ba6e1..aae7f9a09 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,6 @@ install_requires = [ "configobj>=5.0.6", "networkx>=1.11", "pyyaml>=3.12", - "tinydb>=3.7.0", "gitpython>=2.1.8", "ntfsutils>=0.1.3", "checksumdir>=1.1.4", diff --git a/tests/test_checkout.py b/tests/test_checkout.py index 2571c488e..7a248b1a2 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -37,10 +37,6 @@ class TestCheckoutCorruptedCacheFile(TestRepro): self.dvc.checkout() - #FIXME debug - import time - os.system('cat .dvc/state') - print('TIME {}'.format(time.time())) self.assertFalse(os.path.isfile(self.FOO)) self.assertFalse(os.path.isfile(cache)) diff --git a/tests/test_state.py b/tests/test_state.py index f29f49585..639e61645 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -1,5 +1,6 @@ import os import stat +import time from dvc.system import System from dvc.state import State, StateEntry @@ -17,12 +18,15 @@ class TestState(TestDvc): state = State(self.dvc.root_dir, self.dvc.dvc_dir) - state.update(path) - entry = state.get(path) - self.assertIsInstance(entry, StateEntry) - self.assertEqual(entry.md5, md5) - self.assertEqual(entry.mtime, mtime) - self.assertEqual(entry.inode, inode) + entry_md5 = state.update(path) + self.assertEqual(entry_md5, md5) + + # Sleep some time to simulate realistic behavior. + # Some filesystems have a bad date resolution for + # mtime(i.e. 1sec for HFS) that cause problems with + # our 'state' system not being able to distinguish + # files that were modified within that delta. + time.sleep(1) os.chmod(path, stat.S_IWRITE) os.unlink(path) @@ -33,8 +37,5 @@ class TestState(TestDvc): mtime = os.path.getmtime(path) inode = System.inode(path) - entry = state.update(path) - self.assertIsInstance(entry, StateEntry) - self.assertEqual(entry.md5, md5) - self.assertEqual(entry.mtime, mtime) - self.assertEqual(entry.inode, inode) + entry_md5 = state.update(path) + self.assertEqual(entry_md5, md5)