Merge pull request #545 from efiop/master

State and travis fixes

dvc/output.py

@@ -86,7 +86,7 @@ class Dependency(object):
         if not os.path.isfile(self.path) and not os.path.isdir(self.path):
             raise CmdOutputIsNotFileOrDirError(self.rel_path)
 
-        self.md5 = self.project.state.update(self.path).md5
+        self.md5 = self.project.state.update(self.path)
 
     def dumpd(self, cwd):
         return {
@@ -157,8 +157,8 @@ class Output(Dependency):
         return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]
 
     def _changed_cache(self, cache):
-        state = self.project.state.update(cache)
-        if state.md5 != self.project.cache.path_to_md5(cache):
+        md5 = self.project.state.update(cache)
+        if md5 != self.project.cache.path_to_md5(cache):
             self.project.logger.debug('Corrupted cache file {}'.format(cache))
             self.project._remove_cache_file(cache)
             return True
@@ -303,8 +303,10 @@ class Output(Dependency):
                 path = os.path.join(root, fname)
                 relpath = os.path.relpath(path, self.path)
 
-                state = self.project.state.update(path)
-                dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: state.md5})
+                md5 = self.project.state.update(path, dump=False)
+                dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5})
+
+        self.project.state.dump()
 
         return dir_info
 

dvc/state.py

@@ -1,5 +1,5 @@
 import os
-import tinydb
+import json
 from checksumdir import dirhash
 
 from dvc.system import System
@@ -11,30 +11,25 @@ from dvc.exceptions import DvcException
 class StateEntry(object):
     PARAM_MTIME = 'mtime'
     PARAM_MD5 = 'md5'
-    PARAM_INODE = 'inode'
 
-    def __init__(self, md5, mtime, inode):
+    def __init__(self, md5, mtime):
         self.mtime = mtime
         self.md5 = md5
-        self.inode = inode
 
-    def update(self, md5, mtime, inode):
+    def update(self, md5, mtime):
         self.mtime = mtime
         self.md5 = md5
-        self.inode = inode
 
     @staticmethod
     def loadd(d):
         mtime = d[StateEntry.PARAM_MTIME]
         md5 = d[StateEntry.PARAM_MD5]
-        inode = d[StateEntry.PARAM_INODE]
-        return StateEntry(md5, mtime, inode)
+        return StateEntry(md5, mtime)
 
     def dumpd(self):
         return {
             self.PARAM_MD5: self.md5,
             self.PARAM_MTIME: self.mtime,
-            self.PARAM_INODE: self.inode
         }
 
 
@@ -49,8 +44,7 @@ class State(object):
         self.root_dir = root_dir
         self.dvc_dir = dvc_dir
         self.state_file = os.path.join(dvc_dir, self.STATE_FILE)
-        self._db = tinydb.TinyDB(self.state_file)
-        self._q = tinydb.Query()
+        self._db = {}
 
     @staticmethod
     def init(root_dir, dvc_dir):
@@ -63,38 +57,42 @@ class State(object):
             return file_md5(path)[0]
 
     def changed(self, path, md5):
-        state = self.update(path)
-        return state.md5 != md5
+        return self.update(path) != md5
 
-    def update(self, path):
+    def load(self):
+        with open(self.state_file, 'r') as fd:
+            self._db = json.load(fd)
+
+    def dump(self):
+        with open(self.state_file, 'w+') as fd:
+            json.dump(self._db, fd)
+
+    def update(self, path, dump=True):
         mtime = os.path.getmtime(path)
         inode = System.inode(path)
 
-        state = self._get(inode, mtime)
-        if state:
-            return state
+        md5 = self._get(inode, mtime)
+        if md5:
+            return md5
 
         md5 = self.compute_md5(path)
-        state = StateEntry(md5, mtime, inode)
+        state = StateEntry(md5, mtime)
         d = state.dumpd()
-        if self._db.contains(self._q.inode == inode):
-            self._db.update(d, self._q.inode == inode)
-        else:
-            self._db.insert(d)
+        self._db[inode] = d
 
-        return state
+        if dump:
+            self.dump()
+
+        return md5
 
     def _get(self, inode, mtime):
-        d_list = self._db.search(self._q.inode == inode)
-        if not len(d_list):
+        d = self._db.get(inode, None)
+        if not d:
             return None
 
-        if len(d_list) > 1:
-            raise StateDuplicateError()
-
-        state = StateEntry.loadd(d_list[0])
-        if mtime == state.mtime and inode == state.inode:
-            return state
+        state = StateEntry.loadd(d)
+        if mtime == state.mtime:
+            return state.md5
 
         return None
 

requirements.txt

@@ -9,7 +9,6 @@ colorama>=0.3.9
 configobj>=5.0.6
 networkx>=1.11
 pyyaml>=3.12
-tinydb>=3.7.0
 gitpython>=2.1.8
 ntfsutils>=0.1.3
 checksumdir>=1.1.4

scripts/ci/install.sh

@@ -3,14 +3,6 @@
 set -x
 set -e
 
-
-if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-	wget https://s3-us-west-2.amazonaws.com/dataversioncontrol/graphviz-2.38.0.pkg
-	sudo installer -pkg graphviz-2.38.0.pkg -target /
-else
-	sudo apt-get install graphviz
-fi
-
 pip install --upgrade pip
 pip install -r requirements.txt
 pip install -r test-requirements.txt

setup.py

@@ -14,7 +14,6 @@ install_requires = [
     "configobj>=5.0.6",
     "networkx>=1.11",
     "pyyaml>=3.12",
-    "tinydb>=3.7.0",
     "gitpython>=2.1.8",
     "ntfsutils>=0.1.3",
     "checksumdir>=1.1.4",

tests/test_checkout.py

@@ -37,10 +37,6 @@ class TestCheckoutCorruptedCacheFile(TestRepro):
 
         self.dvc.checkout()
 
-        #FIXME debug
-        import time
-        os.system('cat .dvc/state')
-        print('TIME {}'.format(time.time()))
         self.assertFalse(os.path.isfile(self.FOO))
         self.assertFalse(os.path.isfile(cache))
 

tests/test_state.py

@@ -1,5 +1,6 @@
 import os
 import stat
+import time
 
 from dvc.system import System
 from dvc.state import State, StateEntry
@@ -17,12 +18,15 @@ class TestState(TestDvc):
 
         state = State(self.dvc.root_dir, self.dvc.dvc_dir)
 
-        state.update(path)
-        entry = state.get(path)
-        self.assertIsInstance(entry, StateEntry)
-        self.assertEqual(entry.md5, md5)
-        self.assertEqual(entry.mtime, mtime)
-        self.assertEqual(entry.inode, inode)
+        entry_md5 = state.update(path)
+        self.assertEqual(entry_md5, md5)
+
+        # Sleep some time to simulate realistic behavior.
+        # Some filesystems have a bad date resolution for
+        # mtime(i.e. 1sec for HFS) that cause problems with
+        # our 'state' system not being able to distinguish
+        # files that were modified within that delta.
+        time.sleep(1)        
 
         os.chmod(path, stat.S_IWRITE)
         os.unlink(path)
@@ -33,8 +37,5 @@ class TestState(TestDvc):
         mtime = os.path.getmtime(path)
         inode = System.inode(path)
 
-        entry = state.update(path)
-        self.assertIsInstance(entry, StateEntry)
-        self.assertEqual(entry.md5, md5)
-        self.assertEqual(entry.mtime, mtime)
-        self.assertEqual(entry.inode, inode)
+        entry_md5 = state.update(path)
+        self.assertEqual(entry_md5, md5)