dvc: stop using checksumdir and use our own way of checksuming a dir

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/cloud/base.py

@@ -1,7 +1,6 @@
 import os
 import re
 import tempfile
-from checksumdir import dirhash
 
 from dvc.logger import Logger
 from dvc.exceptions import DvcException

dvc/output.py

@@ -5,7 +5,6 @@ import shutil
 import schema
 import posixpath
 import ntpath
-from checksumdir import dirhash
 
 from dvc.system import System
 from dvc.utils import file_md5
@@ -195,8 +194,8 @@ class Output(Dependency):
         return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]
 
     def _changed_cache(self, cache):
-        md5 = self.project.state.update(cache)
-        if md5 != self.project.cache.path_to_md5(cache):
+        md5 = self.project.cache.path_to_md5(cache)
+        if self.project.state.changed(cache, md5=md5):
             self.project.logger.warn('Corrupted cache file {}'.format(os.path.relpath(cache)))
             os.unlink(cache)
             return True
@@ -213,24 +212,12 @@ class Output(Dependency):
         return True
 
     def _changed_dir(self):
-        if not os.path.isdir(self.path) or not os.path.isfile(self.cache):
-            return True
+        if os.path.isdir(self.path) and \
+           os.path.isfile(self.cache) and \
+           not self._changed_cache(self.cache):
+            return False
 
-        dir_info = self._collect_dir() # slow!
-        dir_info_cached = self.load_dir_cache(self.cache) # slow. why?
-
-        if not self.are_dir_info_equal(dir_info, dir_info_cached):
-            return True
-
-        return False
-
-    @staticmethod
-    def are_dir_info_equal(dir_info1, dir_info2):
-        return Output.dir_info_dict(dir_info1) == Output.dir_info_dict(dir_info2)
-
-    @staticmethod
-    def dir_info_dict(dir_info):
-        return {i['relpath']: i['md5'] for i in dir_info}
+        return True
 
     def changed(self):
         if not self.use_cache:
@@ -330,24 +317,9 @@ class Output(Dependency):
             path = os.path.join(self.path, relpath)
             self.hardlink(cache, path)
 
-    def _collect_dir(self):
-        dir_info = []
-
-        for root, dirs, files in os.walk(self.path):
-            for fname in files:
-                path = os.path.join(root, fname)
-                relpath = os.path.relpath(path, self.path)
-
-                md5 = self.project.state.update(path, dump=False)
-                dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5})
-
-        self.project.state.dump()
-
-        return dir_info
-
     def _save_dir(self):
         dname = os.path.dirname(self.cache)
-        dir_info = self._collect_dir()
+        dir_info = self.project.state.collect_dir(self.path)
 
         for entry in dir_info:
             md5 = entry[self.PARAM_MD5]
@@ -365,7 +337,7 @@ class Output(Dependency):
             os.makedirs(dname)
 
         with open(self.cache, 'w+') as fd:
-            json.dump(dir_info, fd)
+            json.dump(dir_info, fd, sort_keys=True)
 
     def save(self):
         super(Output, self).save()

dvc/state.py

@@ -1,11 +1,11 @@
 import os
 import json
 import nanotime
-from checksumdir import dirhash
+import threading
 
 from dvc.system import System
 from dvc.output import Output
-from dvc.utils import file_md5
+from dvc.utils import file_md5, bytes_md5
 from dvc.exceptions import DvcException
 
 
@@ -46,19 +46,43 @@ class State(object):
         self.dvc_dir = dvc_dir
         self.state_file = os.path.join(dvc_dir, self.STATE_FILE)
         self._db = self.load()
+        self._lock = threading.Lock()
 
     @staticmethod
     def init(root_dir, dvc_dir):
         return State(root_dir, dvc_dir)
 
+    def collect_dir(self, dname):
+        dir_info = []
+
+        for root, dirs, files in os.walk(dname):
+            for fname in files:
+                path = os.path.join(root, fname)
+                relpath = os.path.relpath(path, dname)
+
+                md5 = self.update(path, dump=False)
+                dir_info.append({Output.PARAM_RELPATH: relpath, Output.PARAM_MD5: md5})
+
+        self.dump()
+
+        return dir_info
+
     def compute_md5(self, path):
         if os.path.isdir(path):
-            return dirhash(path, hashfunc='md5') + Output.MD5_DIR_SUFFIX
+            dir_info = self.collect_dir(path)
+            byts = json.dumps(dir_info, sort_keys=True).encode('utf-8')
+            md5 = bytes_md5(byts)
+            return md5 + Output.MD5_DIR_SUFFIX
         else:
             return file_md5(path)[0]
 
     def changed(self, path, md5):
-        return self.update(path) != md5
+        actual = self.update(path)
+
+        if not md5 or not actual:
+            return True
+
+        return actual.split('.')[0] != md5.split('.')[0]
 
     def load(self):
         if not os.path.isfile(self.state_file):
@@ -90,15 +114,18 @@ class State(object):
         md5 = self.compute_md5(path)
         state = StateEntry(md5, mtime)
         d = state.dumpd()
-        self._db[inode] = d
 
-        if dump:
-            self.dump()
+        with self._lock:
+            self._db[inode] = d
+
+            if dump:
+                self.dump()
 
         return md5
 
     def _get(self, inode, mtime):
-        d = self._db.get(inode, None)
+        with self._lock:
+            d = self._db.get(inode, None)
         if not d:
             return None
 

dvc/utils.py

@@ -28,6 +28,12 @@ def file_md5(fname):
         return (None, None)
 
 
+def bytes_md5(byts):
+    hasher = hashlib.md5()
+    hasher.update(byts)
+    return hasher.hexdigest()
+
+
 def cached_property(func):
     '''A decorator for caching properties in classes.'''
     def get(self):

requirements.txt

@@ -11,7 +11,6 @@ networkx>=1.11
 pyyaml>=3.12
 gitpython>=2.1.8
 ntfsutils>=0.1.4
-checksumdir>=1.1.4
 setuptools>=34.0.0
 nanotime>=0.5.2
 pyasn1>=0.4.1

setup.py

@@ -17,7 +17,6 @@ install_requires = [
     "pyyaml>=3.12",
     "gitpython>=2.1.8",
     "ntfsutils>=0.1.4",
-    "checksumdir>=1.1.4",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",

tests/test_add.py

@@ -2,7 +2,6 @@ import os
 import stat
 import shutil
 import filecmp
-from checksumdir import dirhash
 
 from dvc.main import main
 from dvc.utils import file_md5