diff --git a/dvc/cloud/base.py b/dvc/cloud/base.py index 934315923..b23e19728 100644 --- a/dvc/cloud/base.py +++ b/dvc/cloud/base.py @@ -1,7 +1,6 @@ import os import re import tempfile -from checksumdir import dirhash from dvc.logger import Logger from dvc.exceptions import DvcException diff --git a/dvc/output.py b/dvc/output.py index e81205aa5..c2d032a98 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -5,7 +5,6 @@ import shutil import schema import posixpath import ntpath -from checksumdir import dirhash from dvc.system import System from dvc.utils import file_md5 @@ -195,8 +194,8 @@ class Output(Dependency): return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list] def _changed_cache(self, cache): - md5 = self.project.state.update(cache) - if md5 != self.project.cache.path_to_md5(cache): + md5 = self.project.cache.path_to_md5(cache) + if self.project.state.changed(cache, md5=md5): self.project.logger.warn('Corrupted cache file {}'.format(os.path.relpath(cache))) os.unlink(cache) return True @@ -213,24 +212,12 @@ class Output(Dependency): return True def _changed_dir(self): - if not os.path.isdir(self.path) or not os.path.isfile(self.cache): - return True + if os.path.isdir(self.path) and \ + os.path.isfile(self.cache) and \ + not self._changed_cache(self.cache): + return False - dir_info = self._collect_dir() # slow! - dir_info_cached = self.load_dir_cache(self.cache) # slow. why? - - if not self.are_dir_info_equal(dir_info, dir_info_cached): - return True - - return False - - @staticmethod - def are_dir_info_equal(dir_info1, dir_info2): - return Output.dir_info_dict(dir_info1) == Output.dir_info_dict(dir_info2) - - @staticmethod - def dir_info_dict(dir_info): - return {i['relpath']: i['md5'] for i in dir_info} + return True def changed(self): if not self.use_cache: @@ -330,24 +317,9 @@ class Output(Dependency): path = os.path.join(self.path, relpath) self.hardlink(cache, path) - def _collect_dir(self): - dir_info = [] - - for root, dirs, files in os.walk(self.path): - for fname in files: - path = os.path.join(root, fname) - relpath = os.path.relpath(path, self.path) - - md5 = self.project.state.update(path, dump=False) - dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5}) - - self.project.state.dump() - - return dir_info - def _save_dir(self): dname = os.path.dirname(self.cache) - dir_info = self._collect_dir() + dir_info = self.project.state.collect_dir(self.path) for entry in dir_info: md5 = entry[self.PARAM_MD5] @@ -365,7 +337,7 @@ class Output(Dependency): os.makedirs(dname) with open(self.cache, 'w+') as fd: - json.dump(dir_info, fd) + json.dump(dir_info, fd, sort_keys=True) def save(self): super(Output, self).save() diff --git a/dvc/state.py b/dvc/state.py index 38011873b..a1df5015a 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -1,11 +1,11 @@ import os import json import nanotime -from checksumdir import dirhash +import threading from dvc.system import System from dvc.output import Output -from dvc.utils import file_md5 +from dvc.utils import file_md5, bytes_md5 from dvc.exceptions import DvcException @@ -46,19 +46,43 @@ class State(object): self.dvc_dir = dvc_dir self.state_file = os.path.join(dvc_dir, self.STATE_FILE) self._db = self.load() + self._lock = threading.Lock() @staticmethod def init(root_dir, dvc_dir): return State(root_dir, dvc_dir) + def collect_dir(self, dname): + dir_info = [] + + for root, dirs, files in os.walk(dname): + for fname in files: + path = os.path.join(root, fname) + relpath = os.path.relpath(path, dname) + + md5 = self.update(path, dump=False) + dir_info.append({Output.PARAM_RELPATH: relpath, Output.PARAM_MD5: md5}) + + self.dump() + + return dir_info + def compute_md5(self, path): if os.path.isdir(path): - return dirhash(path, hashfunc='md5') + Output.MD5_DIR_SUFFIX + dir_info = self.collect_dir(path) + byts = json.dumps(dir_info, sort_keys=True).encode('utf-8') + md5 = bytes_md5(byts) + return md5 + Output.MD5_DIR_SUFFIX else: return file_md5(path)[0] def changed(self, path, md5): - return self.update(path) != md5 + actual = self.update(path) + + if not md5 or not actual: + return True + + return actual.split('.')[0] != md5.split('.')[0] def load(self): if not os.path.isfile(self.state_file): @@ -90,15 +114,18 @@ class State(object): md5 = self.compute_md5(path) state = StateEntry(md5, mtime) d = state.dumpd() - self._db[inode] = d - if dump: - self.dump() + with self._lock: + self._db[inode] = d + + if dump: + self.dump() return md5 def _get(self, inode, mtime): - d = self._db.get(inode, None) + with self._lock: + d = self._db.get(inode, None) if not d: return None diff --git a/dvc/utils.py b/dvc/utils.py index 56270352f..395acc52f 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -28,6 +28,12 @@ def file_md5(fname): return (None, None) +def bytes_md5(byts): + hasher = hashlib.md5() + hasher.update(byts) + return hasher.hexdigest() + + def cached_property(func): '''A decorator for caching properties in classes.''' def get(self): diff --git a/requirements.txt b/requirements.txt index 7914e1987..286627bef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,6 @@ networkx>=1.11 pyyaml>=3.12 gitpython>=2.1.8 ntfsutils>=0.1.4 -checksumdir>=1.1.4 setuptools>=34.0.0 nanotime>=0.5.2 pyasn1>=0.4.1 diff --git a/setup.py b/setup.py index 2e9289c2f..0a59e02e6 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ install_requires = [ "pyyaml>=3.12", "gitpython>=2.1.8", "ntfsutils>=0.1.4", - "checksumdir>=1.1.4", "setuptools>=34.0.0", "nanotime>=0.5.2", "pyasn1>=0.4.1", diff --git a/tests/test_add.py b/tests/test_add.py index ad025d12e..01b5aa665 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -2,7 +2,6 @@ import os import stat import shutil import filecmp -from checksumdir import dirhash from dvc.main import main from dvc.utils import file_md5