From 17a301d73b593f85be2526e2d1036e90a221e96c Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 14 Apr 2018 21:57:57 +0300 Subject: [PATCH 1/4] appveyor: don't use --user when upgrading pip Signed-off-by: Ruslan Kuprieiev --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 1662210da..f8e5f6263 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -48,7 +48,7 @@ install: - cinst awscli - cinst gsutil - cinst openssl.light - - pip install --user --upgrade pip + - python -m pip install -U pip setuptools wheel - pip install -r requirements.txt - python setup.py install From fed22d3ab917f22e12ef90b9bed22491bd52c0b3 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 14 Apr 2018 08:14:13 +0300 Subject: [PATCH 2/4] utils: use dict_md5 Signed-off-by: Ruslan Kuprieiev --- dvc/state.py | 6 ++---- dvc/utils.py | 6 ++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dvc/state.py b/dvc/state.py index 13e54d9bb..4d32bc1d6 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -5,7 +5,7 @@ import threading from dvc.system import System from dvc.output import Output -from dvc.utils import file_md5, bytes_md5 +from dvc.utils import file_md5, dict_md5 from dvc.exceptions import DvcException @@ -70,9 +70,7 @@ class State(object): def compute_md5(self, path): if os.path.isdir(path): dir_info = self.collect_dir(path) - byts = json.dumps(dir_info, sort_keys=True).encode('utf-8') - md5 = bytes_md5(byts) - return md5 + Output.MD5_DIR_SUFFIX + return dict_md5(dir_info) + Output.MD5_DIR_SUFFIX else: return file_md5(path)[0] diff --git a/dvc/utils.py b/dvc/utils.py index b3196dac5..ef4d63f75 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -3,6 +3,7 @@ Helpers for other modules. """ import os import re +import json import stat import shutil import hashlib @@ -34,6 +35,11 @@ def bytes_md5(byts): return hasher.hexdigest() +def dict_md5(d): + byts = json.dumps(d, sort_keys=True).encode('utf-8') + return bytes_md5(byts) + + def cached_property(func): '''A decorator for caching properties in classes.''' def get(self): From 08da4a58d0d20eee5ace96c6c2c25b7170dc4692 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 14 Apr 2018 08:47:32 +0300 Subject: [PATCH 3/4] stage: use 'md5' field to validate stage file as a whole Fixes #643 Signed-off-by: Ruslan Kuprieiev --- dvc/stage.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/dvc/stage.py b/dvc/stage.py index 2cbc6879d..96f806dac 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -7,6 +7,7 @@ import schema from dvc.exceptions import DvcException from dvc.output import Output, Dependency, OutputError from dvc.logger import Logger +from dvc.utils import dict_md5 class StageCmdFailedError(DvcException): @@ -36,23 +37,26 @@ class Stage(object): STAGE_FILE = 'Dvcfile' STAGE_FILE_SUFFIX = '.dvc' + PARAM_MD5 = 'md5' PARAM_CMD = 'cmd' PARAM_DEPS = 'deps' PARAM_OUTS = 'outs' SCHEMA = { + schema.Optional(PARAM_MD5): schema.Or(str, None), schema.Optional(PARAM_CMD): schema.Or(str, None), schema.Optional(PARAM_DEPS): schema.Or(schema.And(list, schema.Schema([Dependency.SCHEMA])), None), schema.Optional(PARAM_OUTS): schema.Or(schema.And(list, schema.Schema([Output.SCHEMA])), None), } - def __init__(self, project, path=None, cmd=None, cwd=None, deps=[], outs=[]): + def __init__(self, project, path=None, cmd=None, cwd=None, deps=[], outs=[], md5=None): self.project = project self.path = path self.cmd = cmd self.cwd = cwd self.outs = outs self.deps = deps + self.md5 = md5 @property def relpath(self): @@ -72,12 +76,29 @@ class Stage(object): return True + def changed_md5(self): + md5 = self.dumpd().get(self.PARAM_MD5, None) + + # backward compatibility + if self.md5 == None: + return False + + if self.md5 and md5 and self.md5 == md5: + return False + + msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')" + self.project.logger.debug(msg.format(self.relpath, self.md5, md5)) + return True + def changed(self): ret = False for entry in itertools.chain(self.outs, self.deps): if entry.changed(): ret = True + if self.changed_md5(): + ret = True + if ret: self.project.logger.debug(u'Dvc file \'{}\' changed'.format(self.relpath)) else: @@ -124,13 +145,16 @@ class Stage(object): cmd = d.get(Stage.PARAM_CMD, None) deps = Dependency.loadd_from(project, d.get(Stage.PARAM_DEPS, []), cwd=cwd) outs = Output.loadd_from(project, d.get(Stage.PARAM_OUTS, []), cwd=cwd) + md5 = d.get(Stage.PARAM_MD5, None) return Stage(project=project, path=path, cmd=cmd, cwd=cwd, deps=deps, - outs=outs) + outs=outs, + md5=md5) + @staticmethod def loads(project=None, cmd=None, @@ -171,6 +195,8 @@ class Stage(object): if len(outs): ret[Stage.PARAM_OUTS] = outs + ret[Stage.PARAM_MD5] = dict_md5(ret) + return ret def dump(self, fname=None): @@ -231,7 +257,7 @@ class Stage(object): ret.update(self._status(self.deps, 'deps')) ret.update(self._status(self.outs, 'outs')) - if ret: + if ret or self.changed_md5(): return {self.relpath: ret} return {} From f30e6e53c56003a65b9e93e59cf4801c8a371421 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 14 Apr 2018 10:39:28 +0300 Subject: [PATCH 4/4] utils: convert to unix format when calculating md5 Fixes #619 Signed-off-by: Ruslan Kuprieiev --- dvc/utils.py | 27 ++++++++++++++++++++++++--- requirements.txt | 1 + setup.py | 1 + 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/dvc/utils.py b/dvc/utils.py index ef4d63f75..d0c707a14 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -7,7 +7,7 @@ import json import stat import shutil import hashlib - +from binaryornot.check import is_binary from multiprocessing.pool import ThreadPool from dvc.progress import progress @@ -17,13 +17,34 @@ from dvc.logger import Logger LOCAL_CHUNK_SIZE = 1024*1024 +def dos2unix(data): + return '\n'.join(data.split('\r\n')) + + def file_md5(fname): """ get the (md5 hexdigest, md5 digest) of a file """ if os.path.exists(fname): hash_md5 = hashlib.md5() - with open(fname, "rb") as fobj: - for chunk in iter(lambda: fobj.read(1024*1000), b""): + binary = is_binary(fname) + + if binary: + mode = "rb" + else: + mode = "r" + + with open(fname, mode) as fobj: + while True: + data = fobj.read(LOCAL_CHUNK_SIZE) + if not data: + break + + if binary: + chunk = data + else: + chunk = dos2unix(data).encode('utf-8') + hash_md5.update(chunk) + return (hash_md5.hexdigest(), hash_md5.digest()) else: return (None, None) diff --git a/requirements.txt b/requirements.txt index e8077d2ca..3ef8074dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ nanotime>=0.5.2 pyasn1>=0.4.1 schema>=0.6.7 paramiko>=2.4.1 +binaryornot==0.4.4 diff --git a/setup.py b/setup.py index 9c2e01d75..9cbdc601b 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ install_requires = [ "pyasn1>=0.4.1", "schema>=0.6.7", "paramiko>=2.4.1", + "binaryornot==0.4.4", ] setup(