Merge pull request #652 from efiop/master

Misc md5 fixes
This commit is contained in:
Ruslan Kuprieiev
2018-04-14 22:13:15 +03:00
committed by GitHub
6 changed files with 64 additions and 11 deletions
+1 -1
View File
@@ -48,7 +48,7 @@ install:
- cinst awscli
- cinst gsutil
- cinst openssl.light
- pip install --user --upgrade pip
- python -m pip install -U pip setuptools wheel
- pip install -r requirements.txt
- python setup.py install
+29 -3
View File
@@ -7,6 +7,7 @@ import schema
from dvc.exceptions import DvcException
from dvc.output import Output, Dependency, OutputError
from dvc.logger import Logger
from dvc.utils import dict_md5
class StageCmdFailedError(DvcException):
@@ -36,23 +37,26 @@ class Stage(object):
STAGE_FILE = 'Dvcfile'
STAGE_FILE_SUFFIX = '.dvc'
PARAM_MD5 = 'md5'
PARAM_CMD = 'cmd'
PARAM_DEPS = 'deps'
PARAM_OUTS = 'outs'
SCHEMA = {
schema.Optional(PARAM_MD5): schema.Or(str, None),
schema.Optional(PARAM_CMD): schema.Or(str, None),
schema.Optional(PARAM_DEPS): schema.Or(schema.And(list, schema.Schema([Dependency.SCHEMA])), None),
schema.Optional(PARAM_OUTS): schema.Or(schema.And(list, schema.Schema([Output.SCHEMA])), None),
}
def __init__(self, project, path=None, cmd=None, cwd=None, deps=[], outs=[]):
def __init__(self, project, path=None, cmd=None, cwd=None, deps=[], outs=[], md5=None):
self.project = project
self.path = path
self.cmd = cmd
self.cwd = cwd
self.outs = outs
self.deps = deps
self.md5 = md5
@property
def relpath(self):
@@ -72,12 +76,29 @@ class Stage(object):
return True
def changed_md5(self):
md5 = self.dumpd().get(self.PARAM_MD5, None)
# backward compatibility
if self.md5 == None:
return False
if self.md5 and md5 and self.md5 == md5:
return False
msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
return True
def changed(self):
ret = False
for entry in itertools.chain(self.outs, self.deps):
if entry.changed():
ret = True
if self.changed_md5():
ret = True
if ret:
self.project.logger.debug(u'Dvc file \'{}\' changed'.format(self.relpath))
else:
@@ -124,13 +145,16 @@ class Stage(object):
cmd = d.get(Stage.PARAM_CMD, None)
deps = Dependency.loadd_from(project, d.get(Stage.PARAM_DEPS, []), cwd=cwd)
outs = Output.loadd_from(project, d.get(Stage.PARAM_OUTS, []), cwd=cwd)
md5 = d.get(Stage.PARAM_MD5, None)
return Stage(project=project,
path=path,
cmd=cmd,
cwd=cwd,
deps=deps,
outs=outs)
outs=outs,
md5=md5)
@staticmethod
def loads(project=None,
cmd=None,
@@ -171,6 +195,8 @@ class Stage(object):
if len(outs):
ret[Stage.PARAM_OUTS] = outs
ret[Stage.PARAM_MD5] = dict_md5(ret)
return ret
def dump(self, fname=None):
@@ -231,7 +257,7 @@ class Stage(object):
ret.update(self._status(self.deps, 'deps'))
ret.update(self._status(self.outs, 'outs'))
if ret:
if ret or self.changed_md5():
return {self.relpath: ret}
return {}
+2 -4
View File
@@ -5,7 +5,7 @@ import threading
from dvc.system import System
from dvc.output import Output
from dvc.utils import file_md5, bytes_md5
from dvc.utils import file_md5, dict_md5
from dvc.exceptions import DvcException
@@ -70,9 +70,7 @@ class State(object):
def compute_md5(self, path):
if os.path.isdir(path):
dir_info = self.collect_dir(path)
byts = json.dumps(dir_info, sort_keys=True).encode('utf-8')
md5 = bytes_md5(byts)
return md5 + Output.MD5_DIR_SUFFIX
return dict_md5(dir_info) + Output.MD5_DIR_SUFFIX
else:
return file_md5(path)[0]
+30 -3
View File
@@ -3,10 +3,11 @@ Helpers for other modules.
"""
import os
import re
import json
import stat
import shutil
import hashlib
from binaryornot.check import is_binary
from multiprocessing.pool import ThreadPool
from dvc.progress import progress
@@ -16,13 +17,34 @@ from dvc.logger import Logger
LOCAL_CHUNK_SIZE = 1024*1024
def dos2unix(data):
return '\n'.join(data.split('\r\n'))
def file_md5(fname):
""" get the (md5 hexdigest, md5 digest) of a file """
if os.path.exists(fname):
hash_md5 = hashlib.md5()
with open(fname, "rb") as fobj:
for chunk in iter(lambda: fobj.read(1024*1000), b""):
binary = is_binary(fname)
if binary:
mode = "rb"
else:
mode = "r"
with open(fname, mode) as fobj:
while True:
data = fobj.read(LOCAL_CHUNK_SIZE)
if not data:
break
if binary:
chunk = data
else:
chunk = dos2unix(data).encode('utf-8')
hash_md5.update(chunk)
return (hash_md5.hexdigest(), hash_md5.digest())
else:
return (None, None)
@@ -34,6 +56,11 @@ def bytes_md5(byts):
return hasher.hexdigest()
def dict_md5(d):
byts = json.dumps(d, sort_keys=True).encode('utf-8')
return bytes_md5(byts)
def cached_property(func):
'''A decorator for caching properties in classes.'''
def get(self):
+1
View File
@@ -16,3 +16,4 @@ nanotime>=0.5.2
pyasn1>=0.4.1
schema>=0.6.7
paramiko>=2.4.1
binaryornot==0.4.4
+1
View File
@@ -22,6 +22,7 @@ install_requires = [
"pyasn1>=0.4.1",
"schema>=0.6.7",
"paramiko>=2.4.1",
"binaryornot==0.4.4",
]
setup(