diff --git a/dvc/utils.py b/dvc/utils.py index ef4d63f75..d0c707a14 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -7,7 +7,7 @@ import json import stat import shutil import hashlib - +from binaryornot.check import is_binary from multiprocessing.pool import ThreadPool from dvc.progress import progress @@ -17,13 +17,34 @@ from dvc.logger import Logger LOCAL_CHUNK_SIZE = 1024*1024 +def dos2unix(data): + return '\n'.join(data.split('\r\n')) + + def file_md5(fname): """ get the (md5 hexdigest, md5 digest) of a file """ if os.path.exists(fname): hash_md5 = hashlib.md5() - with open(fname, "rb") as fobj: - for chunk in iter(lambda: fobj.read(1024*1000), b""): + binary = is_binary(fname) + + if binary: + mode = "rb" + else: + mode = "r" + + with open(fname, mode) as fobj: + while True: + data = fobj.read(LOCAL_CHUNK_SIZE) + if not data: + break + + if binary: + chunk = data + else: + chunk = dos2unix(data).encode('utf-8') + hash_md5.update(chunk) + return (hash_md5.hexdigest(), hash_md5.digest()) else: return (None, None) diff --git a/requirements.txt b/requirements.txt index e8077d2ca..3ef8074dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ nanotime>=0.5.2 pyasn1>=0.4.1 schema>=0.6.7 paramiko>=2.4.1 +binaryornot==0.4.4 diff --git a/setup.py b/setup.py index 9c2e01d75..9cbdc601b 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ install_requires = [ "pyasn1>=0.4.1", "schema>=0.6.7", "paramiko>=2.4.1", + "binaryornot==0.4.4", ] setup(