utils: convert to unix format when calculating md5

Fixes #619

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-04-14 22:07:29 +03:00
parent 08da4a58d0
commit f30e6e53c5
3 changed files with 26 additions and 3 deletions
+24 -3
View File
@@ -7,7 +7,7 @@ import json
import stat
import shutil
import hashlib
from binaryornot.check import is_binary
from multiprocessing.pool import ThreadPool
from dvc.progress import progress
@@ -17,13 +17,34 @@ from dvc.logger import Logger
LOCAL_CHUNK_SIZE = 1024*1024
def dos2unix(data):
return '\n'.join(data.split('\r\n'))
def file_md5(fname):
""" get the (md5 hexdigest, md5 digest) of a file """
if os.path.exists(fname):
hash_md5 = hashlib.md5()
with open(fname, "rb") as fobj:
for chunk in iter(lambda: fobj.read(1024*1000), b""):
binary = is_binary(fname)
if binary:
mode = "rb"
else:
mode = "r"
with open(fname, mode) as fobj:
while True:
data = fobj.read(LOCAL_CHUNK_SIZE)
if not data:
break
if binary:
chunk = data
else:
chunk = dos2unix(data).encode('utf-8')
hash_md5.update(chunk)
return (hash_md5.hexdigest(), hash_md5.digest())
else:
return (None, None)
+1
View File
@@ -16,3 +16,4 @@ nanotime>=0.5.2
pyasn1>=0.4.1
schema>=0.6.7
paramiko>=2.4.1
binaryornot==0.4.4
+1
View File
@@ -22,6 +22,7 @@ install_requires = [
"pyasn1>=0.4.1",
"schema>=0.6.7",
"paramiko>=2.4.1",
"binaryornot==0.4.4",
]
setup(