mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
utils: convert to unix format when calculating md5
Fixes #619 Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
+24
-3
@@ -7,7 +7,7 @@ import json
|
||||
import stat
|
||||
import shutil
|
||||
import hashlib
|
||||
|
||||
from binaryornot.check import is_binary
|
||||
from multiprocessing.pool import ThreadPool
|
||||
|
||||
from dvc.progress import progress
|
||||
@@ -17,13 +17,34 @@ from dvc.logger import Logger
|
||||
LOCAL_CHUNK_SIZE = 1024*1024
|
||||
|
||||
|
||||
def dos2unix(data):
|
||||
return '\n'.join(data.split('\r\n'))
|
||||
|
||||
|
||||
def file_md5(fname):
|
||||
""" get the (md5 hexdigest, md5 digest) of a file """
|
||||
if os.path.exists(fname):
|
||||
hash_md5 = hashlib.md5()
|
||||
with open(fname, "rb") as fobj:
|
||||
for chunk in iter(lambda: fobj.read(1024*1000), b""):
|
||||
binary = is_binary(fname)
|
||||
|
||||
if binary:
|
||||
mode = "rb"
|
||||
else:
|
||||
mode = "r"
|
||||
|
||||
with open(fname, mode) as fobj:
|
||||
while True:
|
||||
data = fobj.read(LOCAL_CHUNK_SIZE)
|
||||
if not data:
|
||||
break
|
||||
|
||||
if binary:
|
||||
chunk = data
|
||||
else:
|
||||
chunk = dos2unix(data).encode('utf-8')
|
||||
|
||||
hash_md5.update(chunk)
|
||||
|
||||
return (hash_md5.hexdigest(), hash_md5.digest())
|
||||
else:
|
||||
return (None, None)
|
||||
|
||||
@@ -16,3 +16,4 @@ nanotime>=0.5.2
|
||||
pyasn1>=0.4.1
|
||||
schema>=0.6.7
|
||||
paramiko>=2.4.1
|
||||
binaryornot==0.4.4
|
||||
|
||||
Reference in New Issue
Block a user