diff --git a/dvc/utils.py b/dvc/utils.py index ea4ba37da..cd724cde5 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -20,7 +20,7 @@ LOCAL_CHUNK_SIZE = 1024*1024 def dos2unix(data): - return '\n'.join(data.split('\r\n')) + return data.replace(b'\r\n', b'\n') def file_md5(fname): @@ -35,12 +35,7 @@ def file_md5(fname): else: binary = is_binary(fname) - if binary: - mode = "rb" - else: - mode = "r" - - with open(fname, mode) as fobj: + with open(fname, 'rb') as fobj: while True: data = fobj.read(LOCAL_CHUNK_SIZE) if not data: @@ -49,9 +44,7 @@ def file_md5(fname): if binary: chunk = data else: - if sys.version_info[0] == 2: - data = data.decode('utf-8') - chunk = dos2unix(data).encode('utf-8') + chunk = dos2unix(data) hash_md5.update(chunk) diff --git a/tests/test_utils.py b/tests/test_utils.py index b1344d89c..4b4527474 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -37,3 +37,11 @@ class TestUtils(TestDvc): for n in n_threads: utils.map_progress(f, targets, n) + + def test_file_md5_crlf(self): + with open('cr', 'wb+') as fd: + fd.write(b'a\nb\nc') + with open('crlf', 'wb+') as fd: + fd.write(b'a\r\nb\r\nc') + + self.assertEqual(utils.file_md5('cr')[0], utils.file_md5('crlf')[0])