Merge pull request #775 from efiop/master

utils: read in binary mode when converting CRLF -> LF

dvc/utils.py

@@ -20,7 +20,7 @@ LOCAL_CHUNK_SIZE = 1024*1024
 
 
 def dos2unix(data):
-    return '\n'.join(data.split('\r\n'))
+    return data.replace(b'\r\n', b'\n')
 
 
 def file_md5(fname):
@@ -35,12 +35,7 @@ def file_md5(fname):
         else:
             binary = is_binary(fname)
 
-        if binary:
-            mode = "rb"
-        else:
-            mode = "r"
-
-        with open(fname, mode) as fobj:
+        with open(fname, 'rb') as fobj:
             while True:
                 data = fobj.read(LOCAL_CHUNK_SIZE)
                 if not data:
@@ -49,9 +44,7 @@ def file_md5(fname):
                 if binary:
                     chunk = data
                 else:
-                    if sys.version_info[0] == 2:
-                        data = data.decode('utf-8')
-                    chunk = dos2unix(data).encode('utf-8')
+                    chunk = dos2unix(data)
 
                 hash_md5.update(chunk)
 

tests/test_utils.py

@@ -37,3 +37,11 @@ class TestUtils(TestDvc):
 
         for n in n_threads:
             utils.map_progress(f, targets, n)
+
+    def test_file_md5_crlf(self):
+        with open('cr', 'wb+') as fd:
+            fd.write(b'a\nb\nc')
+        with open('crlf', 'wb+') as fd:
+            fd.write(b'a\r\nb\r\nc')
+
+        self.assertEqual(utils.file_md5('cr')[0], utils.file_md5('crlf')[0])