dvc: print info when computing md5 of a large file

Fixes #958

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/progress.py

@@ -63,7 +63,10 @@ class Progress(object):
             progress = int((100 * current)/total) if current < total else 100
             percent = str(progress) + "% "
 
-        num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
+        if self._n_total:
+            num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
+        else:
+            num = ""
 
         n_sh = int((progress * bar_len)/100)
         n_sp = bar_len - n_sh

dvc/utils.py

@@ -7,12 +7,13 @@ import json
 import shutil
 import hashlib
 
-from dvc.istextfile import istextfile
 from dvc.progress import progress
+from dvc.istextfile import istextfile
 from dvc.logger import Logger
 
 
 LOCAL_CHUNK_SIZE = 1024*1024
+LARGE_FILE_SIZE = 1024*1024*1024
 
 
 def dos2unix(data):
@@ -24,6 +25,14 @@ def file_md5(fname):
     if os.path.exists(fname):
         hash_md5 = hashlib.md5()
         binary = not istextfile(fname)
+        size = os.path.getsize(fname)
+        bar = False
+        if size >= LARGE_FILE_SIZE:
+            bar = True
+            msg = "Computing md5 for a large file {}. This is only done once."
+            Logger.info(msg.format(os.path.relpath(fname)))
+            name = os.path.relpath(fname)
+            total = 0
 
         with open(fname, 'rb') as fobj:
             while True:
@@ -31,6 +40,10 @@ def file_md5(fname):
                 if not data:
                     break
 
+                if bar:
+                    total += len(data)
+                    progress.update_target(name, total, size)
+
                 if binary:
                     chunk = data
                 else:
@@ -38,6 +51,9 @@ def file_md5(fname):
 
                 hash_md5.update(chunk)
 
+        if bar:
+            progress.finish_target(name)
+
         return (hash_md5.hexdigest(), hash_md5.digest())
     else:
         return (None, None)