diff --git a/dvc/progress.py b/dvc/progress.py index 6de01d021..c6445d85b 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -63,7 +63,10 @@ class Progress(object): progress = int((100 * current)/total) if current < total else 100 percent = str(progress) + "% " - num = "({}/{}): ".format(self._n_finished + 1, self._n_total) + if self._n_total: + num = "({}/{}): ".format(self._n_finished + 1, self._n_total) + else: + num = "" n_sh = int((progress * bar_len)/100) n_sp = bar_len - n_sh diff --git a/dvc/utils.py b/dvc/utils.py index 0eb9c8ace..1956dfbfa 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -7,12 +7,13 @@ import json import shutil import hashlib -from dvc.istextfile import istextfile from dvc.progress import progress +from dvc.istextfile import istextfile from dvc.logger import Logger LOCAL_CHUNK_SIZE = 1024*1024 +LARGE_FILE_SIZE = 1024*1024*1024 def dos2unix(data): @@ -24,6 +25,14 @@ def file_md5(fname): if os.path.exists(fname): hash_md5 = hashlib.md5() binary = not istextfile(fname) + size = os.path.getsize(fname) + bar = False + if size >= LARGE_FILE_SIZE: + bar = True + msg = "Computing md5 for a large file {}. This is only done once." + Logger.info(msg.format(os.path.relpath(fname))) + name = os.path.relpath(fname) + total = 0 with open(fname, 'rb') as fobj: while True: @@ -31,6 +40,10 @@ def file_md5(fname): if not data: break + if bar: + total += len(data) + progress.update_target(name, total, size) + if binary: chunk = data else: @@ -38,6 +51,9 @@ def file_md5(fname): hash_md5.update(chunk) + if bar: + progress.finish_target(name) + return (hash_md5.hexdigest(), hash_md5.digest()) else: return (None, None)