import/sync: add progress bar for local copy

Currently progress bar is only supported for downloading
from web(aws,http* etc), but now it is also supported for
local files(i.e. LOCAL for 'data sync' and just local files
for 'import')

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/command/import_file.py

@@ -1,5 +1,4 @@
 import os
-from shutil import copyfile
 import re
 import requests
 from multiprocessing.pool import ThreadPool
@@ -13,7 +12,7 @@ from dvc.state_file import StateFile
 from dvc.system import System
 from dvc.command.data_sync import POOL_SIZE
 from dvc.progress import progress
-
+from dvc.utils import copyfile
 
 class ImportFileError(DvcException):
     def __init__(self, msg):

dvc/data_cloud.py

@@ -3,7 +3,6 @@ import hashlib
 import os
 import threading
 import configparser
-import shutil
 
 from boto.s3.connection import S3Connection
 from google.cloud import storage as gc
@@ -13,6 +12,7 @@ from dvc.exceptions import DvcException
 from dvc.config import ConfigError
 from dvc.settings import Settings
 from dvc.progress import progress
+from dvc.utils import copyfile
 
 class DataCloudError(DvcException):
     def __init__(self, msg):
@@ -36,7 +36,7 @@ def percent_cb(name, complete, total):
 
 
 def create_cb(name):
-    return (lambda cur,tot: return percent_cb(name, cur, tot))
+    return (lambda cur,tot: percent_cb(name, cur, tot))
 
 
 def file_md5(fname):
@@ -131,11 +131,11 @@ class DataCloudBase(object):
 class DataCloudLOCAL(DataCloudBase):
     def sync_to_cloud(self, item):
         Logger.debug('sync to cloud ' + item.resolved_cache.dvc + " " + self.storage_path)
-        shutil.copy(item.resolved_cache.dvc, self.storage_path)
+        copyfile(item.resolved_cache.dvc, self.storage_path)
 
     def sync_from_cloud(self, item):
         Logger.debug('sync from cloud ' + self.storage_path + " " + item.resolved_cache.dvc)
-        shutil.copy(self.storage_path, item.resolved_cache.dvc)
+        copyfile(self.storage_path, item.resolved_cache.dvc)
 
     def remove_from_cloud(self, item):
         Logger.debug('rm from cloud ' + item.resolved_cache.dvc)

dvc/utils.py

@@ -2,6 +2,10 @@ import os
 import stat
 import shutil
 
+from dvc.progress import progress
+
+LOCAL_CHUNK_SIZE = 1024*1024*1024
+
 def cached_property(f):
     def get(self):
         try:
@@ -24,3 +28,29 @@ def rmtree(dir):
             os.chmod(dir, stat.S_IWUSR)
 
     shutil.rmtree(dir, ignore_errors=True)
+
+
+def copyfile(src, dest):
+    copied = 0
+    name = os.path.basename(src)
+    total = os.stat(src).st_size
+
+    f = open(src, 'rb')
+
+    if os.path.isdir(dest):
+        o = open(dest + '/' + name, 'wb+')
+    else:
+        o = open(dest, 'wb+')
+
+    while True:
+        buf = f.read(LOCAL_CHUNK_SIZE)
+        if not buf:
+            break
+        o.write(buf)
+        copied += len(buf)
+        progress.update_target(name, copied, total)
+
+    progress.finish_target(name)
+
+    f.close()
+    o.close()