mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-08 23:09:48 +01:00
Ruslan Kuprieiev
27bf57f415
I.e. if network is gone for some reason, report that we're able to sync some targets instead of just throwing exceptions about broken pipe or something. Signed-off-by: Ruslan Kuprieiev <[email protected]>
74 lines
1.6 KiB
Python
74 lines
1.6 KiB
Python
import os
|
|
import stat
|
|
import shutil
|
|
|
|
from multiprocessing.pool import ThreadPool
|
|
|
|
from dvc.progress import progress
|
|
|
|
LOCAL_CHUNK_SIZE = 1024*1024*1024
|
|
|
|
def cached_property(f):
|
|
def get(self):
|
|
try:
|
|
return self._property_cache[f]
|
|
except AttributeError:
|
|
self._property_cache = {}
|
|
x = self._property_cache[f] = f(self)
|
|
return x
|
|
except KeyError:
|
|
x = self._property_cache[f] = f(self)
|
|
return x
|
|
|
|
return property(get)
|
|
|
|
|
|
def rmtree(dir):
|
|
'''Cross platform rmtree()'''
|
|
if os.name == 'nt':
|
|
if os.path.exists(dir) and not os.access(dir, os.W_OK):
|
|
os.chmod(dir, stat.S_IWUSR)
|
|
|
|
shutil.rmtree(dir, ignore_errors=True)
|
|
|
|
|
|
def copyfile(src, dest):
|
|
copied = 0
|
|
name = os.path.basename(src)
|
|
total = os.stat(src).st_size
|
|
|
|
f = open(src, 'rb')
|
|
|
|
if os.path.isdir(dest):
|
|
o = open(dest + '/' + name, 'wb+')
|
|
else:
|
|
o = open(dest, 'wb+')
|
|
|
|
while True:
|
|
buf = f.read(LOCAL_CHUNK_SIZE)
|
|
if not buf:
|
|
break
|
|
o.write(buf)
|
|
copied += len(buf)
|
|
progress.update_target(name, copied, total)
|
|
|
|
progress.finish_target(name)
|
|
|
|
f.close()
|
|
o.close()
|
|
|
|
|
|
def map_progress(func, targets, n_threads):
|
|
"""
|
|
Process targets in multi-threaded mode with progress bar
|
|
"""
|
|
progress.set_n_total(len(targets))
|
|
p = ThreadPool(processes=n_threads)
|
|
|
|
try:
|
|
p.map(func, targets)
|
|
except Exception as exc:
|
|
Logger.error('Unexpected exception while processing targets: {}'.format(exc))
|
|
finally:
|
|
progress.finish()
|