Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

remote: Show progress bar when linking large dirs (#1309)

Browse Source 
* remote: Show progress bar when linking large dirs

Close #1222

* Fix pycodestyle

* Use Logger.debug instead of local verbose argument

* Show the progress bar also during checkout
This commit is contained in:
Mr. Outis
2018-11-10 23:56:50 +02:00
committed by Ruslan Kuprieiev
parent 31585c8d1a
commit 4d03f69a86
1 changed files with 44 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/remote/local.py
+44 -12
View File
@@ -132,9 +132,13 @@ class RemoteLOCAL(RemoteBase):
# NOTE: just create an empty file for an empty cache
if os.path.getsize(cache) == 0:
open(path, 'w+').close()
msg = "Created empty file: {} -> {}"
self.project.logger.info(msg.format(os.path.relpath(cache),
os.path.relpath(path)))
msg = "Created empty file: {} -> {}".format(
os.path.relpath(cache),
os.path.relpath(path),
)
Logger.debug(msg)
return
i = len(self.cache_types)
@@ -145,11 +149,14 @@ class RemoteLOCAL(RemoteBase):
if self.protected:
os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
msg = "Created {}'{}': {} -> {}"
Logger.info(msg.format('protected ' if self.protected else '',
self.cache_types[0],
os.path.relpath(cache),
os.path.relpath(path)))
msg = "Created {}'{}': {} -> {}".format(
'protected ' if self.protected else '',
self.cache_types[0],
os.path.relpath(cache),
os.path.relpath(path)
)
Logger.debug(msg)
return
except DvcException as exc:
msg = 'Cache type \'{}\' is not supported: {}'
@@ -300,14 +307,28 @@ class RemoteLOCAL(RemoteBase):
if not os.path.exists(path):
os.makedirs(path)
for entry in self.load_dir_cache(md5):
md5 = entry[self.PARAM_MD5]
c = self.get(md5)
dir_info = self.load_dir_cache(md5)
dir_relpath = os.path.relpath(path)
dir_size = len(dir_info)
bar = dir_size > LARGE_DIR_SIZE
Logger.info("Linking directory '{}'.".format(dir_relpath))
for processed, entry in enumerate(dir_info):
relpath = entry[self.PARAM_RELPATH]
m = entry[self.PARAM_MD5]
p = os.path.join(path, relpath)
c = self.get(m)
self.link(c, p)
if bar:
progress.update_target(dir_relpath, processed, dir_size)
self.state.update_link(path)
if bar:
progress.finish_target(dir_relpath)
def _move(self, inp, outp):
# moving in two stages to make last the move atomic in
# case inp and outp are in different filesystems
@@ -335,8 +356,13 @@ class RemoteLOCAL(RemoteBase):
def _save_dir(self, path_info):
path = path_info['path']
md5, dir_info = self.state.update_info(path)
dir_relpath = os.path.relpath(path)
dir_size = len(dir_info)
bar = dir_size > LARGE_DIR_SIZE
for entry in dir_info:
Logger.info("Linking directory '{}'.".format(dir_relpath))
for processed, entry in enumerate(dir_info):
relpath = entry[self.PARAM_RELPATH]
m = entry[self.PARAM_MD5]
p = os.path.join(path, relpath)
@@ -349,8 +375,14 @@ class RemoteLOCAL(RemoteBase):
self.link(c, p)
if bar:
progress.update_target(dir_relpath, processed, dir_size)
self.state.update_link(path)
if bar:
progress.finish_target(dir_relpath)
return {self.PARAM_MD5: md5}
def save(self, path_info):