Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

progress: add main bars (#3594)

Browse Source 
* progress: add main push/pull bar

Fixes #3452
Related #1840
Related #3565

* adjust desc

* fix merge

Co-authored-by: Ruslan Kuprieiev <[email protected]>
Co-authored-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Casper da Costa-Luis
2020-04-25 22:04:47 +03:00
committed by GitHub
co-authored by Ruslan Kuprieiev Ruslan Kuprieiev
parent e2981d2743
commit b312895e77
1 changed files with 37 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/remote/local.py
+37 -32
View File
@@ -448,9 +448,11 @@ class RemoteLOCAL(RemoteBASE):
file_mode=self._file_mode, file_mode=self._file_mode,
) )
status = STATUS_DELETED status = STATUS_DELETED
desc = "Downloading"
else: else:
func = remote.upload func = remote.upload
status = STATUS_NEW status = STATUS_NEW
desc = "Uploading"
if jobs is None: if jobs is None:
jobs = remote.JOBS jobs = remote.JOBS
@@ -466,42 +468,45 @@ class RemoteLOCAL(RemoteBASE):
dir_plans = self._get_plans(download, remote, dir_status, status) dir_plans = self._get_plans(download, remote, dir_status, status)
file_plans = self._get_plans(download, remote, file_status, status) file_plans = self._get_plans(download, remote, file_status, status)
if len(dir_plans[0]) + len(file_plans[0]) == 0: total = len(dir_plans[0]) + len(file_plans[0])
if total == 0:
return 0 return 0
with ThreadPoolExecutor(max_workers=jobs) as executor: with Tqdm(total=total, unit="file", desc=desc) as pbar:
if download: func = pbar.wrap_fn(func)
fails = sum(executor.map(func, *dir_plans)) with ThreadPoolExecutor(max_workers=jobs) as executor:
fails += sum(executor.map(func, *file_plans)) if download:
else: fails = sum(executor.map(func, *dir_plans))
# for uploads, push files first, and any .dir files last fails += sum(executor.map(func, *file_plans))
else:
# for uploads, push files first, and any .dir files last
file_futures = {} file_futures = {}
for from_info, to_info, name in zip(*file_plans): for from_info, to_info, name in zip(*file_plans):
file_futures[to_info] = executor.submit( file_futures[to_info] = executor.submit(
func, from_info, to_info, name func, from_info, to_info, name
)
dir_futures = {}
for from_info, to_info, name in zip(*dir_plans):
wait_futures = {
future
for file_path, future in file_futures.items()
if file_path in dir_paths[to_info]
}
dir_futures[to_info] = executor.submit(
self._dir_upload,
func,
wait_futures,
from_info,
to_info,
name,
)
fails = sum(
future.result()
for future in concat(
file_futures.values(), dir_futures.values()
)
) )
dir_futures = {}
for from_info, to_info, name in zip(*dir_plans):
wait_futures = {
future
for file_path, future in file_futures.items()
if file_path in dir_paths[to_info]
}
dir_futures[to_info] = executor.submit(
self._dir_upload,
func,
wait_futures,
from_info,
to_info,
name,
)
fails = sum(
future.result()
for future in concat(
file_futures.values(), dir_futures.values()
)
)
if fails: if fails:
if download: if download: