mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-10 07:18:27 +01:00
Merge pull request #2703 from kurianbenoy/master
Remove multi-threaded prefix before progress bar(#2686)
This commit is contained in:
@@ -3,35 +3,12 @@ from __future__ import print_function
|
||||
import logging
|
||||
import sys
|
||||
from tqdm import tqdm
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from funcy import merge
|
||||
from dvc.utils import env2bool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TqdmThreadPoolExecutor(ThreadPoolExecutor):
|
||||
"""
|
||||
Ensure worker progressbars are cleared away properly.
|
||||
"""
|
||||
|
||||
def __enter__(self):
|
||||
"""
|
||||
Creates a blank initial dummy progress bar if needed so that workers
|
||||
are forced to create "nested" bars.
|
||||
"""
|
||||
blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
|
||||
if blank_bar.pos > 0:
|
||||
# already nested - don't need a placeholder bar
|
||||
blank_bar.close()
|
||||
self.bar = blank_bar
|
||||
return super(TqdmThreadPoolExecutor, self).__enter__()
|
||||
|
||||
def __exit__(self, *a, **k):
|
||||
super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
|
||||
self.bar.close()
|
||||
|
||||
|
||||
class Tqdm(tqdm):
|
||||
"""
|
||||
maximum-compatibility tqdm-based progressbars
|
||||
|
||||
+3
-2
@@ -4,6 +4,7 @@ import stat
|
||||
import errno
|
||||
import logging
|
||||
from functools import partial
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from shortuuid import uuid
|
||||
|
||||
@@ -29,7 +30,7 @@ from dvc.utils import (
|
||||
)
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import DvcException, DownloadError, UploadError
|
||||
from dvc.progress import Tqdm, TqdmThreadPoolExecutor
|
||||
from dvc.progress import Tqdm
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
@@ -359,7 +360,7 @@ class RemoteLOCAL(RemoteBASE):
|
||||
return 0
|
||||
|
||||
if jobs > 1:
|
||||
with TqdmThreadPoolExecutor(max_workers=jobs) as executor:
|
||||
with ThreadPoolExecutor(max_workers=jobs) as executor:
|
||||
fails = sum(executor.map(func, *plans))
|
||||
else:
|
||||
fails = sum(map(func, *plans))
|
||||
|
||||
Reference in New Issue
Block a user