Merge pull request #2703 from kurianbenoy/master

Remove multi-threaded prefix before progress bar(#2686)
This commit is contained in:
Ruslan Kuprieiev
2019-11-03 00:34:19 +02:00
committed by GitHub
2 changed files with 3 additions and 25 deletions
-23
View File
@@ -3,35 +3,12 @@ from __future__ import print_function
import logging
import sys
from tqdm import tqdm
from concurrent.futures import ThreadPoolExecutor
from funcy import merge
from dvc.utils import env2bool
logger = logging.getLogger(__name__)
class TqdmThreadPoolExecutor(ThreadPoolExecutor):
"""
Ensure worker progressbars are cleared away properly.
"""
def __enter__(self):
"""
Creates a blank initial dummy progress bar if needed so that workers
are forced to create "nested" bars.
"""
blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
if blank_bar.pos > 0:
# already nested - don't need a placeholder bar
blank_bar.close()
self.bar = blank_bar
return super(TqdmThreadPoolExecutor, self).__enter__()
def __exit__(self, *a, **k):
super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
self.bar.close()
class Tqdm(tqdm):
"""
maximum-compatibility tqdm-based progressbars
+3 -2
View File
@@ -4,6 +4,7 @@ import stat
import errno
import logging
from functools import partial
from concurrent.futures import ThreadPoolExecutor
from shortuuid import uuid
@@ -29,7 +30,7 @@ from dvc.utils import (
)
from dvc.config import Config
from dvc.exceptions import DvcException, DownloadError, UploadError
from dvc.progress import Tqdm, TqdmThreadPoolExecutor
from dvc.progress import Tqdm
from dvc.path_info import PathInfo
@@ -359,7 +360,7 @@ class RemoteLOCAL(RemoteBASE):
return 0
if jobs > 1:
with TqdmThreadPoolExecutor(max_workers=jobs) as executor:
with ThreadPoolExecutor(max_workers=jobs) as executor:
fails = sum(executor.map(func, *plans))
else:
fails = sum(map(func, *plans))