mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
remove Python 3.8 compat for threadpoolexecutor.shutdown(cancel_futures=True) (#10266)
This commit is contained in:
+3
-37
@@ -1,5 +1,3 @@
|
||||
import queue
|
||||
import sys
|
||||
from collections.abc import Iterable, Iterator
|
||||
from concurrent import futures
|
||||
from itertools import islice
|
||||
@@ -9,8 +7,6 @@ _T = TypeVar("_T")
|
||||
|
||||
|
||||
class ThreadPoolExecutor(futures.ThreadPoolExecutor):
|
||||
_max_workers: int
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_workers: Optional[int] = None,
|
||||
@@ -20,10 +16,6 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor):
|
||||
super().__init__(max_workers=max_workers, **kwargs)
|
||||
self._cancel_on_error = cancel_on_error
|
||||
|
||||
@property
|
||||
def max_workers(self) -> int:
|
||||
return self._max_workers
|
||||
|
||||
def imap_unordered(
|
||||
self, fn: Callable[..., _T], *iterables: Iterable[Any]
|
||||
) -> Iterator[_T]:
|
||||
@@ -36,40 +28,14 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor):
|
||||
return {self.submit(fn, *args) for args in islice(it, n)}
|
||||
|
||||
it = zip(*iterables)
|
||||
tasks = create_taskset(self.max_workers * 5)
|
||||
tasks = create_taskset(self._max_workers * 5)
|
||||
while tasks:
|
||||
done, tasks = futures.wait(tasks, return_when=futures.FIRST_COMPLETED)
|
||||
for fut in done:
|
||||
yield fut.result()
|
||||
tasks.update(create_taskset(len(done)))
|
||||
|
||||
def shutdown(self, wait=True, *, cancel_futures=False):
|
||||
if sys.version_info > (3, 9):
|
||||
return super().shutdown(wait=wait, cancel_futures=cancel_futures)
|
||||
else: # noqa: RET505
|
||||
with self._shutdown_lock:
|
||||
self._shutdown = True
|
||||
if cancel_futures:
|
||||
# Drain all work items from the queue, and then cancel
|
||||
# their associated futures.
|
||||
while True:
|
||||
try:
|
||||
work_item = self._work_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
if work_item is not None:
|
||||
work_item.future.cancel()
|
||||
|
||||
# Send a wake-up to prevent threads calling
|
||||
# _work_queue.get(block=True) from permanently blocking.
|
||||
self._work_queue.put(None) # type: ignore[arg-type]
|
||||
if wait:
|
||||
for t in self._threads:
|
||||
t.join()
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self._cancel_on_error:
|
||||
self.shutdown(wait=True, cancel_futures=exc_val is not None)
|
||||
else:
|
||||
self.shutdown(wait=True)
|
||||
cancel_futures = self._cancel_on_error and exc_val is not None
|
||||
self.shutdown(wait=True, cancel_futures=cancel_futures)
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user