mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-21 23:08:59 +01:00
executors: add support for cancel_futures=True in Python<3.9
Also adds a kwarg `cancel_on_error` on the constructor to support cancelling futures on `shutdown`. This is not done by default to be backward compatible with `concurrent.futures.ThreadPool` executor.
This commit is contained in:
committed by
Saugat Pachhai
parent
5c561572b0
commit
6ffba62b75
@@ -1,3 +1,5 @@
|
||||
import queue
|
||||
import sys
|
||||
from concurrent import futures
|
||||
from itertools import islice
|
||||
from typing import Any, Callable, Iterable, Iterator, Set, TypeVar
|
||||
@@ -8,6 +10,12 @@ _T = TypeVar("_T")
|
||||
class ThreadPoolExecutor(futures.ThreadPoolExecutor):
|
||||
_max_workers: int
|
||||
|
||||
def __init__(
|
||||
self, max_workers: int = None, cancel_on_error: bool = False, **kwargs
|
||||
):
|
||||
super().__init__(max_workers=max_workers, **kwargs)
|
||||
self._cancel_on_error = cancel_on_error
|
||||
|
||||
@property
|
||||
def max_workers(self) -> int:
|
||||
return self._max_workers
|
||||
@@ -32,3 +40,35 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor):
|
||||
for fut in done:
|
||||
yield fut.result()
|
||||
tasks.update(create_taskset(len(done)))
|
||||
|
||||
def shutdown(self, wait=True, *, cancel_futures=False):
|
||||
if sys.version_info > (3, 9):
|
||||
# pylint: disable=unexpected-keyword-arg
|
||||
return super().shutdown(wait=wait, cancel_futures=cancel_futures)
|
||||
|
||||
with self._shutdown_lock:
|
||||
self._shutdown = True
|
||||
if cancel_futures:
|
||||
# Drain all work items from the queue, and then cancel their
|
||||
# associated futures.
|
||||
while True:
|
||||
try:
|
||||
work_item = self._work_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
if work_item is not None:
|
||||
work_item.future.cancel()
|
||||
|
||||
# Send a wake-up to prevent threads calling
|
||||
# _work_queue.get(block=True) from permanently blocking.
|
||||
self._work_queue.put(None)
|
||||
if wait:
|
||||
for t in self._threads:
|
||||
t.join()
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self._cancel_on_error:
|
||||
self.shutdown(wait=True, cancel_futures=exc_val is not None)
|
||||
else:
|
||||
self.shutdown(wait=True)
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import operator
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from funcy import raiser
|
||||
|
||||
from dvc.utils.threadpool import ThreadPoolExecutor
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wait", [True, False])
|
||||
@pytest.mark.parametrize("cancel_futures", [True, False])
|
||||
def test_cancel_futures(wait, cancel_futures):
|
||||
"""Modified from
|
||||
https://github.com/python/cpython/blob/4d2403f/Lib/test/test_concurrent_futures.py#L354
|
||||
"""
|
||||
executor = ThreadPoolExecutor(max_workers=2)
|
||||
fs = [executor.submit(time.sleep, 0.1) for _ in range(50)]
|
||||
executor.shutdown(wait=wait, cancel_futures=cancel_futures)
|
||||
|
||||
if not wait:
|
||||
for t in executor._threads:
|
||||
t.join()
|
||||
|
||||
cancelled = [fut for fut in fs if fut.cancelled()]
|
||||
# Use "not fut.cancelled()" instead of "fut.done()" to include futures
|
||||
# that may have been left in a pending state.
|
||||
others = [fut for fut in fs if not fut.cancelled()]
|
||||
|
||||
# Ensure the other futures were able to finish.
|
||||
for fut in others:
|
||||
assert fut.done()
|
||||
assert fut.exception() is None
|
||||
|
||||
if not cancel_futures:
|
||||
# there should be no cancelled futures
|
||||
assert len(cancelled) == 0
|
||||
assert len(others) == len(fs)
|
||||
else:
|
||||
# We can't guarantee the exact number of cancellations, but we can
|
||||
# guarantee that *some* were cancelled. With few workers, many of
|
||||
# the submitted futures should have been cancelled.
|
||||
assert len(cancelled) > 20
|
||||
# Similar to the number of cancelled futures, we can't guarantee the
|
||||
# exact number that completed. But, we can guarantee that at least
|
||||
# one finished.
|
||||
assert len(others) > 0
|
||||
|
||||
|
||||
def test_cancel_on_error_context_manager(mocker):
|
||||
executor = ThreadPoolExecutor(max_workers=2, cancel_on_error=True)
|
||||
spy = mocker.spy(executor, "shutdown")
|
||||
with pytest.raises(RuntimeError), executor:
|
||||
future1 = executor.submit(operator.mul, 2, 21)
|
||||
future2 = executor.submit(time.sleep, 0.1)
|
||||
future3 = executor.submit(raiser(RuntimeError), "This is an error")
|
||||
fs = [executor.submit(time.sleep, 0.1) for _ in range(50)]
|
||||
|
||||
assert future1.result() == 42
|
||||
assert future2.result() is None
|
||||
_ = future3.result()
|
||||
|
||||
spy.assert_called_once_with(wait=True, cancel_futures=True)
|
||||
|
||||
cancelled = [fut for fut in fs if fut.cancelled()]
|
||||
others = [fut for fut in fs if not fut.cancelled()]
|
||||
|
||||
for fut in others:
|
||||
assert fut.done()
|
||||
assert fut.exception() is None
|
||||
|
||||
assert len(cancelled) > 20
|
||||
assert len(others) > 0
|
||||
Reference in New Issue
Block a user