executors: add support for cancel_futures=True in Python<3.9

Also adds a kwarg `cancel_on_error` on the constructor to support
cancelling futures on `shutdown`. This is not done by default to
be backward compatible with `concurrent.futures.ThreadPool` executor.
This commit is contained in:
Saugat Pachhai (सौगात)
2022-04-21 14:03:10 +05:45
committed by Saugat Pachhai
parent 5c561572b0
commit 6ffba62b75
2 changed files with 112 additions and 0 deletions
+40
View File
@@ -1,3 +1,5 @@
import queue
import sys
from concurrent import futures
from itertools import islice
from typing import Any, Callable, Iterable, Iterator, Set, TypeVar
@@ -8,6 +10,12 @@ _T = TypeVar("_T")
class ThreadPoolExecutor(futures.ThreadPoolExecutor):
_max_workers: int
def __init__(
self, max_workers: int = None, cancel_on_error: bool = False, **kwargs
):
super().__init__(max_workers=max_workers, **kwargs)
self._cancel_on_error = cancel_on_error
@property
def max_workers(self) -> int:
return self._max_workers
@@ -32,3 +40,35 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor):
for fut in done:
yield fut.result()
tasks.update(create_taskset(len(done)))
def shutdown(self, wait=True, *, cancel_futures=False):
if sys.version_info > (3, 9):
# pylint: disable=unexpected-keyword-arg
return super().shutdown(wait=wait, cancel_futures=cancel_futures)
with self._shutdown_lock:
self._shutdown = True
if cancel_futures:
# Drain all work items from the queue, and then cancel their
# associated futures.
while True:
try:
work_item = self._work_queue.get_nowait()
except queue.Empty:
break
if work_item is not None:
work_item.future.cancel()
# Send a wake-up to prevent threads calling
# _work_queue.get(block=True) from permanently blocking.
self._work_queue.put(None)
if wait:
for t in self._threads:
t.join()
def __exit__(self, exc_type, exc_val, exc_tb):
if self._cancel_on_error:
self.shutdown(wait=True, cancel_futures=exc_val is not None)
else:
self.shutdown(wait=True)
return False
+72
View File
@@ -0,0 +1,72 @@
import operator
import time
import pytest
from funcy import raiser
from dvc.utils.threadpool import ThreadPoolExecutor
@pytest.mark.parametrize("wait", [True, False])
@pytest.mark.parametrize("cancel_futures", [True, False])
def test_cancel_futures(wait, cancel_futures):
"""Modified from
https://github.com/python/cpython/blob/4d2403f/Lib/test/test_concurrent_futures.py#L354
"""
executor = ThreadPoolExecutor(max_workers=2)
fs = [executor.submit(time.sleep, 0.1) for _ in range(50)]
executor.shutdown(wait=wait, cancel_futures=cancel_futures)
if not wait:
for t in executor._threads:
t.join()
cancelled = [fut for fut in fs if fut.cancelled()]
# Use "not fut.cancelled()" instead of "fut.done()" to include futures
# that may have been left in a pending state.
others = [fut for fut in fs if not fut.cancelled()]
# Ensure the other futures were able to finish.
for fut in others:
assert fut.done()
assert fut.exception() is None
if not cancel_futures:
# there should be no cancelled futures
assert len(cancelled) == 0
assert len(others) == len(fs)
else:
# We can't guarantee the exact number of cancellations, but we can
# guarantee that *some* were cancelled. With few workers, many of
# the submitted futures should have been cancelled.
assert len(cancelled) > 20
# Similar to the number of cancelled futures, we can't guarantee the
# exact number that completed. But, we can guarantee that at least
# one finished.
assert len(others) > 0
def test_cancel_on_error_context_manager(mocker):
executor = ThreadPoolExecutor(max_workers=2, cancel_on_error=True)
spy = mocker.spy(executor, "shutdown")
with pytest.raises(RuntimeError), executor:
future1 = executor.submit(operator.mul, 2, 21)
future2 = executor.submit(time.sleep, 0.1)
future3 = executor.submit(raiser(RuntimeError), "This is an error")
fs = [executor.submit(time.sleep, 0.1) for _ in range(50)]
assert future1.result() == 42
assert future2.result() is None
_ = future3.result()
spy.assert_called_once_with(wait=True, cancel_futures=True)
cancelled = [fut for fut in fs if fut.cancelled()]
others = [fut for fut in fs if not fut.cancelled()]
for fut in others:
assert fut.done()
assert fut.exception() is None
assert len(cancelled) > 20
assert len(others) > 0