diff --git a/dvc/utils/threadpool.py b/dvc/utils/threadpool.py index 8a768ebbd..dde15ba6e 100644 --- a/dvc/utils/threadpool.py +++ b/dvc/utils/threadpool.py @@ -1,3 +1,5 @@ +import queue +import sys from concurrent import futures from itertools import islice from typing import Any, Callable, Iterable, Iterator, Set, TypeVar @@ -8,6 +10,12 @@ _T = TypeVar("_T") class ThreadPoolExecutor(futures.ThreadPoolExecutor): _max_workers: int + def __init__( + self, max_workers: int = None, cancel_on_error: bool = False, **kwargs + ): + super().__init__(max_workers=max_workers, **kwargs) + self._cancel_on_error = cancel_on_error + @property def max_workers(self) -> int: return self._max_workers @@ -32,3 +40,35 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor): for fut in done: yield fut.result() tasks.update(create_taskset(len(done))) + + def shutdown(self, wait=True, *, cancel_futures=False): + if sys.version_info > (3, 9): + # pylint: disable=unexpected-keyword-arg + return super().shutdown(wait=wait, cancel_futures=cancel_futures) + + with self._shutdown_lock: + self._shutdown = True + if cancel_futures: + # Drain all work items from the queue, and then cancel their + # associated futures. + while True: + try: + work_item = self._work_queue.get_nowait() + except queue.Empty: + break + if work_item is not None: + work_item.future.cancel() + + # Send a wake-up to prevent threads calling + # _work_queue.get(block=True) from permanently blocking. + self._work_queue.put(None) + if wait: + for t in self._threads: + t.join() + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._cancel_on_error: + self.shutdown(wait=True, cancel_futures=exc_val is not None) + else: + self.shutdown(wait=True) + return False diff --git a/tests/unit/utils/test_executors.py b/tests/unit/utils/test_executors.py new file mode 100644 index 000000000..46a073c61 --- /dev/null +++ b/tests/unit/utils/test_executors.py @@ -0,0 +1,72 @@ +import operator +import time + +import pytest +from funcy import raiser + +from dvc.utils.threadpool import ThreadPoolExecutor + + +@pytest.mark.parametrize("wait", [True, False]) +@pytest.mark.parametrize("cancel_futures", [True, False]) +def test_cancel_futures(wait, cancel_futures): + """Modified from + https://github.com/python/cpython/blob/4d2403f/Lib/test/test_concurrent_futures.py#L354 + """ + executor = ThreadPoolExecutor(max_workers=2) + fs = [executor.submit(time.sleep, 0.1) for _ in range(50)] + executor.shutdown(wait=wait, cancel_futures=cancel_futures) + + if not wait: + for t in executor._threads: + t.join() + + cancelled = [fut for fut in fs if fut.cancelled()] + # Use "not fut.cancelled()" instead of "fut.done()" to include futures + # that may have been left in a pending state. + others = [fut for fut in fs if not fut.cancelled()] + + # Ensure the other futures were able to finish. + for fut in others: + assert fut.done() + assert fut.exception() is None + + if not cancel_futures: + # there should be no cancelled futures + assert len(cancelled) == 0 + assert len(others) == len(fs) + else: + # We can't guarantee the exact number of cancellations, but we can + # guarantee that *some* were cancelled. With few workers, many of + # the submitted futures should have been cancelled. + assert len(cancelled) > 20 + # Similar to the number of cancelled futures, we can't guarantee the + # exact number that completed. But, we can guarantee that at least + # one finished. + assert len(others) > 0 + + +def test_cancel_on_error_context_manager(mocker): + executor = ThreadPoolExecutor(max_workers=2, cancel_on_error=True) + spy = mocker.spy(executor, "shutdown") + with pytest.raises(RuntimeError), executor: + future1 = executor.submit(operator.mul, 2, 21) + future2 = executor.submit(time.sleep, 0.1) + future3 = executor.submit(raiser(RuntimeError), "This is an error") + fs = [executor.submit(time.sleep, 0.1) for _ in range(50)] + + assert future1.result() == 42 + assert future2.result() is None + _ = future3.result() + + spy.assert_called_once_with(wait=True, cancel_futures=True) + + cancelled = [fut for fut in fs if fut.cancelled()] + others = [fut for fut in fs if not fut.cancelled()] + + for fut in others: + assert fut.done() + assert fut.exception() is None + + assert len(cancelled) > 20 + assert len(others) > 0