From 6ba4aba41964398b75977882e924dd4618f36276 Mon Sep 17 00:00:00 2001 From: pawel Date: Mon, 16 Dec 2019 08:23:53 +0100 Subject: [PATCH] remote: base: [down/up]load: raise on too many open files --- dvc/exceptions.py | 9 +++++---- dvc/remote/base.py | 11 +++++++---- tests/func/test_remote.py | 22 +++++++++++++++++++++- tests/unit/remote/test_s3.py | 23 ++--------------------- 4 files changed, 35 insertions(+), 30 deletions(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 2b77f0d50..1373179f5 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -346,10 +346,11 @@ class HTTPError(DvcException): super(HTTPError, self).__init__("'{} {}'".format(code, reason)) -class TooManyOpenFilesException(DvcException): - def __init__(self): - super(TooManyOpenFilesException, self).__init__( +class TooManyOpenFilesError(DvcException): + def __init__(self, cause): + super(TooManyOpenFilesError, self).__init__( "Operation failed due to too many open file descriptors. reduce " "the number of jobs or increase open file descriptors limit to " - "prevent this." + "prevent this.", + cause=cause, ) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index a2324ba2c..5f184e4f3 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -15,9 +15,12 @@ from shortuuid import uuid import dvc.prompt as prompt from dvc.config import Config -from dvc.exceptions import ConfirmRemoveError, TooManyOpenFilesException -from dvc.exceptions import DvcException -from dvc.exceptions import DvcIgnoreInCollectedDirError +from dvc.exceptions import ( + DvcException, + ConfirmRemoveError, + DvcIgnoreInCollectedDirError, + TooManyOpenFilesError, +) from dvc.ignore import DvcIgnore from dvc.path_info import PathInfo, URLInfo from dvc.progress import Tqdm @@ -521,7 +524,7 @@ class RemoteBASE(object): self, from_info, to_info, exception, operation ): if isinstance(exception, OSError) and exception.errno == 24: - raise TooManyOpenFilesException() + raise TooManyOpenFilesError(exception) msg = "failed to {} '{}' to '{}'".format(operation, from_info, to_info) logger.exception(msg) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 23f318861..cf41a5f26 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -2,13 +2,16 @@ import os import shutil import configobj +import pytest from mock import patch from dvc.config import Config +from dvc.exceptions import TooManyOpenFilesError from dvc.main import main from dvc.path_info import PathInfo -from dvc.remote import RemoteLOCAL +from dvc.remote import RemoteLOCAL, RemoteConfig from dvc.remote.base import RemoteBASE +from dvc.utils.compat import fspath from tests.basic_env import TestDvc from tests.remotes import get_local_url, get_local_storagepath @@ -253,3 +256,20 @@ def test_partial_push_n_pull(dvc_repo, repo_dir, caplog): def get_last_exc(caplog): _, exc, _ = caplog.records[-2].exc_info return exc + + +def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker): + storage = tmp_path_factory.mktemp("test_remote_base") + remote_config = RemoteConfig(dvc.config) + remote_config.add("local_remote", fspath(storage), default=True) + + tmp_dir.dvc_gen({"file": "file content"}) + + too_many_open_files_error = OSError() + mocker.patch.object(too_many_open_files_error, "errno", 24) + mocker.patch.object( + RemoteLOCAL, "_upload", side_effect=too_many_open_files_error + ) + + with pytest.raises(TooManyOpenFilesError): + dvc.push() diff --git a/tests/unit/remote/test_s3.py b/tests/unit/remote/test_s3.py index 453741158..ddcf5e7f8 100644 --- a/tests/unit/remote/test_s3.py +++ b/tests/unit/remote/test_s3.py @@ -1,7 +1,8 @@ -import logging import os import pytest + +from dvc.config import ConfigError from dvc.remote import RemoteS3 from tests.utils import empty_caplog @@ -73,23 +74,3 @@ def test_adjust_default_jobs_number( with empty_caplog(caplog): assert remote.adjust_jobs() == expected_result - - -def test_warn_on_too_many_jobs(mocker, caplog): - remote = RemoteS3(None, {}) - - if os.name == "nt": - mocker.patch("win32file._getmaxstdio", return_value=256) - else: - mocker.patch("resource.getrlimit", return_value=(256, 1024)) - - with caplog.at_level(logging.INFO, "dvc"): - assert remote.adjust_jobs(64) == 64 - assert len(caplog.messages) == 1 - assert caplog.messages[0] == ( - "Provided jobs number '64' might result " - "in 'Too many open files error'. " - "Consider decreasing jobs number to '12' " - "or increasing file descriptors limit to " - "'1290'." - ) \ No newline at end of file