remote: base: [down/up]load: raise on too many open files

This commit is contained in:
pawel
2019-12-17 11:41:03 +01:00
committed by Paweł Redzyński
parent d253f1319d
commit 6ba4aba419
4 changed files with 35 additions and 30 deletions
+5 -4
View File
@@ -346,10 +346,11 @@ class HTTPError(DvcException):
super(HTTPError, self).__init__("'{} {}'".format(code, reason))
class TooManyOpenFilesException(DvcException):
def __init__(self):
super(TooManyOpenFilesException, self).__init__(
class TooManyOpenFilesError(DvcException):
def __init__(self, cause):
super(TooManyOpenFilesError, self).__init__(
"Operation failed due to too many open file descriptors. reduce "
"the number of jobs or increase open file descriptors limit to "
"prevent this."
"prevent this.",
cause=cause,
)
+7 -4
View File
@@ -15,9 +15,12 @@ from shortuuid import uuid
import dvc.prompt as prompt
from dvc.config import Config
from dvc.exceptions import ConfirmRemoveError, TooManyOpenFilesException
from dvc.exceptions import DvcException
from dvc.exceptions import DvcIgnoreInCollectedDirError
from dvc.exceptions import (
DvcException,
ConfirmRemoveError,
DvcIgnoreInCollectedDirError,
TooManyOpenFilesError,
)
from dvc.ignore import DvcIgnore
from dvc.path_info import PathInfo, URLInfo
from dvc.progress import Tqdm
@@ -521,7 +524,7 @@ class RemoteBASE(object):
self, from_info, to_info, exception, operation
):
if isinstance(exception, OSError) and exception.errno == 24:
raise TooManyOpenFilesException()
raise TooManyOpenFilesError(exception)
msg = "failed to {} '{}' to '{}'".format(operation, from_info, to_info)
logger.exception(msg)
+21 -1
View File
@@ -2,13 +2,16 @@ import os
import shutil
import configobj
import pytest
from mock import patch
from dvc.config import Config
from dvc.exceptions import TooManyOpenFilesError
from dvc.main import main
from dvc.path_info import PathInfo
from dvc.remote import RemoteLOCAL
from dvc.remote import RemoteLOCAL, RemoteConfig
from dvc.remote.base import RemoteBASE
from dvc.utils.compat import fspath
from tests.basic_env import TestDvc
from tests.remotes import get_local_url, get_local_storagepath
@@ -253,3 +256,20 @@ def test_partial_push_n_pull(dvc_repo, repo_dir, caplog):
def get_last_exc(caplog):
_, exc, _ = caplog.records[-2].exc_info
return exc
def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker):
storage = tmp_path_factory.mktemp("test_remote_base")
remote_config = RemoteConfig(dvc.config)
remote_config.add("local_remote", fspath(storage), default=True)
tmp_dir.dvc_gen({"file": "file content"})
too_many_open_files_error = OSError()
mocker.patch.object(too_many_open_files_error, "errno", 24)
mocker.patch.object(
RemoteLOCAL, "_upload", side_effect=too_many_open_files_error
)
with pytest.raises(TooManyOpenFilesError):
dvc.push()
+2 -21
View File
@@ -1,7 +1,8 @@
import logging
import os
import pytest
from dvc.config import ConfigError
from dvc.remote import RemoteS3
from tests.utils import empty_caplog
@@ -73,23 +74,3 @@ def test_adjust_default_jobs_number(
with empty_caplog(caplog):
assert remote.adjust_jobs() == expected_result
def test_warn_on_too_many_jobs(mocker, caplog):
remote = RemoteS3(None, {})
if os.name == "nt":
mocker.patch("win32file._getmaxstdio", return_value=256)
else:
mocker.patch("resource.getrlimit", return_value=(256, 1024))
with caplog.at_level(logging.INFO, "dvc"):
assert remote.adjust_jobs(64) == 64
assert len(caplog.messages) == 1
assert caplog.messages[0] == (
"Provided jobs number '64' might result "
"in 'Too many open files error'. "
"Consider decreasing jobs number to '12' "
"or increasing file descriptors limit to "
"'1290'."
)