mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
remote: base: [down/up]load: raise on too many open files
This commit is contained in:
+5
-4
@@ -346,10 +346,11 @@ class HTTPError(DvcException):
|
||||
super(HTTPError, self).__init__("'{} {}'".format(code, reason))
|
||||
|
||||
|
||||
class TooManyOpenFilesException(DvcException):
|
||||
def __init__(self):
|
||||
super(TooManyOpenFilesException, self).__init__(
|
||||
class TooManyOpenFilesError(DvcException):
|
||||
def __init__(self, cause):
|
||||
super(TooManyOpenFilesError, self).__init__(
|
||||
"Operation failed due to too many open file descriptors. reduce "
|
||||
"the number of jobs or increase open file descriptors limit to "
|
||||
"prevent this."
|
||||
"prevent this.",
|
||||
cause=cause,
|
||||
)
|
||||
|
||||
+7
-4
@@ -15,9 +15,12 @@ from shortuuid import uuid
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import ConfirmRemoveError, TooManyOpenFilesException
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.exceptions import DvcIgnoreInCollectedDirError
|
||||
from dvc.exceptions import (
|
||||
DvcException,
|
||||
ConfirmRemoveError,
|
||||
DvcIgnoreInCollectedDirError,
|
||||
TooManyOpenFilesError,
|
||||
)
|
||||
from dvc.ignore import DvcIgnore
|
||||
from dvc.path_info import PathInfo, URLInfo
|
||||
from dvc.progress import Tqdm
|
||||
@@ -521,7 +524,7 @@ class RemoteBASE(object):
|
||||
self, from_info, to_info, exception, operation
|
||||
):
|
||||
if isinstance(exception, OSError) and exception.errno == 24:
|
||||
raise TooManyOpenFilesException()
|
||||
raise TooManyOpenFilesError(exception)
|
||||
|
||||
msg = "failed to {} '{}' to '{}'".format(operation, from_info, to_info)
|
||||
logger.exception(msg)
|
||||
|
||||
@@ -2,13 +2,16 @@ import os
|
||||
import shutil
|
||||
|
||||
import configobj
|
||||
import pytest
|
||||
from mock import patch
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import TooManyOpenFilesError
|
||||
from dvc.main import main
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote import RemoteLOCAL
|
||||
from dvc.remote import RemoteLOCAL, RemoteConfig
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.utils.compat import fspath
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.remotes import get_local_url, get_local_storagepath
|
||||
|
||||
@@ -253,3 +256,20 @@ def test_partial_push_n_pull(dvc_repo, repo_dir, caplog):
|
||||
def get_last_exc(caplog):
|
||||
_, exc, _ = caplog.records[-2].exc_info
|
||||
return exc
|
||||
|
||||
|
||||
def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker):
|
||||
storage = tmp_path_factory.mktemp("test_remote_base")
|
||||
remote_config = RemoteConfig(dvc.config)
|
||||
remote_config.add("local_remote", fspath(storage), default=True)
|
||||
|
||||
tmp_dir.dvc_gen({"file": "file content"})
|
||||
|
||||
too_many_open_files_error = OSError()
|
||||
mocker.patch.object(too_many_open_files_error, "errno", 24)
|
||||
mocker.patch.object(
|
||||
RemoteLOCAL, "_upload", side_effect=too_many_open_files_error
|
||||
)
|
||||
|
||||
with pytest.raises(TooManyOpenFilesError):
|
||||
dvc.push()
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.config import ConfigError
|
||||
from dvc.remote import RemoteS3
|
||||
from tests.utils import empty_caplog
|
||||
|
||||
@@ -73,23 +74,3 @@ def test_adjust_default_jobs_number(
|
||||
|
||||
with empty_caplog(caplog):
|
||||
assert remote.adjust_jobs() == expected_result
|
||||
|
||||
|
||||
def test_warn_on_too_many_jobs(mocker, caplog):
|
||||
remote = RemoteS3(None, {})
|
||||
|
||||
if os.name == "nt":
|
||||
mocker.patch("win32file._getmaxstdio", return_value=256)
|
||||
else:
|
||||
mocker.patch("resource.getrlimit", return_value=(256, 1024))
|
||||
|
||||
with caplog.at_level(logging.INFO, "dvc"):
|
||||
assert remote.adjust_jobs(64) == 64
|
||||
assert len(caplog.messages) == 1
|
||||
assert caplog.messages[0] == (
|
||||
"Provided jobs number '64' might result "
|
||||
"in 'Too many open files error'. "
|
||||
"Consider decreasing jobs number to '12' "
|
||||
"or increasing file descriptors limit to "
|
||||
"'1290'."
|
||||
)
|
||||
Reference in New Issue
Block a user