mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-20 07:18:27 +01:00
Merge pull request #2866 from pared/2473
remote: s3: adjust jobs number basing on file descriptors number
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
"""Main entry point for dvc CLI."""
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import errno
|
||||
import logging
|
||||
|
||||
from dvc import analytics
|
||||
@@ -64,6 +65,10 @@ def main(argv=None):
|
||||
"unicode is not supported in DVC for Python 2 "
|
||||
"(end-of-life January 1, 2020), please upgrade to Python 3"
|
||||
)
|
||||
elif isinstance(exc, OSError) and exc.errno == errno.EMFILE:
|
||||
logger.exception(
|
||||
"too many open files, please increase your `ulimit`"
|
||||
)
|
||||
else:
|
||||
logger.exception("unexpected error")
|
||||
ret = 255
|
||||
|
||||
+21
-8
@@ -1,4 +1,7 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import errno
|
||||
|
||||
from dvc.utils.compat import basestring, FileNotFoundError, str, urlparse
|
||||
|
||||
import itertools
|
||||
@@ -516,6 +519,16 @@ class RemoteBASE(object):
|
||||
return
|
||||
self._save_file(path_info, checksum)
|
||||
|
||||
def _handle_transfer_exception(
|
||||
self, from_info, to_info, exception, operation
|
||||
):
|
||||
if isinstance(exception, OSError) and exception.errno == errno.EMFILE:
|
||||
raise exception
|
||||
|
||||
msg = "failed to {} '{}' to '{}'".format(operation, from_info, to_info)
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
|
||||
def upload(self, from_info, to_info, name=None, no_progress_bar=False):
|
||||
if not hasattr(self, "_upload"):
|
||||
raise RemoteActionNotImplemented("upload", self.scheme)
|
||||
@@ -537,10 +550,10 @@ class RemoteBASE(object):
|
||||
name=name,
|
||||
no_progress_bar=no_progress_bar,
|
||||
)
|
||||
except Exception:
|
||||
msg = "failed to upload '{}' to '{}'"
|
||||
logger.exception(msg.format(from_info, to_info))
|
||||
return 1 # 1 fail
|
||||
except Exception as e:
|
||||
return self._handle_transfer_exception(
|
||||
from_info, to_info, e, "upload"
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
@@ -614,10 +627,10 @@ class RemoteBASE(object):
|
||||
self._download(
|
||||
from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
|
||||
)
|
||||
except Exception:
|
||||
msg = "failed to download '{}' to '{}'"
|
||||
logger.exception(msg.format(from_info, to_info))
|
||||
return 1 # 1 fail
|
||||
except Exception as e:
|
||||
return self._handle_transfer_exception(
|
||||
from_info, to_info, e, "download"
|
||||
)
|
||||
|
||||
move(tmp_file, to_info, mode=file_mode)
|
||||
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
import errno
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import configobj
|
||||
import pytest
|
||||
from mock import patch
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.main import main
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote import RemoteLOCAL
|
||||
from dvc.remote import RemoteLOCAL, RemoteConfig
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.utils.compat import fspath
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.remotes import get_local_url, get_local_storagepath
|
||||
|
||||
@@ -253,3 +256,21 @@ def test_partial_push_n_pull(dvc_repo, repo_dir, caplog):
|
||||
def get_last_exc(caplog):
|
||||
_, exc, _ = caplog.records[-2].exc_info
|
||||
return exc
|
||||
|
||||
|
||||
def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker):
|
||||
storage = tmp_path_factory.mktemp("test_remote_base")
|
||||
remote_config = RemoteConfig(dvc.config)
|
||||
remote_config.add("local_remote", fspath(storage), default=True)
|
||||
|
||||
tmp_dir.dvc_gen({"file": "file content"})
|
||||
|
||||
mocker.patch.object(
|
||||
RemoteLOCAL,
|
||||
"_upload",
|
||||
side_effect=OSError(errno.EMFILE, "Too many open files"),
|
||||
)
|
||||
|
||||
with pytest.raises(OSError) as e:
|
||||
dvc.push()
|
||||
assert e.errno == errno.EMFILE
|
||||
|
||||
Reference in New Issue
Block a user