mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
refactor: use a module for slow link detection
This commit is contained in:
Mr. Outis
@@ -1,60 +1,49 @@
|
||||
import logging
|
||||
import time
|
||||
import sys
|
||||
from functools import wraps
|
||||
|
||||
import colorama
|
||||
from dvc.config import Config
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
this = sys.modules[__name__]
|
||||
|
||||
this.timeout_seconds = 10.0
|
||||
this.already_displayed = False
|
||||
this.message = (
|
||||
"You can cut execution time considerably by using a different "
|
||||
"`cache.type` configuration.\n"
|
||||
"See {blue}https://dvc.org/doc/commands-reference/config#cache{reset} "
|
||||
"for more information.\n"
|
||||
"To disable this message, run:\n"
|
||||
"'dvc config cache.slow_link_warning false'"
|
||||
.format(blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET)
|
||||
)
|
||||
|
||||
|
||||
class SlowLinkDetectorDecorator(object):
|
||||
LINKING_TIMEOUT_SECONDS = 10.0
|
||||
was_displayed = False
|
||||
def slow_link_guard(f):
|
||||
@wraps(f)
|
||||
def wrapper(remote, *args, **kwargs):
|
||||
if this.already_displayed:
|
||||
return f(remote, *args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def should_display(cls):
|
||||
if not cls.was_displayed:
|
||||
cls.was_displayed = True
|
||||
return True
|
||||
return False
|
||||
config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
|
||||
cache_type = config.get(Config.SECTION_CACHE_TYPE)
|
||||
should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True)
|
||||
|
||||
def __init__(self, method):
|
||||
self.method = method
|
||||
if not should_warn or cache_type:
|
||||
return f(remote, *args, **kwargs)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
start = time.time()
|
||||
result = self.method(*args, **kwargs)
|
||||
execution_time_seconds = time.time() - start
|
||||
result = f(remote, *args, **kwargs)
|
||||
delta = time.time() - start
|
||||
|
||||
if (
|
||||
execution_time_seconds >= self.LINKING_TIMEOUT_SECONDS
|
||||
and self.should_display()
|
||||
):
|
||||
msg = (
|
||||
"You can cut execution time considerably by using a different"
|
||||
" `cache.type` configuration.\nSee "
|
||||
"{blue}https://dvc.org/doc/commands-reference/config#cache{"
|
||||
"reset} for more information.\n"
|
||||
"To disable this message, run:\n"
|
||||
"'dvc config cache.slow_link_warning false'".format(
|
||||
blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
|
||||
)
|
||||
)
|
||||
logger.warning(msg)
|
||||
if delta >= this.timeout_seconds:
|
||||
logger.warning(this.message)
|
||||
this.already_displayed = True
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def slow_link_guard(method):
|
||||
def call(remote, *args, **kwargs):
|
||||
cache_config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
|
||||
should_warn = cache_config.get(
|
||||
Config.SECTION_CACHE_SLOW_LINK_WARNING, True
|
||||
) and not cache_config.get(Config.SECTION_CACHE_TYPE, None)
|
||||
|
||||
if should_warn:
|
||||
decorated = SlowLinkDetectorDecorator(method)
|
||||
return decorated(remote, *args, **kwargs)
|
||||
return method(remote, *args, **kwargs)
|
||||
|
||||
return call
|
||||
return wrapper
|
||||
|
||||
@@ -1,111 +1,49 @@
|
||||
import logging
|
||||
|
||||
import colorama
|
||||
import pytest
|
||||
import mock
|
||||
|
||||
import dvc.remote.slow_link_detection
|
||||
from dvc.remote.slow_link_detection import slow_link_guard
|
||||
from dvc.config import Config
|
||||
from dvc.remote.slow_link_detection import (
|
||||
slow_link_guard,
|
||||
SlowLinkDetectorDecorator,
|
||||
)
|
||||
from mock import Mock, patch
|
||||
|
||||
|
||||
@patch(
|
||||
"dvc.remote.slow_link_detection.SlowLinkDetectorDecorator", autospec=True
|
||||
)
|
||||
class TestSlowLinkGuard(object):
|
||||
@pytest.fixture(autouse=True)
|
||||
def setUp(self):
|
||||
self.method = Mock()
|
||||
self.remote_mock = Mock()
|
||||
self.config_mock = Mock()
|
||||
self.remote_mock.repo.config.config.get.return_value = self.config_mock
|
||||
|
||||
def _cache_config(self, slow_link_warning, cache_type):
|
||||
def config_side_effect(section_name, _):
|
||||
if section_name == Config.SECTION_CACHE_SLOW_LINK_WARNING:
|
||||
return slow_link_warning
|
||||
elif section_name == Config.SECTION_CACHE_TYPE:
|
||||
return cache_type
|
||||
|
||||
self.config_mock.get.side_effect = config_side_effect
|
||||
|
||||
def test_should_decorate_on_slow_link_warning_and_no_cache_type(
|
||||
self, SlowLinkDetectorDecoratorClassMock
|
||||
):
|
||||
self._cache_config(slow_link_warning=True, cache_type=None)
|
||||
|
||||
slow_link_guard(self.method)(self.remote_mock)
|
||||
|
||||
assert 1 == SlowLinkDetectorDecoratorClassMock.call_count
|
||||
assert 0 == self.method.call_count
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"slow_link_warning, cache_type",
|
||||
[(True, "any_cache_type"), (False, None)],
|
||||
@pytest.fixture(autouse=True)
|
||||
def timeout_immediately(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
dvc.remote.slow_link_detection, 'timeout_seconds', 0.0
|
||||
)
|
||||
def test_should_not_decorate(
|
||||
self, SlowLinkDetectorDecoratorClassMock, slow_link_warning, cache_type
|
||||
):
|
||||
self._cache_config(
|
||||
slow_link_warning=slow_link_warning, cache_type=cache_type
|
||||
)
|
||||
|
||||
slow_link_guard(self.method)(self.remote_mock)
|
||||
|
||||
assert 0 == SlowLinkDetectorDecoratorClassMock.call_count
|
||||
assert 1 == self.method.call_count
|
||||
|
||||
|
||||
class TestSlowLinkDetectorDecorator(object):
|
||||
@pytest.fixture(autouse=True)
|
||||
def setUp(self):
|
||||
self.args_mock = Mock()
|
||||
self.method_result_mock = Mock()
|
||||
@pytest.fixture
|
||||
def make_remote():
|
||||
def _make_remote(cache_type=None, should_warn=True):
|
||||
remote = mock.Mock()
|
||||
remote.repo.config.config.get.return_value = {
|
||||
Config.SECTION_CACHE_TYPE: cache_type,
|
||||
Config.SECTION_CACHE_SLOW_LINK_WARNING: should_warn,
|
||||
}
|
||||
return remote
|
||||
|
||||
def method_effect(arg):
|
||||
assert self.args_mock == arg
|
||||
return self.method_result_mock
|
||||
return _make_remote
|
||||
|
||||
self.method_mock = Mock(side_effect=method_effect)
|
||||
|
||||
@patch.object(SlowLinkDetectorDecorator, "LINKING_TIMEOUT_SECONDS", 0.0)
|
||||
def test_should_log_only_once(self, caplog):
|
||||
def test_show_warning_once(caplog, make_remote):
|
||||
remote = make_remote()
|
||||
slow_link_guard(lambda x: None)(remote)
|
||||
slow_link_guard(lambda x: None)(remote)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="dvc"):
|
||||
test_instance = SlowLinkDetectorDecorator(self.method_mock)
|
||||
result = test_instance(self.args_mock)
|
||||
assert self.method_result_mock == result
|
||||
assert caplog.records[0].message == dvc.remote.slow_link_detection.message
|
||||
assert len(caplog.records) == 1
|
||||
|
||||
# slow link guard keeps creating new SlowLinkDetectorDecorators
|
||||
# each time it is called, so we need to make sure that we are
|
||||
# preventing every new instance of this class from displaying the
|
||||
# message
|
||||
test_instance2 = SlowLinkDetectorDecorator(self.method_mock)
|
||||
result = test_instance2(self.args_mock)
|
||||
assert self.method_result_mock == result
|
||||
|
||||
msg = (
|
||||
"You can cut execution time considerably by using a different"
|
||||
" `cache.type` configuration.\nSee "
|
||||
"{blue}https://dvc.org/doc/commands-reference/config#cache{"
|
||||
"reset} for more information.\n"
|
||||
"To disable this message, run:\n"
|
||||
"'dvc config cache.slow_link_warning false'".format(
|
||||
blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
|
||||
)
|
||||
)
|
||||
assert 1 == len(caplog.messages)
|
||||
assert msg == caplog.messages[0]
|
||||
def test_dont_warn_when_cache_type_is_set(caplog, make_remote):
|
||||
remote = make_remote(cache_type="copy")
|
||||
slow_link_guard(lambda x: None)(remote)
|
||||
|
||||
@patch.object(
|
||||
SlowLinkDetectorDecorator, "LINKING_TIMEOUT_SECONDS", 99999.0
|
||||
)
|
||||
def test_should_not_log(self, caplog):
|
||||
test_instance = SlowLinkDetectorDecorator(self.method_mock)
|
||||
assert len(caplog.records) == 0
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="dvc"):
|
||||
result = test_instance(self.args_mock)
|
||||
|
||||
assert self.method_result_mock == result
|
||||
assert 0 == len(caplog.messages)
|
||||
def test_dont_warn_when_warning_is_disabled(caplog, make_remote):
|
||||
remote = make_remote(should_warn=False)
|
||||
slow_link_guard(lambda x: None)(remote)
|
||||
|
||||
assert len(caplog.records) == 0
|
||||
|
||||
Reference in New Issue
Block a user