From 5faae2a6e9fe0ef4a7cf3749b89fbcb6b7869170 Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Mon, 23 Nov 2020 11:43:39 +0545 Subject: [PATCH] test: allow selectively running tests for remotes (#4920) Some of the tests are run by default, other requires `--enable-{remote}` to run. The remotes that are run by default also have a `--disable-{remote}` flag. All of the remotes can be enabled using `--all` flag. To use this functionality in the fixture, `test_config` fixture has to be used, and requires to call `test_config.requires(remote)`. In an individual test, you can simply use `pytest.mark.remote` to make them skip or run. --- .github/workflows/tests.yaml | 2 +- tests/conftest.py | 102 +++++++++++++++++++++++++++++ tests/func/test_output.py | 5 +- tests/remotes/azure.py | 4 +- tests/remotes/gdrive.py | 3 +- tests/remotes/gs.py | 3 +- tests/remotes/hdfs.py | 4 +- tests/remotes/http.py | 3 +- tests/remotes/oss.py | 6 +- tests/remotes/s3.py | 6 +- tests/remotes/ssh.py | 3 +- tests/remotes/webdav.py | 4 +- tests/unit/dependency/test_hdfs.py | 1 + tests/unit/output/test_hdfs.py | 1 + 14 files changed, 134 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b16bbd38f..0dcaff073 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -65,7 +65,7 @@ jobs: mkdir -p scripts/ci echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json - name: run tests - run: python -m tests -ra --cov-report=xml --cov-report=term --tap-combined + run: python -m tests --all -ra --cov-report=xml --cov-report=term --tap-combined - name: upload coverage report uses: codecov/codecov-action@v1.0.13 with: diff --git a/tests/conftest.py b/tests/conftest.py index 796f465ea..4fddb0387 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,19 @@ os.environ["DVC_TEST"] = "true" # Ensure progress output even when not outputting to raw sys.stderr console os.environ["DVC_IGNORE_ISATTY"] = "true" +REMOTES = { + # remote: enabled_by_default? + "azure": False, + "gdrive": False, + "gs": False, + "hdfs": False, + "http": True, + "oss": False, + "s3": True, + "ssh": True, + "webdav": True, +} + @pytest.fixture(autouse=True) def reset_loglevel(request, caplog): @@ -31,3 +44,92 @@ def _close_pools(): yield close_pools() + + +def _get_opt(remote_name, action): + return f"--{action}-{remote_name}" + + +def pytest_addoption(parser): + """Adds remote-related flags to selectively disable/enable for tests + Eg: If some remotes, eg: ssh is enabled to be tested for by default + (see above `REMOTES`), then, `--disable-ssh` flag is added. If remotes + like `hdfs` are disabled by default, `--enable-hdfs` is added to make them + run. + + You can also make everything run-by-default with `--all` flag, which takes + precedence on all previous `--enable-*`/`--disable-*` flags. + """ + parser.addoption( + "--all", + action="store_true", + default=False, + help="Test all of the remotes, unless other flags also supplied", + ) + for remote_name in REMOTES: + for action in ("enable", "disable"): + opt = _get_opt(remote_name, action) + parser.addoption( + opt, + action="store_true", + default=None, + help=f"{action} tests for {remote_name}", + ) + + +class DVCTestConfig: + def __init__(self): + self.enabled_remotes = set() + + def requires(self, remote_name): + if remote_name not in REMOTES or remote_name in self.enabled_remotes: + return + + pytest.skip(f"{remote_name} tests not enabled through CLI") + + def apply_marker(self, marker): + self.requires(marker.name) + + +def pytest_runtest_setup(item): + # Apply test markers to skip tests selectively + # NOTE: this only works on individual tests, + # for fixture, use `test_config` fixture and + # run `test_config.requires(remote_name)`. + for marker in item.iter_markers(): + item.config.dvc_config.apply_marker(marker) + + +@pytest.fixture(scope="session") +def test_config(request): + return request.config.dvc_config + + +def pytest_configure(config): + config.dvc_config = DVCTestConfig() + + for remote_name in REMOTES: + config.addinivalue_line( + "markers", f"{remote_name}: mark test as requiring {remote_name}" + ) + + enabled_remotes = config.dvc_config.enabled_remotes + if config.getoption("--all"): + enabled_remotes.update(REMOTES) + else: + default_enabled = {k for k, v in REMOTES.items() if v} + enabled_remotes.update(default_enabled) + + for remote_name in REMOTES: + enabled_opt = _get_opt(remote_name, "enable") + disabled_opt = _get_opt(remote_name, "disable") + + enabled = config.getoption(enabled_opt) + disabled = config.getoption(disabled_opt) + if disabled and enabled: + continue # default behavior if both flags are supplied + + if disabled: + enabled_remotes.discard(remote_name) + if enabled: + enabled_remotes.add(remote_name) diff --git a/tests/func/test_output.py b/tests/func/test_output.py index 2baf7b7b0..d4ef361fd 100644 --- a/tests/func/test_output.py +++ b/tests/func/test_output.py @@ -4,7 +4,10 @@ from dvc.output import OUTS_MAP, _get from dvc.stage import Stage from tests import PY39, PYARROW_NOT_AVAILABLE -MARKERS = [pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE)] +MARKERS = [ + pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE), + pytest.mark.hdfs, +] TESTS = [ ("s3://bucket/path", "s3"), diff --git a/tests/remotes/azure.py b/tests/remotes/azure.py index b0b31b1d2..92662a934 100644 --- a/tests/remotes/azure.py +++ b/tests/remotes/azure.py @@ -23,7 +23,9 @@ class Azure(Base, CloudURLInfo): @pytest.fixture(scope="session") -def azure_server(docker_compose, docker_services): +def azure_server(test_config, docker_compose, docker_services): + test_config.requires("azure") + from azure.core.exceptions import ( # pylint: disable=no-name-in-module AzureError, ) diff --git a/tests/remotes/gdrive.py b/tests/remotes/gdrive.py index 2fd90df22..21e755991 100644 --- a/tests/remotes/gdrive.py +++ b/tests/remotes/gdrive.py @@ -38,7 +38,8 @@ class GDrive(Base, CloudURLInfo): @pytest.fixture -def gdrive(make_tmp_dir): +def gdrive(test_config, make_tmp_dir): + test_config.requires("gdrive") if not GDrive.should_test(): pytest.skip("no gdrive") diff --git a/tests/remotes/gs.py b/tests/remotes/gs.py index 431e41146..6909e5540 100644 --- a/tests/remotes/gs.py +++ b/tests/remotes/gs.py @@ -102,7 +102,8 @@ class GCP(Base, CloudURLInfo): @pytest.fixture -def gs(): +def gs(test_config): + test_config.requires("gs") if not GCP.should_test(): pytest.skip("no gs") yield GCP(GCP.get_url()) diff --git a/tests/remotes/hdfs.py b/tests/remotes/hdfs.py index 45d687f20..953ba0c68 100644 --- a/tests/remotes/hdfs.py +++ b/tests/remotes/hdfs.py @@ -70,7 +70,9 @@ class HDFS(Base, URLInfo): # pylint: disable=abstract-method @pytest.fixture(scope="session") -def hadoop(): +def hadoop(test_config): + test_config.requires("hdfs") + if platform.system() != "Linux": pytest.skip("only supported on Linux") diff --git a/tests/remotes/http.py b/tests/remotes/http.py index 87bc5d57c..83f80ab1d 100644 --- a/tests/remotes/http.py +++ b/tests/remotes/http.py @@ -35,9 +35,10 @@ class HTTP(Base, HTTPURLInfo): @pytest.fixture(scope="session") -def http_server(tmp_path_factory): +def http_server(test_config, tmp_path_factory): from tests.utils.httpd import StaticFileServer + test_config.requires("http") directory = os.fspath(tmp_path_factory.mktemp("http")) with StaticFileServer(directory=directory) as httpd: yield httpd diff --git a/tests/remotes/oss.py b/tests/remotes/oss.py index 8bf7fa14b..43fbb4a77 100644 --- a/tests/remotes/oss.py +++ b/tests/remotes/oss.py @@ -35,9 +35,10 @@ class OSS(Base, CloudURLInfo): @pytest.fixture(scope="session") -def oss_server(docker_compose, docker_services): +def oss_server(test_config, docker_compose, docker_services): import oss2 + test_config.requires("oss") port = docker_services.port_for("oss", 8880) endpoint = EMULATOR_OSS_ENDPOINT.format(port=port) @@ -72,7 +73,8 @@ def oss(oss_server): @pytest.fixture -def real_oss(): +def real_oss(test_config): + test_config.requires("oss") if not OSS.should_test(): pytest.skip("no real OSS") diff --git a/tests/remotes/s3.py b/tests/remotes/s3.py index c5de508e4..23c85cceb 100644 --- a/tests/remotes/s3.py +++ b/tests/remotes/s3.py @@ -99,7 +99,8 @@ class S3(Base, CloudURLInfo): @pytest.fixture -def s3(): +def s3(test_config): + test_config.requires("s3") with mock_s3(): import boto3 @@ -109,7 +110,8 @@ def s3(): @pytest.fixture -def real_s3(): +def real_s3(test_config): + test_config.requires("s3") if not S3.should_test(): pytest.skip("no real s3") yield S3(S3.get_url()) diff --git a/tests/remotes/ssh.py b/tests/remotes/ssh.py index 68cf73726..b078775e2 100644 --- a/tests/remotes/ssh.py +++ b/tests/remotes/ssh.py @@ -113,9 +113,10 @@ class SSHMocked(Base, URLInfo): @pytest.fixture -def ssh_server(): +def ssh_server(test_config): import mockssh + test_config.requires("ssh") users = {TEST_SSH_USER: TEST_SSH_KEY_PATH} with mockssh.Server(users) as s: yield s diff --git a/tests/remotes/webdav.py b/tests/remotes/webdav.py index 696db8e09..26f0b1fe6 100644 --- a/tests/remotes/webdav.py +++ b/tests/remotes/webdav.py @@ -33,7 +33,9 @@ class Webdav(Base, WebDAVURLInfo): @pytest.fixture -def webdav_server(tmp_path_factory): +def webdav_server(test_config, tmp_path_factory): + test_config.requires("webdav") + host, port = "localhost", 0 directory = os.fspath(tmp_path_factory.mktemp("http")) dirmap = {"/": directory} diff --git a/tests/unit/dependency/test_hdfs.py b/tests/unit/dependency/test_hdfs.py index 10118908c..2dacc5edc 100644 --- a/tests/unit/dependency/test_hdfs.py +++ b/tests/unit/dependency/test_hdfs.py @@ -6,6 +6,7 @@ from tests.unit.dependency.test_local import TestLocalDependency @pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE) +@pytest.mark.hdfs class TestHDFSDependency(TestLocalDependency): def _get_cls(self): return HDFSDependency diff --git a/tests/unit/output/test_hdfs.py b/tests/unit/output/test_hdfs.py index ec81081c4..dffffb88b 100644 --- a/tests/unit/output/test_hdfs.py +++ b/tests/unit/output/test_hdfs.py @@ -6,6 +6,7 @@ from tests.unit.output.test_local import TestLocalOutput @pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE) +@pytest.mark.hdfs class TestHDFSOutput(TestLocalOutput): def _get_cls(self): return HDFSOutput