test: allow selectively running tests for remotes (#4920)

Some of the tests are run by default, other requires
`--enable-{remote}` to run. The remotes that are run by default
also have a `--disable-{remote}` flag. All of the remotes can be
enabled using `--all` flag.

To use this functionality in the fixture,
`test_config` fixture has to be used, and requires to call
`test_config.requires(remote)`.
In an individual test, you can simply use `pytest.mark.remote`
to make them skip or run.

.github/workflows/tests.yaml

@@ -65,7 +65,7 @@ jobs:
         mkdir -p scripts/ci
         echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
     - name: run tests
-      run: python -m tests -ra --cov-report=xml --cov-report=term --tap-combined
+      run: python -m tests --all -ra --cov-report=xml --cov-report=term --tap-combined
     - name: upload coverage report
       uses: codecov/[email protected]
       with:

tests/conftest.py

@@ -10,6 +10,19 @@ os.environ["DVC_TEST"] = "true"
 # Ensure progress output even when not outputting to raw sys.stderr console
 os.environ["DVC_IGNORE_ISATTY"] = "true"
 
+REMOTES = {
+    # remote: enabled_by_default?
+    "azure": False,
+    "gdrive": False,
+    "gs": False,
+    "hdfs": False,
+    "http": True,
+    "oss": False,
+    "s3": True,
+    "ssh": True,
+    "webdav": True,
+}
+
 
 @pytest.fixture(autouse=True)
 def reset_loglevel(request, caplog):
@@ -31,3 +44,92 @@ def _close_pools():
 
     yield
     close_pools()
+
+
+def _get_opt(remote_name, action):
+    return f"--{action}-{remote_name}"
+
+
+def pytest_addoption(parser):
+    """Adds remote-related flags to selectively disable/enable for tests
+    Eg: If some remotes, eg: ssh is enabled to be tested for by default
+    (see above `REMOTES`), then, `--disable-ssh` flag is added. If remotes
+    like `hdfs` are disabled by default, `--enable-hdfs` is added to make them
+    run.
+
+    You can also make everything run-by-default with `--all` flag, which takes
+    precedence on all previous `--enable-*`/`--disable-*` flags.
+    """
+    parser.addoption(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Test all of the remotes, unless other flags also supplied",
+    )
+    for remote_name in REMOTES:
+        for action in ("enable", "disable"):
+            opt = _get_opt(remote_name, action)
+            parser.addoption(
+                opt,
+                action="store_true",
+                default=None,
+                help=f"{action} tests for {remote_name}",
+            )
+
+
+class DVCTestConfig:
+    def __init__(self):
+        self.enabled_remotes = set()
+
+    def requires(self, remote_name):
+        if remote_name not in REMOTES or remote_name in self.enabled_remotes:
+            return
+
+        pytest.skip(f"{remote_name} tests not enabled through CLI")
+
+    def apply_marker(self, marker):
+        self.requires(marker.name)
+
+
+def pytest_runtest_setup(item):
+    # Apply test markers to skip tests selectively
+    # NOTE: this only works on individual tests,
+    # for fixture, use `test_config` fixture and
+    # run `test_config.requires(remote_name)`.
+    for marker in item.iter_markers():
+        item.config.dvc_config.apply_marker(marker)
+
+
+@pytest.fixture(scope="session")
+def test_config(request):
+    return request.config.dvc_config
+
+
+def pytest_configure(config):
+    config.dvc_config = DVCTestConfig()
+
+    for remote_name in REMOTES:
+        config.addinivalue_line(
+            "markers", f"{remote_name}: mark test as requiring {remote_name}"
+        )
+
+    enabled_remotes = config.dvc_config.enabled_remotes
+    if config.getoption("--all"):
+        enabled_remotes.update(REMOTES)
+    else:
+        default_enabled = {k for k, v in REMOTES.items() if v}
+        enabled_remotes.update(default_enabled)
+
+    for remote_name in REMOTES:
+        enabled_opt = _get_opt(remote_name, "enable")
+        disabled_opt = _get_opt(remote_name, "disable")
+
+        enabled = config.getoption(enabled_opt)
+        disabled = config.getoption(disabled_opt)
+        if disabled and enabled:
+            continue  # default behavior if both flags are supplied
+
+        if disabled:
+            enabled_remotes.discard(remote_name)
+        if enabled:
+            enabled_remotes.add(remote_name)

tests/func/test_output.py

@@ -4,7 +4,10 @@ from dvc.output import OUTS_MAP, _get
 from dvc.stage import Stage
 from tests import PY39, PYARROW_NOT_AVAILABLE
 
-MARKERS = [pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE)]
+MARKERS = [
+    pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE),
+    pytest.mark.hdfs,
+]
 
 TESTS = [
     ("s3://bucket/path", "s3"),

tests/remotes/azure.py

@@ -23,7 +23,9 @@ class Azure(Base, CloudURLInfo):
 
 
 @pytest.fixture(scope="session")
-def azure_server(docker_compose, docker_services):
+def azure_server(test_config, docker_compose, docker_services):
+    test_config.requires("azure")
+
     from azure.core.exceptions import (  # pylint: disable=no-name-in-module
         AzureError,
     )

tests/remotes/gdrive.py

@@ -38,7 +38,8 @@ class GDrive(Base, CloudURLInfo):
 
 
 @pytest.fixture
-def gdrive(make_tmp_dir):
+def gdrive(test_config, make_tmp_dir):
+    test_config.requires("gdrive")
     if not GDrive.should_test():
         pytest.skip("no gdrive")
 

tests/remotes/gs.py

@@ -102,7 +102,8 @@ class GCP(Base, CloudURLInfo):
 
 
 @pytest.fixture
-def gs():
+def gs(test_config):
+    test_config.requires("gs")
     if not GCP.should_test():
         pytest.skip("no gs")
     yield GCP(GCP.get_url())

tests/remotes/hdfs.py

@@ -70,7 +70,9 @@ class HDFS(Base, URLInfo):  # pylint: disable=abstract-method
 
 
 @pytest.fixture(scope="session")
-def hadoop():
+def hadoop(test_config):
+    test_config.requires("hdfs")
+
     if platform.system() != "Linux":
         pytest.skip("only supported on Linux")
 

tests/remotes/http.py

@@ -35,9 +35,10 @@ class HTTP(Base, HTTPURLInfo):
 
 
 @pytest.fixture(scope="session")
-def http_server(tmp_path_factory):
+def http_server(test_config, tmp_path_factory):
     from tests.utils.httpd import StaticFileServer
 
+    test_config.requires("http")
     directory = os.fspath(tmp_path_factory.mktemp("http"))
     with StaticFileServer(directory=directory) as httpd:
         yield httpd

tests/remotes/oss.py

@@ -35,9 +35,10 @@ class OSS(Base, CloudURLInfo):
 
 
 @pytest.fixture(scope="session")
-def oss_server(docker_compose, docker_services):
+def oss_server(test_config, docker_compose, docker_services):
     import oss2
 
+    test_config.requires("oss")
     port = docker_services.port_for("oss", 8880)
     endpoint = EMULATOR_OSS_ENDPOINT.format(port=port)
 
@@ -72,7 +73,8 @@ def oss(oss_server):
 
 
 @pytest.fixture
-def real_oss():
+def real_oss(test_config):
+    test_config.requires("oss")
     if not OSS.should_test():
         pytest.skip("no real OSS")
 

tests/remotes/s3.py

@@ -99,7 +99,8 @@ class S3(Base, CloudURLInfo):
 
 
 @pytest.fixture
-def s3():
+def s3(test_config):
+    test_config.requires("s3")
     with mock_s3():
         import boto3
 
@@ -109,7 +110,8 @@ def s3():
 
 
 @pytest.fixture
-def real_s3():
+def real_s3(test_config):
+    test_config.requires("s3")
     if not S3.should_test():
         pytest.skip("no real s3")
     yield S3(S3.get_url())

tests/remotes/ssh.py

@@ -113,9 +113,10 @@ class SSHMocked(Base, URLInfo):
 
 
 @pytest.fixture
-def ssh_server():
+def ssh_server(test_config):
     import mockssh
 
+    test_config.requires("ssh")
     users = {TEST_SSH_USER: TEST_SSH_KEY_PATH}
     with mockssh.Server(users) as s:
         yield s

tests/remotes/webdav.py

@@ -33,7 +33,9 @@ class Webdav(Base, WebDAVURLInfo):
 
 
 @pytest.fixture
-def webdav_server(tmp_path_factory):
+def webdav_server(test_config, tmp_path_factory):
+    test_config.requires("webdav")
+
     host, port = "localhost", 0
     directory = os.fspath(tmp_path_factory.mktemp("http"))
     dirmap = {"/": directory}

tests/unit/dependency/test_hdfs.py

@@ -6,6 +6,7 @@ from tests.unit.dependency.test_local import TestLocalDependency
 
 
 @pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE)
+@pytest.mark.hdfs
 class TestHDFSDependency(TestLocalDependency):
     def _get_cls(self):
         return HDFSDependency

tests/unit/output/test_hdfs.py

@@ -6,6 +6,7 @@ from tests.unit.output.test_local import TestLocalOutput
 
 
 @pytest.mark.skipif(PY39, reason=PYARROW_NOT_AVAILABLE)
+@pytest.mark.hdfs
 class TestHDFSOutput(TestLocalOutput):
     def _get_cls(self):
         return HDFSOutput