tests: rerun gdrive and gs on failures (#5959)

we just need to mark them with needs_internet marker.
Unfortunately, it's not possible to do that automatically
with the fixtures as fixtures are per-session and we need
something on more higher level (which the marker is not passed
through in the pytest).

setup.cfg

@@ -24,6 +24,8 @@ timeout = 600
 timeout_method = thread
 log_level = debug
 addopts = -ra
+markers =
+    needs_internet: Might need network access for the tests
 
 [mypy]
 # Error output

tests/conftest.py

@@ -111,6 +111,14 @@ def pytest_runtest_setup(item):
     for marker in item.iter_markers():
         item.config.dvc_config.apply_marker(marker)
 
+    if (
+        "CI" in os.environ
+        and item.get_closest_marker("needs_internet") is not None
+    ):
+        # remotes that need internet connection might be flaky,
+        # so we rerun them in case it fails.
+        item.add_marker(pytest.mark.flaky(max_runs=5, min_passes=1))
+
 
 @pytest.fixture(scope="session")
 def test_config(request):

tests/func/test_add.py

@@ -1146,7 +1146,9 @@ def test_add_to_cache_invalid_combinations(dvc, invalid_opt, kwargs):
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
-        pytest.lazy_fixture("gs"),
+        pytest.param(
+            pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
+        ),
         pytest.lazy_fixture("hdfs"),
         pytest.param(
             pytest.lazy_fixture("ssh"),

tests/func/test_api.py

@@ -1,7 +1,6 @@
 import os
 
 import pytest
-from flaky.flaky_decorator import flaky
 from funcy import first, get_in
 
 from dvc import api
@@ -28,6 +27,7 @@ all_clouds = [pytest.lazy_fixture("local_cloud")] + clouds
 # `lazy_fixture` is confusing pylint, pylint: disable=unused-argument
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("remote", clouds, indirect=True)
 def test_get_url(tmp_dir, dvc, remote):
     tmp_dir.dvc_gen("foo", "foo")
@@ -36,6 +36,7 @@ def test_get_url(tmp_dir, dvc, remote):
     assert api.get_url("foo") == expected_url
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("cloud", clouds)
 def test_get_url_external(tmp_dir, erepo_dir, cloud):
     erepo_dir.add_remote(config=cloud.config)
@@ -58,6 +59,7 @@ def test_get_url_requires_dvc(tmp_dir, scm):
         api.get_url("foo", repo=f"file://{tmp_dir}")
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("remote", all_clouds, indirect=True)
 def test_open(tmp_dir, dvc, remote):
     tmp_dir.dvc_gen("foo", "foo-text")
@@ -70,6 +72,7 @@ def test_open(tmp_dir, dvc, remote):
         assert fd.read() == "foo-text"
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize(
     "cloud",
     [
@@ -110,7 +113,7 @@ def test_open_external(tmp_dir, erepo_dir, cloud):
     assert api.read("version", repo=repo_url, rev="branch") == "branchver"
 
 
-@flaky(max_runs=3, min_passes=1)
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("remote", all_clouds, indirect=True)
 def test_open_granular(tmp_dir, dvc, remote):
     tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}})
@@ -123,6 +126,7 @@ def test_open_granular(tmp_dir, dvc, remote):
         assert fd.read() == "foo-text"
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize(
     "remote",
     [

tests/func/test_data_cloud.py

@@ -42,6 +42,7 @@ all_clouds = [
 ]
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("remote", all_clouds, indirect=True)
 def test_cloud(tmp_dir, dvc, remote):  # pylint:disable=unused-argument
     (stage,) = tmp_dir.dvc_gen("foo", "foo")
@@ -136,6 +137,7 @@ def test_cloud(tmp_dir, dvc, remote):  # pylint:disable=unused-argument
     assert status_dir == expected
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("remote", all_clouds, indirect=True)
 def test_cloud_cli(tmp_dir, dvc, remote):
     args = ["-v", "-j", "2"]

tests/func/test_fs.py

@@ -237,7 +237,9 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
-        pytest.lazy_fixture("gs"),
+        pytest.param(
+            pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
+        ),
         pytest.lazy_fixture("hdfs"),
         pytest.lazy_fixture("http"),
     ],
@@ -251,6 +253,7 @@ def test_fs_getsize(dvc, cloud):
     assert fs.getsize(path_info / "data" / "foo") == 3
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize(
     "cloud",
     [
@@ -281,6 +284,7 @@ def test_fs_upload_fobj(dvc, tmp_dir, cloud):
         assert stream.read() == b"foo"
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("cloud", [pytest.lazy_fixture("gdrive")])
 def test_fs_ls(dvc, cloud):
     cloud.gen(
@@ -309,6 +313,7 @@ def test_fs_ls(dvc, cloud):
     } == {("file", "quux"), ("directory", "egg")}
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize(
     "cloud",
     [
@@ -334,7 +339,9 @@ def test_fs_find_recursive(dvc, cloud):
     [
         pytest.lazy_fixture("s3"),
         pytest.lazy_fixture("azure"),
-        pytest.lazy_fixture("gs"),
+        pytest.param(
+            pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
+        ),
         pytest.lazy_fixture("webdav"),
     ],
 )
@@ -351,7 +358,13 @@ def test_fs_find_with_etag(dvc, cloud):
 
 
 @pytest.mark.parametrize(
-    "cloud", [pytest.lazy_fixture("azure"), pytest.lazy_fixture("gs")]
+    "cloud",
+    [
+        pytest.lazy_fixture("azure"),
+        pytest.param(
+            pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
+        ),
+    ],
 )
 def test_fs_fsspec_path_management(dvc, cloud):
     cloud.gen({"foo": "foo", "data": {"bar": "bar", "baz": {"foo": "foo"}}})

tests/func/test_import_url.py

@@ -123,7 +123,9 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
-        pytest.lazy_fixture("gs"),
+        pytest.param(
+            pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
+        ),
         pytest.lazy_fixture("hdfs"),
         pytest.lazy_fixture("webhdfs"),
         pytest.param(
@@ -249,7 +251,9 @@ def test_import_url_preserve_meta(tmp_dir, dvc):
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
-        pytest.lazy_fixture("gs"),
+        pytest.param(
+            pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
+        ),
         pytest.lazy_fixture("hdfs"),
         pytest.param(
             pytest.lazy_fixture("ssh"),
@@ -285,7 +289,9 @@ def test_import_url_to_remote_single_file(
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
-        pytest.lazy_fixture("gs"),
+        pytest.param(
+            pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
+        ),
         pytest.lazy_fixture("hdfs"),
         pytest.param(
             pytest.lazy_fixture("ssh"),

tests/func/test_update.py

@@ -163,7 +163,9 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
-        pytest.lazy_fixture("gs"),
+        pytest.param(
+            pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
+        ),
         pytest.lazy_fixture("hdfs"),
         pytest.lazy_fixture("webhdfs"),
         pytest.param(
@@ -331,7 +333,9 @@ def test_update_import_to_remote(tmp_dir, dvc, erepo_dir, local_remote):
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
-        pytest.lazy_fixture("gs"),
+        pytest.param(
+            pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
+        ),
         pytest.lazy_fixture("hdfs"),
     ],
     indirect=True,
@@ -352,7 +356,9 @@ def test_update_import_url_to_remote(tmp_dir, dvc, workspace, local_remote):
     [
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
-        pytest.lazy_fixture("gs"),
+        pytest.param(
+            pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
+        ),
         pytest.lazy_fixture("hdfs"),
     ],
     indirect=True,

tests/unit/remote/test_remote_tree.py

@@ -32,6 +32,7 @@ def remote(request, dvc):
     return get_remote(dvc, **cloud.config)
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("remote", remotes, indirect=True)
 def test_isdir(remote):
     test_cases = [
@@ -49,6 +50,7 @@ def test_isdir(remote):
         assert remote.fs.isdir(remote.fs.path_info / path) == expected
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("remote", remotes, indirect=True)
 def test_exists(remote):
     test_cases = [
@@ -69,6 +71,7 @@ def test_exists(remote):
         assert remote.fs.exists(remote.fs.path_info / path) == expected
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("remote", remotes, indirect=True)
 def test_walk_files(remote):
     files = [
@@ -125,6 +128,7 @@ def test_makedirs(remote):
     assert fs.isdir(empty_dir)
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("remote", remotes, indirect=True)
 def test_isfile(remote):
     test_cases = [
@@ -147,6 +151,7 @@ def test_isfile(remote):
         assert remote.fs.isfile(remote.fs.path_info / path) == expected
 
 
+@pytest.mark.needs_internet
 @pytest.mark.parametrize("remote", remotes, indirect=True)
 def test_download_dir(remote, tmpdir):
     path = str(tmpdir / "data")