tests: convert remote unit tests to pytest (#3604)

- update tests/unit/remote tests to use pytest style tests
- use pytest dvc repo fixtures instead of improperly mocked repo's or
  None

tests/remotes.py

@@ -78,9 +78,9 @@ class S3Mocked(S3):
 
     @classmethod
     @contextmanager
-    def remote(cls):
+    def remote(cls, repo):
         with mock_s3():
-            yield RemoteS3(None, {"url": cls.get_url()})
+            yield RemoteS3(repo, {"url": cls.get_url()})
 
     @staticmethod
     def put_objects(remote, objects):
@@ -127,8 +127,8 @@ class GCP:
 
     @classmethod
     @contextmanager
-    def remote(cls):
-        yield RemoteGS(None, {"url": cls.get_url()})
+    def remote(cls, repo):
+        yield RemoteGS(repo, {"url": cls.get_url()})
 
     @staticmethod
     def put_objects(remote, objects):

tests/unit/remote/ssh/test_ssh.py

@@ -1,7 +1,6 @@
 import getpass
 import os
 import sys
-from unittest import TestCase
 
 import pytest
 from mock import mock_open
@@ -12,31 +11,31 @@ from dvc.system import System
 from tests.remotes import SSHMocked
 
 
-class TestRemoteSSH(TestCase):
-    def test_url(self):
-        user = "test"
-        host = "123.45.67.89"
-        port = 1234
-        path = "/path/to/dir"
+def test_url(dvc):
+    user = "test"
+    host = "123.45.67.89"
+    port = 1234
+    path = "/path/to/dir"
 
-        # URL ssh://[user@]host.xz[:port]/path
-        url = "ssh://{}@{}:{}{}".format(user, host, port, path)
-        config = {"url": url}
+    # URL ssh://[user@]host.xz[:port]/path
+    url = "ssh://{}@{}:{}{}".format(user, host, port, path)
+    config = {"url": url}
 
-        remote = RemoteSSH(None, config)
-        self.assertEqual(remote.path_info, url)
+    remote = RemoteSSH(dvc, config)
+    assert remote.path_info == url
 
-        # SCP-like URL ssh://[user@]host.xz:/absolute/path
-        url = "ssh://{}@{}:{}".format(user, host, path)
-        config = {"url": url}
+    # SCP-like URL ssh://[user@]host.xz:/absolute/path
+    url = "ssh://{}@{}:{}".format(user, host, path)
+    config = {"url": url}
 
-        remote = RemoteSSH(None, config)
-        self.assertEqual(remote.path_info, url)
+    remote = RemoteSSH(dvc, config)
+    assert remote.path_info == url
 
-    def test_no_path(self):
-        config = {"url": "ssh://127.0.0.1"}
-        remote = RemoteSSH(None, config)
-        self.assertEqual(remote.path_info.path, "")
+
+def test_no_path(dvc):
+    config = {"url": "ssh://127.0.0.1"}
+    remote = RemoteSSH(dvc, config)
+    assert remote.path_info.path == ""
 
 
 mock_ssh_config = """
@@ -67,9 +66,9 @@ else:
     read_data=mock_ssh_config,
 )
 def test_ssh_host_override_from_config(
-    mock_file, mock_exists, config, expected_host
+    mock_file, mock_exists, dvc, config, expected_host
 ):
-    remote = RemoteSSH(None, config)
+    remote = RemoteSSH(dvc, config)
 
     mock_exists.assert_called_with(RemoteSSH.ssh_config_filename())
     mock_file.assert_called_with(RemoteSSH.ssh_config_filename())
@@ -96,8 +95,8 @@ def test_ssh_host_override_from_config(
     new_callable=mock_open,
     read_data=mock_ssh_config,
 )
-def test_ssh_user(mock_file, mock_exists, config, expected_user):
-    remote = RemoteSSH(None, config)
+def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user):
+    remote = RemoteSSH(dvc, config)
 
     mock_exists.assert_called_with(RemoteSSH.ssh_config_filename())
     mock_file.assert_called_with(RemoteSSH.ssh_config_filename())
@@ -121,8 +120,8 @@ def test_ssh_user(mock_file, mock_exists, config, expected_user):
     new_callable=mock_open,
     read_data=mock_ssh_config,
 )
-def test_ssh_port(mock_file, mock_exists, config, expected_port):
-    remote = RemoteSSH(None, config)
+def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port):
+    remote = RemoteSSH(dvc, config)
 
     mock_exists.assert_called_with(RemoteSSH.ssh_config_filename())
     mock_file.assert_called_with(RemoteSSH.ssh_config_filename())
@@ -156,8 +155,8 @@ def test_ssh_port(mock_file, mock_exists, config, expected_port):
     new_callable=mock_open,
     read_data=mock_ssh_config,
 )
-def test_ssh_keyfile(mock_file, mock_exists, config, expected_keyfile):
-    remote = RemoteSSH(None, config)
+def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile):
+    remote = RemoteSSH(dvc, config)
 
     mock_exists.assert_called_with(RemoteSSH.ssh_config_filename())
     mock_file.assert_called_with(RemoteSSH.ssh_config_filename())
@@ -178,15 +177,15 @@ def test_ssh_keyfile(mock_file, mock_exists, config, expected_keyfile):
     new_callable=mock_open,
     read_data=mock_ssh_config,
 )
-def test_ssh_gss_auth(mock_file, mock_exists, config, expected_gss_auth):
-    remote = RemoteSSH(None, config)
+def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth):
+    remote = RemoteSSH(dvc, config)
 
     mock_exists.assert_called_with(RemoteSSH.ssh_config_filename())
     mock_file.assert_called_with(RemoteSSH.ssh_config_filename())
     assert remote.gss_auth == expected_gss_auth
 
 
-def test_hardlink_optimization(tmp_dir, ssh_server):
+def test_hardlink_optimization(dvc, tmp_dir, ssh_server):
     port = ssh_server.test_creds["port"]
     user = ssh_server.test_creds["username"]
 
@@ -196,7 +195,7 @@ def test_hardlink_optimization(tmp_dir, ssh_server):
         "user": user,
         "keyfile": ssh_server.test_creds["key_filename"],
     }
-    remote = RemoteSSH(None, config)
+    remote = RemoteSSH(dvc, config)
 
     from_info = remote.path_info / "empty"
     to_info = remote.path_info / "link"

tests/unit/remote/test_azure.py

@@ -1,33 +1,31 @@
-from unittest import TestCase
-
 from dvc.remote.azure import RemoteAZURE
 
 
-class TestRemoteAZURE(TestCase):
-    container_name = "container-name"
-    connection_string = (
-        "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;"
-        "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu"
-        "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
-        "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
+container_name = "container-name"
+connection_string = (
+    "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;"
+    "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu"
+    "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
+    "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
+)
+
+
+def test_init_compat(dvc):
+    url = (
+        "azure://ContainerName={container_name};{connection_string}"
+    ).format(
+        container_name=container_name, connection_string=connection_string,
     )
+    config = {"url": url}
+    remote = RemoteAZURE(dvc, config)
+    assert remote.path_info == "azure://" + container_name
+    assert remote.connection_string == connection_string
 
-    def test_init_compat(self):
-        url = (
-            "azure://ContainerName={container_name};{connection_string}"
-        ).format(
-            container_name=self.container_name,
-            connection_string=self.connection_string,
-        )
-        config = {"url": url}
-        remote = RemoteAZURE(None, config)
-        self.assertEqual(remote.path_info, "azure://" + self.container_name)
-        self.assertEqual(remote.connection_string, self.connection_string)
 
-    def test_init(self):
-        prefix = "some/prefix"
-        url = "azure://{}/{}".format(self.container_name, prefix)
-        config = {"url": url, "connection_string": self.connection_string}
-        remote = RemoteAZURE(None, config)
-        self.assertEqual(remote.path_info, url)
-        self.assertEqual(remote.connection_string, self.connection_string)
+def test_init(dvc):
+    prefix = "some/prefix"
+    url = "azure://{}/{}".format(container_name, prefix)
+    config = {"url": url, "connection_string": connection_string}
+    remote = RemoteAZURE(dvc, config)
+    assert remote.path_info == url
+    assert remote.connection_string == connection_string

tests/unit/remote/test_base.py

@@ -1,7 +1,6 @@
-from unittest import TestCase
-
 import math
 import mock
+import pytest
 
 from dvc.path_info import PathInfo
 from dvc.remote.base import RemoteBASE
@@ -17,42 +16,36 @@ class _CallableOrNone(object):
 
 
 CallableOrNone = _CallableOrNone()
+REMOTE_CLS = RemoteBASE
 
 
-class TestRemoteBASE(object):
-    REMOTE_CLS = RemoteBASE
+def test_missing_deps(dvc):
+    requires = {"missing": "missing"}
+    with mock.patch.object(REMOTE_CLS, "REQUIRES", requires):
+        with pytest.raises(RemoteMissingDepsError):
+            REMOTE_CLS(dvc, {})
 
 
-class TestMissingDeps(TestCase, TestRemoteBASE):
-    def test(self):
-        requires = {"missing": "missing"}
-        with mock.patch.object(self.REMOTE_CLS, "REQUIRES", requires):
-            with self.assertRaises(RemoteMissingDepsError):
-                self.REMOTE_CLS(None, {})
+def test_cmd_error(dvc):
+    config = {}
 
+    cmd = "sed 'hello'"
+    ret = "1"
+    err = "sed: expression #1, char 2: extra characters after command"
 
-class TestCmdError(TestCase, TestRemoteBASE):
-    def test(self):
-        repo = None
-        config = {}
-
-        cmd = "sed 'hello'"
-        ret = "1"
-        err = "sed: expression #1, char 2: extra characters after command"
-
-        with mock.patch.object(
-            self.REMOTE_CLS,
-            "remove",
-            side_effect=RemoteCmdError("base", cmd, ret, err),
-        ):
-            with self.assertRaises(RemoteCmdError):
-                self.REMOTE_CLS(repo, config).remove("file")
+    with mock.patch.object(
+        REMOTE_CLS,
+        "remove",
+        side_effect=RemoteCmdError("base", cmd, ret, err),
+    ):
+        with pytest.raises(RemoteCmdError):
+            REMOTE_CLS(dvc, config).remove("file")
 
 
 @mock.patch.object(RemoteBASE, "_cache_checksums_traverse")
 @mock.patch.object(RemoteBASE, "_cache_object_exists")
-def test_cache_exists(object_exists, traverse):
-    remote = RemoteBASE(None, {})
+def test_cache_exists(object_exists, traverse, dvc):
+    remote = RemoteBASE(dvc, {})
 
     # remote does not support traverse
     remote.CAN_TRAVERSE = False
@@ -110,8 +103,8 @@ def test_cache_exists(object_exists, traverse):
 @mock.patch.object(
     RemoteBASE, "path_to_checksum", side_effect=lambda x: x,
 )
-def test_cache_checksums_traverse(path_to_checksum, cache_checksums):
-    remote = RemoteBASE(None, {})
+def test_cache_checksums_traverse(path_to_checksum, cache_checksums, dvc):
+    remote = RemoteBASE(dvc, {})
     remote.path_info = PathInfo("foo")
 
     # parallel traverse
@@ -135,8 +128,8 @@ def test_cache_checksums_traverse(path_to_checksum, cache_checksums):
     )
 
 
-def test_cache_checksums():
-    remote = RemoteBASE(None, {})
+def test_cache_checksums(dvc):
+    remote = RemoteBASE(dvc, {})
     remote.path_info = PathInfo("foo")
 
     with mock.patch.object(

tests/unit/remote/test_gdrive.py

@@ -1,7 +1,6 @@
 import pytest
 import os
 
-from dvc.config import Config
 from dvc.remote.gdrive import (
     RemoteGDrive,
     GDriveAccessTokenRefreshError,
@@ -13,11 +12,6 @@ USER_CREDS_TOKEN_REFRESH_ERROR = '{"access_token": "", "client_id": "", "client_
 USER_CREDS_MISSED_KEY_ERROR = "{}"
 
 
-class Repo(object):
-    tmp_dir = ""
-    config = Config()
-
-
 class TestRemoteGDrive(object):
     CONFIG = {
         "url": "gdrive://root/data",
@@ -25,12 +19,12 @@ class TestRemoteGDrive(object):
         "gdrive_client_secret": "secret",
     }
 
-    def test_init(self):
-        remote = RemoteGDrive(Repo(), self.CONFIG)
+    def test_init(self, dvc):
+        remote = RemoteGDrive(dvc, self.CONFIG)
         assert str(remote.path_info) == self.CONFIG["url"]
 
-    def test_drive(self):
-        remote = RemoteGDrive(Repo(), self.CONFIG)
+    def test_drive(self, dvc):
+        remote = RemoteGDrive(dvc, self.CONFIG)
         os.environ[
             RemoteGDrive.GDRIVE_CREDENTIALS_DATA
         ] = USER_CREDS_TOKEN_REFRESH_ERROR
@@ -38,7 +32,7 @@ class TestRemoteGDrive(object):
             remote._drive
 
         os.environ[RemoteGDrive.GDRIVE_CREDENTIALS_DATA] = ""
-        remote = RemoteGDrive(Repo(), self.CONFIG)
+        remote = RemoteGDrive(dvc, self.CONFIG)
         os.environ[
             RemoteGDrive.GDRIVE_CREDENTIALS_DATA
         ] = USER_CREDS_MISSED_KEY_ERROR

tests/unit/remote/test_gs.py

@@ -1,5 +1,3 @@
-from unittest import TestCase
-
 import mock
 import pytest
 import requests
@@ -8,38 +6,40 @@ from dvc.remote.gs import dynamic_chunk_size
 from dvc.remote.gs import RemoteGS
 
 
-class TestRemoteGS(TestCase):
-    BUCKET = "bucket"
-    PREFIX = "prefix"
-    URL = "gs://{}/{}".format(BUCKET, PREFIX)
-    CREDENTIALPATH = "/path/to/gcp_credentials.json"
-    PROJECT = "PROJECT"
-    CONFIG = {
-        "projectname": PROJECT,
-        "url": URL,
-        "credentialpath": CREDENTIALPATH,
-    }
+BUCKET = "bucket"
+PREFIX = "prefix"
+URL = "gs://{}/{}".format(BUCKET, PREFIX)
+CREDENTIALPATH = "/path/to/gcp_credentials.json"
+PROJECT = "PROJECT"
+CONFIG = {
+    "projectname": PROJECT,
+    "url": URL,
+    "credentialpath": CREDENTIALPATH,
+}
 
-    def test_init(self):
-        remote = RemoteGS(None, self.CONFIG)
-        self.assertEqual(remote.path_info, self.URL)
-        self.assertEqual(remote.projectname, self.PROJECT)
-        self.assertEqual(remote.credentialpath, self.CREDENTIALPATH)
 
-    @mock.patch("google.cloud.storage.Client.from_service_account_json")
-    def test_gs(self, mock_client):
-        remote = RemoteGS(None, self.CONFIG)
-        self.assertTrue(remote.credentialpath)
-        remote.gs()
-        mock_client.assert_called_once_with(self.CREDENTIALPATH)
+def test_init(dvc):
+    remote = RemoteGS(dvc, CONFIG)
+    assert remote.path_info == URL
+    assert remote.projectname == PROJECT
+    assert remote.credentialpath == CREDENTIALPATH
 
-    @mock.patch("google.cloud.storage.Client")
-    def test_gs_no_credspath(self, mock_client):
-        config = self.CONFIG.copy()
-        del config["credentialpath"]
-        remote = RemoteGS(None, config)
-        remote.gs()
-        mock_client.assert_called_with(self.PROJECT)
+
+@mock.patch("google.cloud.storage.Client.from_service_account_json")
+def test_gs(mock_client, dvc):
+    remote = RemoteGS(dvc, CONFIG)
+    assert remote.credentialpath
+    remote.gs()
+    mock_client.assert_called_once_with(CREDENTIALPATH)
+
+
+@mock.patch("google.cloud.storage.Client")
+def test_gs_no_credspath(mock_client, dvc):
+    config = CONFIG.copy()
+    del config["credentialpath"]
+    remote = RemoteGS(dvc, config)
+    remote.gs()
+    mock_client.assert_called_with(PROJECT)
 
 
 def test_dynamic_chunk_size():

tests/unit/remote/test_oss.py

@@ -1,25 +1,23 @@
-from unittest import TestCase
-
 from dvc.remote.oss import RemoteOSS
 
 
-class TestRemoteOSS(TestCase):
-    bucket_name = "bucket-name"
-    endpoint = "endpoint"
-    key_id = "Fq2UVErCz4I6tq"
-    key_secret = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu"
+bucket_name = "bucket-name"
+endpoint = "endpoint"
+key_id = "Fq2UVErCz4I6tq"
+key_secret = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu"
 
-    def test_init(self):
-        prefix = "some/prefix"
-        url = "oss://{}/{}".format(self.bucket_name, prefix)
-        config = {
-            "url": url,
-            "oss_key_id": self.key_id,
-            "oss_key_secret": self.key_secret,
-            "oss_endpoint": self.endpoint,
-        }
-        remote = RemoteOSS(None, config)
-        self.assertEqual(remote.path_info, url)
-        self.assertEqual(remote.endpoint, self.endpoint)
-        self.assertEqual(remote.key_id, self.key_id)
-        self.assertEqual(remote.key_secret, self.key_secret)
+
+def test_init(dvc):
+    prefix = "some/prefix"
+    url = "oss://{}/{}".format(bucket_name, prefix)
+    config = {
+        "url": url,
+        "oss_key_id": key_id,
+        "oss_key_secret": key_secret,
+        "oss_endpoint": endpoint,
+    }
+    remote = RemoteOSS(dvc, config)
+    assert remote.path_info == url
+    assert remote.endpoint == endpoint
+    assert remote.key_id == key_id
+    assert remote.key_secret == key_secret

tests/unit/remote/test_remote.py

@@ -30,9 +30,9 @@ def test_remote_without_checksum_jobs_default(dvc):
 
 
 @pytest.mark.parametrize("remote_cls", [RemoteGS, RemoteS3])
-def test_makedirs_not_create_for_top_level_path(remote_cls, mocker):
+def test_makedirs_not_create_for_top_level_path(remote_cls, dvc, mocker):
     url = "{.scheme}://bucket/".format(remote_cls)
-    remote = remote_cls(None, {"url": url})
+    remote = remote_cls(dvc, {"url": url})
     mocked_client = mocker.PropertyMock()
     # we use remote clients with same name as scheme to interact with remote
     mocker.patch.object(remote_cls, remote.scheme, mocked_client)

tests/unit/remote/test_remote_dir.py

@@ -25,10 +25,10 @@ FILE_WITH_CONTENTS = {
 
 
 @pytest.fixture
-def remote(request):
+def remote(request, dvc):
     if not request.param.should_test():
         raise pytest.skip()
-    with request.param.remote() as remote:
+    with request.param.remote(dvc) as remote:
         request.param.put_objects(remote, FILE_WITH_CONTENTS)
         yield remote
 
@@ -86,11 +86,11 @@ def test_walk_files(remote):
 
 
 @pytest.mark.parametrize("remote", [S3Mocked], indirect=True)
-def test_copy_preserve_etag_across_buckets(remote):
+def test_copy_preserve_etag_across_buckets(remote, dvc):
     s3 = remote.s3
     s3.create_bucket(Bucket="another")
 
-    another = RemoteS3(None, {"url": "s3://another", "region": "us-east-1"})
+    another = RemoteS3(dvc, {"url": "s3://another", "region": "us-east-1"})
 
     from_info = remote.path_info / "foo"
     to_info = another.path_info / "foo"

tests/unit/remote/test_s3.py

@@ -19,14 +19,14 @@ def grants():
     }
 
 
-def test_init():
+def test_init(dvc):
     config = {"url": url}
-    remote = RemoteS3(None, config)
+    remote = RemoteS3(dvc, config)
 
     assert remote.path_info == url
 
 
-def test_grants():
+def test_grants(dvc):
     config = {
         "url": url,
         "grant_read": "id=read-permission-id,id=other-read-permission-id",
@@ -34,7 +34,7 @@ def test_grants():
         "grant_write_acp": "id=write-acp-permission-id",
         "grant_full_control": "id=full-control-permission-id",
     }
-    remote = RemoteS3(None, config)
+    remote = RemoteS3(dvc, config)
 
     assert (
         remote.extra_args["GrantRead"]
@@ -48,9 +48,9 @@ def test_grants():
     )
 
 
-def test_grants_mutually_exclusive_acl_error(grants):
+def test_grants_mutually_exclusive_acl_error(dvc, grants):
     for grant_option, grant_value in grants.items():
         config = {"url": url, "acl": "public-read", grant_option: grant_value}
 
         with pytest.raises(ConfigError):
-            RemoteS3(None, config)
+            RemoteS3(dvc, config)