test: merge gcp related functions to GCP helper class

This commit is contained in:
Saugat Pachhai
2020-01-13 23:49:22 +05:45
parent d55427dbd9
commit 26d85f1a52
3 changed files with 41 additions and 42 deletions
+5 -6
View File
@@ -31,8 +31,8 @@ from tests.basic_env import TestDvc
from tests.utils import spy
from tests.remotes import (
_should_test_gcp,
Azure,
GCP,
GDrive,
HDFS,
S3,
@@ -44,7 +44,6 @@ from tests.remotes import (
TEST_GDRIVE_CLIENT_ID,
TEST_GDRIVE_CLIENT_SECRET,
TEST_REMOTE,
get_gcp_url,
get_local_url,
)
@@ -231,7 +230,7 @@ class TestRemoteGDrive(TestDataCloudBase):
class TestRemoteGS(TestDataCloudBase):
def _should_test(self):
return _should_test_gcp()
return GCP.should_test()
def _setup_cloud(self):
self._ensure_should_run()
@@ -249,7 +248,7 @@ class TestRemoteGS(TestDataCloudBase):
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
def _get_url(self):
return get_gcp_url()
return GCP.get_url()
def _get_cloud_class(self):
return RemoteGS
@@ -484,10 +483,10 @@ class TestRemoteGDriveCLI(TestDataCloudCLIBase):
class TestRemoteGSCLI(TestDataCloudCLIBase):
def _should_test(self):
return _should_test_gcp()
return GCP.should_test()
def _test(self):
url = get_gcp_url()
url = GCP.get_url()
self.main(["remote", "add", TEST_REMOTE, url])
self.main(
+6 -3
View File
@@ -9,6 +9,7 @@ from pathlib import Path
from subprocess import PIPE
from subprocess import Popen
from urllib.parse import urljoin
from unittest import SkipTest
import boto3
import paramiko
@@ -35,7 +36,7 @@ from dvc.utils.stage import dump_stage_file
from dvc.utils.stage import load_stage_file
from tests.basic_env import TestDvc
from tests.remotes import (
_should_test_gcp,
GCP,
HDFS,
S3,
SSH,
@@ -857,7 +858,9 @@ class TestReproExternalBase(TestDvc):
@patch("dvc.prompt.confirm", return_value=True)
def test(self, mock_prompt):
if not self.should_test():
return
raise SkipTest(
"Test {} is disabled".format(self.__class__.__name__)
)
cache = (
self.scheme
@@ -975,7 +978,7 @@ class TestReproExternalS3(TestReproExternalBase):
class TestReproExternalGS(TestReproExternalBase):
def should_test(self):
return _should_test_gcp()
return GCP.should_test()
@property
def scheme(self):
+30 -33
View File
@@ -43,29 +43,6 @@ TEST_GDRIVE_CLIENT_ID = (
TEST_GDRIVE_CLIENT_SECRET = "2fy_HyzSwkxkGzEken7hThXb"
def _should_test_gcp():
do_test = env2bool("DVC_TEST_GCP", undefined=None)
if do_test is not None:
return do_test
if not os.path.exists(TEST_GCP_CREDS_FILE):
return False
try:
check_output(
[
"gcloud",
"auth",
"activate-service-account",
"--key-file",
TEST_GCP_CREDS_FILE,
]
)
except (CalledProcessError, OSError):
return False
return True
def get_local_storagepath():
return TestDvc.mkdtemp()
@@ -74,14 +51,6 @@ def get_local_url():
return get_local_storagepath()
def get_gcp_storagepath():
return TEST_GCP_REPO_BUCKET + "/" + str(uuid.uuid4())
def get_gcp_url():
return "gs://" + get_gcp_storagepath()
class Local:
should_test = lambda: True # noqa: E731
get_url = get_local_url
@@ -131,8 +100,36 @@ class S3Mocked(S3):
class GCP:
should_test = _should_test_gcp
get_url = get_gcp_url
@staticmethod
def should_test():
do_test = env2bool("DVC_TEST_GCP", undefined=None)
if do_test is not None:
return do_test
if not os.path.exists(TEST_GCP_CREDS_FILE):
return False
try:
check_output(
[
"gcloud",
"auth",
"activate-service-account",
"--key-file",
TEST_GCP_CREDS_FILE,
]
)
except (CalledProcessError, OSError):
return False
return True
@staticmethod
def get_storagepath():
return TEST_GCP_REPO_BUCKET + "/" + str(uuid.uuid4())
@staticmethod
def get_url():
return "gs://" + GCP.get_storagepath()
@classmethod
@contextmanager