diff --git a/.appveyor.yml b/.appveyor.yml index e91886771..f019b8bb5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -10,7 +10,6 @@ environment: secure: veTfhOrd4wUpG7NlRsvS3tufuaFfST8i7SZa2HVH6Vc= aws_secret_access_key: secure: nCQcN/Mzce0erwcNOU3yAuWgF6ZuTvPBEqTF3XFp1poGcDbSIg+s67eVxsxWCXQj - GOOGLE_APPLICATION_CREDENTIALS: gcp-creds.json GCP_CREDS: secure: 96fJ3r2i2GohbXHwnSs5N4EplQ7q8YmLpPWM0AC+f4s= CODECOV_TOKEN: diff --git a/dvc/config.py b/dvc/config.py index b726620a4..2281c2fbd 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -210,6 +210,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes # backward compatibility SECTION_GCP = "gcp" SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH + SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH SECTION_GCP_PROJECTNAME = "projectname" SECTION_GCP_SCHEMA = { SECTION_GCP_STORAGEPATH: str, diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 0752d782c..ac5d1f2a6 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -4,6 +4,7 @@ import os try: from google.cloud import storage + from google.auth.credentials import Credentials except ImportError: storage = None @@ -28,6 +29,7 @@ class RemoteGS(RemoteBase): storagepath.lstrip("/") self.url = config.get(Config.SECTION_REMOTE_URL, storagepath) self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None) + self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH) parsed = urlparse(self.url) self.bucket = parsed.netloc @@ -44,7 +46,11 @@ class RemoteGS(RemoteBase): @property def gs(self): - return storage.Client(self.projectname) + return ( + storage.Client.from_service_account_json(self.credentialpath) + if self.credentialpath + else storage.Client(self.projectname) + ) def get_md5(self, bucket, path): import base64 diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index aba9a48b5..22c1c9718 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -63,23 +63,7 @@ def _should_test_gcp(): if not os.path.exists(TestDvc.GCP_CREDS_FILE): return False - creds = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") - if creds and os.getenv("GCP_CREDS"): - if os.path.exists(creds): - os.unlink(creds) - shutil.copyfile(TestDvc.GCP_CREDS_FILE, creds) - try: - check_output( - [ - "gcloud", - "auth", - "activate-service-account", - "--key-file", - creds, - ] - ) - except (CalledProcessError, OSError): - return False + if os.getenv("GCP_CREDS"): return True return False @@ -331,6 +315,20 @@ class TestRemoteGS(TestDataCloudBase): def _should_test(self): return _should_test_gcp() + def _setup_cloud(self): + self._ensure_should_run() + + repo = self._get_url() + + config = TEST_CONFIG + config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo + config[TEST_SECTION][ + Config.SECTION_GCP_CREDENTIALPATH + ] = TestDvc.GCP_CREDS_FILE + self.cloud = DataCloud(self.dvc, config) + + self.assertIsInstance(self.cloud._cloud, self._get_cloud_class()) + def _get_url(self): return get_gcp_url() @@ -561,6 +559,15 @@ class TestRemoteGSCLI(TestDataCloudCLIBase): url = get_gcp_url() self.main(["remote", "add", TEST_REMOTE, url]) + self.main( + [ + "remote", + "modify", + TEST_REMOTE, + "credentialpath", + TestDvc.GCP_CREDS_FILE, + ] + ) self._test_cloud(TEST_REMOTE) diff --git a/tests/unit/remote/gs.py b/tests/unit/remote/gs.py index 20f503952..f4ec8c7e2 100644 --- a/tests/unit/remote/gs.py +++ b/tests/unit/remote/gs.py @@ -8,8 +8,13 @@ class TestRemoteGS(TestCase): BUCKET = "bucket" PREFIX = "prefix" URL = "gs://{}/{}".format(BUCKET, PREFIX) + CREDENTIALPATH = "/path/to/gcp_credentials.json" PROJECT = "PROJECT" - CONFIG = {"projectname": PROJECT, "url": URL} + CONFIG = { + "projectname": PROJECT, + "url": URL, + "credentialpath": CREDENTIALPATH, + } def test_init(self): remote = RemoteGS(None, self.CONFIG) @@ -17,9 +22,19 @@ class TestRemoteGS(TestCase): self.assertEqual(remote.prefix, self.PREFIX) self.assertEqual(remote.bucket, self.BUCKET) self.assertEqual(remote.projectname, self.PROJECT) + self.assertEqual(remote.credentialpath, self.CREDENTIALPATH) - @mock.patch("google.cloud.storage.Client") + @mock.patch("google.cloud.storage.Client.from_service_account_json") def test_gs(self, mock_client): remote = RemoteGS(None, self.CONFIG) + self.assertTrue(remote.credentialpath) + remote.gs() + mock_client.assert_called_once_with(self.CREDENTIALPATH) + + @mock.patch("google.cloud.storage.Client") + def test_gs_no_credspath(self, mock_client): + config = self.CONFIG.copy() + del config["credentialpath"] + remote = RemoteGS(None, config) remote.gs() mock_client.assert_called_with(self.PROJECT)