Add GCP credentials environment variable to config (#1567)

.appveyor.yml

@@ -10,7 +10,6 @@ environment:
     secure: veTfhOrd4wUpG7NlRsvS3tufuaFfST8i7SZa2HVH6Vc=
   aws_secret_access_key:
     secure: nCQcN/Mzce0erwcNOU3yAuWgF6ZuTvPBEqTF3XFp1poGcDbSIg+s67eVxsxWCXQj
-  GOOGLE_APPLICATION_CREDENTIALS: gcp-creds.json
   GCP_CREDS:
     secure: 96fJ3r2i2GohbXHwnSs5N4EplQ7q8YmLpPWM0AC+f4s=
   CODECOV_TOKEN:

dvc/config.py

@@ -210,6 +210,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     # backward compatibility
     SECTION_GCP = "gcp"
     SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
+    SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH
     SECTION_GCP_PROJECTNAME = "projectname"
     SECTION_GCP_SCHEMA = {
         SECTION_GCP_STORAGEPATH: str,

dvc/remote/gs.py

@@ -4,6 +4,7 @@ import os
 
 try:
     from google.cloud import storage
+    from google.auth.credentials import Credentials
 except ImportError:
     storage = None
 
@@ -28,6 +29,7 @@ class RemoteGS(RemoteBase):
         storagepath.lstrip("/")
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
+        self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
 
         parsed = urlparse(self.url)
         self.bucket = parsed.netloc
@@ -44,7 +46,11 @@ class RemoteGS(RemoteBase):
 
     @property
     def gs(self):
-        return storage.Client(self.projectname)
+        return (
+            storage.Client.from_service_account_json(self.credentialpath)
+            if self.credentialpath
+            else storage.Client(self.projectname)
+        )
 
     def get_md5(self, bucket, path):
         import base64

tests/test_data_cloud.py

@@ -63,23 +63,7 @@ def _should_test_gcp():
     if not os.path.exists(TestDvc.GCP_CREDS_FILE):
         return False
 
-    creds = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
-    if creds and os.getenv("GCP_CREDS"):
-        if os.path.exists(creds):
-            os.unlink(creds)
-        shutil.copyfile(TestDvc.GCP_CREDS_FILE, creds)
-        try:
-            check_output(
-                [
-                    "gcloud",
-                    "auth",
-                    "activate-service-account",
-                    "--key-file",
-                    creds,
-                ]
-            )
-        except (CalledProcessError, OSError):
-            return False
+    if os.getenv("GCP_CREDS"):
         return True
 
     return False
@@ -331,6 +315,20 @@ class TestRemoteGS(TestDataCloudBase):
     def _should_test(self):
         return _should_test_gcp()
 
+    def _setup_cloud(self):
+        self._ensure_should_run()
+
+        repo = self._get_url()
+
+        config = TEST_CONFIG
+        config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
+        config[TEST_SECTION][
+            Config.SECTION_GCP_CREDENTIALPATH
+        ] = TestDvc.GCP_CREDS_FILE
+        self.cloud = DataCloud(self.dvc, config)
+
+        self.assertIsInstance(self.cloud._cloud, self._get_cloud_class())
+
     def _get_url(self):
         return get_gcp_url()
 
@@ -561,6 +559,15 @@ class TestRemoteGSCLI(TestDataCloudCLIBase):
         url = get_gcp_url()
 
         self.main(["remote", "add", TEST_REMOTE, url])
+        self.main(
+            [
+                "remote",
+                "modify",
+                TEST_REMOTE,
+                "credentialpath",
+                TestDvc.GCP_CREDS_FILE,
+            ]
+        )
 
         self._test_cloud(TEST_REMOTE)
 

tests/unit/remote/gs.py

@@ -8,8 +8,13 @@ class TestRemoteGS(TestCase):
     BUCKET = "bucket"
     PREFIX = "prefix"
     URL = "gs://{}/{}".format(BUCKET, PREFIX)
+    CREDENTIALPATH = "/path/to/gcp_credentials.json"
     PROJECT = "PROJECT"
-    CONFIG = {"projectname": PROJECT, "url": URL}
+    CONFIG = {
+        "projectname": PROJECT,
+        "url": URL,
+        "credentialpath": CREDENTIALPATH,
+    }
 
     def test_init(self):
         remote = RemoteGS(None, self.CONFIG)
@@ -17,9 +22,19 @@ class TestRemoteGS(TestCase):
         self.assertEqual(remote.prefix, self.PREFIX)
         self.assertEqual(remote.bucket, self.BUCKET)
         self.assertEqual(remote.projectname, self.PROJECT)
+        self.assertEqual(remote.credentialpath, self.CREDENTIALPATH)
 
-    @mock.patch("google.cloud.storage.Client")
+    @mock.patch("google.cloud.storage.Client.from_service_account_json")
     def test_gs(self, mock_client):
         remote = RemoteGS(None, self.CONFIG)
+        self.assertTrue(remote.credentialpath)
+        remote.gs()
+        mock_client.assert_called_once_with(self.CREDENTIALPATH)
+
+    @mock.patch("google.cloud.storage.Client")
+    def test_gs_no_credspath(self, mock_client):
+        config = self.CONFIG.copy()
+        del config["credentialpath"]
+        remote = RemoteGS(None, config)
         remote.gs()
         mock_client.assert_called_with(self.PROJECT)