tests around reading aws credentials

dvc/config.py

@@ -36,14 +36,21 @@ class ConfigI(object):
 
 class Config(ConfigI):
     CONFIG = 'dvc.conf'
-
-    def __init__(self, conf_file):
+    def __init__(self, conf_file, conf_pseudo_file=None):
+        """
+        Params:
+            conf_file (String): configuration file
+            conf_pseudo_file (String): for unit testing, something that supports readline; supersedes conf_file
+        """
         self._conf_file = conf_file
         self._config = configparser.SafeConfigParser()
 
-        if not os.path.isfile(conf_file):
-            raise ConfigError('Config file "{}" does not exist'.format(conf_file))
-        self._config.read(conf_file)
+        if conf_pseudo_file is not None:
+            self._config.readfp(conf_pseudo_file)
+        else:
+            if not os.path.isfile(conf_file):
+                raise ConfigError('Config file "{}" does not exist'.format(conf_file))
+            self._config.read(conf_file)
 
         level = self._config['Global']['LogLevel']
         Logger.set_level(level)
@@ -167,10 +174,11 @@ class Config(ConfigI):
             cc = configparser.SafeConfigParser()
             threw = False
             try:
-                read = cc.read(path)
-            except:
+                # use readfp(open( ... to aid mocking.
+                cc.readfp(open(path, 'r'))
+            except Exception as e:
                 threw = True
-            if not threw and len(read) > 0 and read[0] == path and 'default' in cc.keys():
+            if not threw and 'default' in cc.keys():
                 access_key = cc['default'].get('aws_access_key_id', None)
                 secret = cc['default'].get('aws_secret_access_key', None)
 
@@ -194,7 +202,7 @@ class Config(ConfigI):
         errors = []
         for key in ['Cloud']:
             if key.lower() not in self._config['Global'].keys() or len(self._config['Global'][key]) < 1:
-                errors.append('Please set %s in section Global in config file %s' % (key, self.CONFIG))
+                errors.append('Please set %s in section Global in config file %s' % (key, self.file))
 
         # now that a cloud is chosen, can check StoragePath
         sp = self.storage_path

requirements.txt

@@ -9,6 +9,7 @@ decorator==4.0.11
 dill==0.2.6
 enum34==1.1.6
 fasteners==0.14.1
+funcsigs==1.0.2
 future==0.16.0
 futures==3.0.5
 gapic-google-cloud-datastore-v1==0.15.3
@@ -48,6 +49,7 @@ httplib2==0.10.3
 ipython==5.3.0
 ipython-genutils==0.2.0
 macholib==1.8
+mock==2.0.0
 modulegraph==0.14
 monotonic==1.3
 nose==1.3.7
@@ -55,6 +57,7 @@ oauth2client==3.0.0
 packaging==16.8
 pathlib==1.0.1
 pathlib2==2.2.1
+pbr==3.0.0
 pexpect==4.2.1
 pickleshare==0.7.4
 ply==3.8

tests/test_config.py

@@ -0,0 +1,129 @@
+import io
+import os
+import shutil
+import StringIO
+import traceback
+from unittest import TestCase
+import mock
+
+
+from dvc.path.path import Path
+from dvc.system import System
+from dvc.config import Config
+
+
+class TestConfigTest(TestCase):
+    def setUp(self):
+        pass
+
+    def storage_path_hierarchy_test(self):
+        """ StoragePath should be read first from global and then from cloud section """
+
+        c = ("[Global]",
+             "LogLevel =",
+             "DataDir = ",
+             "CacheDir = ",
+             "StateDir = ",
+             "Cloud = amazon",
+             "StoragePath = globalsb/global_storage_path",
+             "[AWS]",
+             "StoragePath = awssb/aws_storage_path",
+             "CredentialPath =",
+             "[GC]",
+             "StoragePath = googlesb/google_storage_path"
+           )
+        s = StringIO.StringIO('\n'.join(c))
+        conf = Config(s, conf_pseudo_file = s)
+
+        self.assertEqual(conf.cloud, 'amazon')
+        self.assertEqual(conf.storage_bucket, 'globalsb')
+        self.assertEqual(conf.storage_prefix, 'global_storage_path')
+
+        c = ("[Global]",
+             "LogLevel =",
+             "DataDir = ",
+             "CacheDir = ",
+             "StateDir = ",
+             "Cloud = amazon",
+             "[AWS]",
+             "StoragePath = awssb/aws_storage_path",
+             "CredentialPath =",
+             "[GC]",
+             "StoragePath = googlesb/google_storage_path"
+           )
+        s = StringIO.StringIO('\n'.join(c))
+        conf = Config(s, conf_pseudo_file=s)
+
+        self.assertEqual(conf.cloud, 'amazon')
+        self.assertEqual(conf.storage_bucket, 'awssb')
+        self.assertEqual(conf.storage_prefix, 'aws_storage_path')
+
+
+    def mocked_open_aws_default_credentials(klass, file, asdf):
+        """ mocked open for ~/.aws/credentials """
+        if file == os.path.expanduser('~/.aws/credentials'):
+            creds0 = ("[default]",
+                      "aws_access_key_id = default_access_id",
+                      "aws_secret_access_key = default_sekret")
+            return io.BytesIO('\n'.join(creds0))
+
+        creds1 = ("[default]",
+                  "aws_access_key_id = override_access_id",
+                  "aws_secret_access_key = override_sekret")
+
+        return io.BytesIO('\n'.join(creds1))
+
+    def aws_credentials_default_test(self):
+        """ in absence of [AWS] -> CredentialPath, aws creds should be read from ~/.aws/credentials """
+
+        default_path = os.path.expanduser('~/.aws/credentials')
+        c = ("[Global]",
+             "LogLevel =",
+             "DataDir = ",
+             "CacheDir = ",
+             "StateDir = ",
+             "Cloud = amazon",
+             "[AWS]",
+             "StoragePath = awssb/aws_storage_path",
+             "CredentialPath =",
+           )
+        s = StringIO.StringIO('\n'.join(c))
+
+        patcher = mock.patch('__builtin__.open', side_effect=self.mocked_open_aws_default_credentials)
+        patcher.start()
+        conf = Config(s, conf_pseudo_file=s)
+        aws_creds = conf.get_aws_credentials()
+        patcher.stop()
+
+        self.assertEqual(aws_creds[0], 'default_access_id')
+        self.assertEqual(aws_creds[1], 'default_sekret')
+
+
+    def mocked_isfile(file):
+        """ tell os some_credential_path exists """
+        return file == 'some_credential_path'
+
+    @mock.patch('dvc.config.os.path.isfile', side_effect = mocked_isfile)
+    def aws_credentials_specified_test(self, isfile_function):
+        """ in presence of [AWS] -> CredentialPath, use those credentials """
+
+        c = ("[Global]",
+             "LogLevel =",
+             "DataDir = ",
+             "CacheDir = ",
+             "StateDir = ",
+             "Cloud = amazon",
+             "[AWS]",
+             "StoragePath = awssb/aws_storage_path",
+             "CredentialPath = some_credential_path",
+           )
+        s = StringIO.StringIO('\n'.join(c))
+
+        patcher = mock.patch('__builtin__.open', side_effect=self.mocked_open_aws_default_credentials)
+        patcher.start()
+        conf = Config(s, conf_pseudo_file=s)
+        aws_creds = conf.get_aws_credentials()
+        patcher.stop()
+
+        self.assertEqual(aws_creds[0], 'override_access_id')
+        self.assertEqual(aws_creds[1], 'override_sekret')