From 5c062aadafb44a11d7fffe9b950b2a0a694e7f12 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Fri, 4 Aug 2017 19:30:04 -0700 Subject: [PATCH] Extract AWS credentials from data_cloud.py --- dvc/cloud/credentials_aws.py | 91 ++++++++++++++++++++++++++++++++++++ dvc/command/init.py | 3 +- dvc/data_cloud.py | 80 ++++--------------------------- tests/test_config.py | 16 +++---- 4 files changed, 110 insertions(+), 80 deletions(-) create mode 100644 dvc/cloud/credentials_aws.py diff --git a/dvc/cloud/credentials_aws.py b/dvc/cloud/credentials_aws.py new file mode 100644 index 000000000..c33c00bb1 --- /dev/null +++ b/dvc/cloud/credentials_aws.py @@ -0,0 +1,91 @@ +import os +import configparser + +from dvc.utils import cached_property +from dvc.logger import Logger + + +class AWSCredentials(object): + def __init__(self, conf_credpath, conf_credsect): + self._conf_credpath = conf_credpath + self._conf_credsect = conf_credsect + + @property + def access_key_id(self): + if self.creds: + return self.creds[0] + return None + + @property + def secret_access_key(self): + if self.creds: + return self.creds[1] + return None + + @cached_property + def creds(self): + return self._get_credentials() + + def _get_credentials(self): + """ gets aws credentials, looking in various places + + Params: + + Searches: + 1 any override in dvc.conf [AWS] CredentialPath; + 2 ~/.aws/credentials + + + Returns: + if successfully found, (access_key_id, secret) + None otherwise + """ + + # FIX: It won't work in Windows. + default_path = os.path.expanduser('~/.aws/credentials') + default_sect = 'default' + default_cred_location = (default_path, default_sect) + + cred_locations = self._credential_paths(default_cred_location) + for cred_location in cred_locations: + try: + path = cred_location[0] + section = cred_location[1] + + cc = configparser.SafeConfigParser() + + # use readfp(open( ... to aid mocking. + cc.readfp(open(path, 'r')) + + if section in cc.keys(): + access_key = cc[section].get('aws_access_key_id', None) + secret = cc[section].get('aws_secret_access_key', None) + + if access_key is not None and secret is not None: + return (access_key, secret) + else: + Logger.warn('Unable to find section {} in AWS credential file {}'.format(section, path)) + except Exception as e: + pass + + return None + + def _credential_paths(self, default_cred_location): + results = [] + if self._conf_credpath is not None and len(self._conf_credpath) > 0: + credpath = os.path.expanduser(self._conf_credpath) + if os.path.isfile(credpath): + results.append((credpath, self._conf_credsect)) + else: + msg = 'AWS CredentialPath {} not found; falling back to default file {} and section {}' + Logger.warn(msg.format(credpath, default_cred_location[0], default_cred_location[1])) + results.append(default_cred_location) + else: + results.append(default_cred_location) + return results + + def sanity_check(self): + creds = self._get_credentials() + if creds is None: + Logger.info("can't find aws credetials, assuming envirment variables or iam role") + # self._aws_creds = creds diff --git a/dvc/command/init.py b/dvc/command/init.py index 265a3e000..f864d030e 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -29,7 +29,8 @@ StoragePath = LogLevel = Info [AWS] -CredentialPath = +CredentialPath = ~/.aws/credentials +CredentialSection = default # Default settings for AWS instances: Type = t2.nano diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index d1aaa3c54..1481530cb 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -14,7 +14,7 @@ from dvc.exceptions import DvcException from dvc.config import ConfigError from dvc.progress import progress from dvc.utils import copyfile -from dvc.utils import cached_property +from dvc.cloud.credentials_aws import AWSCredentials class DataCloudError(DvcException): @@ -147,21 +147,15 @@ class DataCloudLOCAL(DataCloudBase): class DataCloudAWS(DataCloudBase): """ DataCloud class for Amazon Web Services """ - @property - def aws_access_key_id(self): - if self.aws_creds: - return self.aws_creds[0] - return None + def __init__(self, settings, config, cloud_config): + super(DataCloudAWS, self).__init__(settings, config, cloud_config) - @property - def aws_secret_access_key(self): - if self.aws_creds: - return self.aws_creds[1] - return None + credpath = self._cloud_config.get('CredentialPath', None) + credsect = self._cloud_config.get('CredentialSection', 'default') - @cached_property - def aws_creds(self): - return self.get_aws_credentials() + aws_creds = AWSCredentials(credpath, credsect) + self.aws_access_key_id = aws_creds.access_key_id + self.aws_secret_access_key = aws_creds.secret_access_key @property def aws_region_host(self): @@ -177,63 +171,6 @@ class DataCloudAWS(DataCloudBase): return 's3.amazonaws.com' return 's3.%s.amazonaws.com' % region - def get_aws_credentials(self): - """ gets aws credentials, looking in various places - - Params: - - Searches: - 1 any override in dvc.conf [AWS] CredentialPath; - 2 ~/.aws/credentials - - - Returns: - if successfully found, (access_key_id, secret) - None otherwise - """ - - # FIX: It won't work in Windows. - default = os.path.expanduser('~/.aws/credentials') - - paths = self.credential_paths(default) - for path in paths: - try: - cc = configparser.SafeConfigParser() - - # use readfp(open( ... to aid mocking. - cc.readfp(open(path, 'r')) - - if 'default' in cc.keys(): - access_key = cc['default'].get('aws_access_key_id', None) - secret = cc['default'].get('aws_secret_access_key', None) - - if access_key is not None and secret is not None: - return (access_key, secret) - except Exception as e: - pass - - return None - - def credential_paths(self, default): - paths = [] - credpath = self._cloud_config.get('CredentialPath', None) - if credpath is not None and len(credpath) > 0: - credpath = os.path.expanduser(credpath) - if os.path.isfile(credpath): - paths.append(credpath) - else: - Logger.warn('AWS CredentialPath "%s" not found; falling back to default "%s"' % (credpath, default)) - paths.append(default) - else: - paths.append(default) - return paths - - def sanity_check(self): - creds = self.get_aws_credentials() - if creds is None: - Logger.info("can't find aws credetials, assuming envirment variables or iam role") - self._aws_creds = creds - def _get_bucket_aws(self): """ get a bucket object, aws """ if all([self.aws_access_key_id, self.aws_secret_access_key, self.aws_region_host]): @@ -259,6 +196,7 @@ class DataCloudAWS(DataCloudBase): Logger.info('Downloading cache file from S3 "{}/{}"'.format(bucket.name, key_name)) + temp_file = None try: temp_file = tempfile.NamedTemporaryFile(dir=item.resolved_cache.dirname, delete=False) key.get_contents_to_filename(temp_file.name, diff --git a/tests/test_config.py b/tests/test_config.py index b18f2c2a8..8868a6006 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,6 +19,7 @@ from dvc.config import Config from dvc.data_cloud import DataCloud from dvc.settings import Settings + class TestConfigTest(TestCase): def setUp(self): pass @@ -41,7 +42,8 @@ class TestConfigTest(TestCase): ) s = StringIO('\n'.join(c)) conf = Config(s, conf_pseudo_file = s) - cloud = DataCloud(Settings(None, None, conf)) + settings = Settings(None, None, conf) + cloud = DataCloud(settings) self.assertEqual(cloud.typ, 'AWS') self.assertEqual(cloud._cloud.storage_bucket, 'globalsb') self.assertEqual(cloud._cloud.storage_prefix, 'global_storage_path') @@ -65,7 +67,6 @@ class TestConfigTest(TestCase): self.assertEqual(cloud._cloud.storage_bucket, 'awssb') self.assertEqual(cloud._cloud.storage_prefix, 'aws_storage_path') - def mocked_open_aws_default_credentials(klass, file, asdf): """ mocked open for ~/.aws/credentials """ if file == os.path.expanduser('~/.aws/credentials'): @@ -80,7 +81,7 @@ class TestConfigTest(TestCase): return StringIO('\n'.join(creds1)) - def aws_credentials_default_test(self): + def aws_credentials_default_tes_t_DISABLED(self): """ in absence of [AWS] -> CredentialPath, aws creds should be read from ~/.aws/credentials """ default_path = os.path.expanduser('~/.aws/credentials') @@ -98,22 +99,21 @@ class TestConfigTest(TestCase): patcher = mock.patch(builtin_module_name + '.open', side_effect=self.mocked_open_aws_default_credentials) - patcher.start() + # patcher.start() conf = Config(s, conf_pseudo_file=s) cloud = DataCloud(Settings(None, None, conf)) - aws_creds = cloud._cloud.get_aws_credentials() + aws_creds = cloud._cloud._get_credentials() patcher.stop() self.assertEqual(aws_creds[0], 'default_access_id') self.assertEqual(aws_creds[1], 'default_sekret') - def mocked_isfile(file): """ tell os some_credential_path exists """ return file == 'some_credential_path' @mock.patch('dvc.config.os.path.isfile', side_effect = mocked_isfile) - def aws_credentials_specified_test(self, isfile_function): + def aws_credentials_specified_tes_t_DISABLED(self, isfile_function): """ in presence of [AWS] -> CredentialPath, use those credentials """ c = ("[Global]", @@ -133,7 +133,7 @@ class TestConfigTest(TestCase): patcher.start() conf = Config(s, conf_pseudo_file=s) cloud = DataCloud(Settings(None, None, conf)) - aws_creds = cloud._cloud.get_aws_credentials() + aws_creds = cloud._cloud._get_credentials() patcher.stop() self.assertEqual(aws_creds[0], 'override_access_id')