From 96f9c50de8b6959e4524561dd1d7fa35c12333b3 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 10 Jan 2018 14:24:41 +0200 Subject: [PATCH] cloud: fixup for #404 Signed-off-by: Ruslan Kuprieiev --- dvc/cloud/__init__.py | 0 dvc/cloud/credentials_aws.py | 91 ++++++++++++++++++++++++++++++++ dvc/cloud/instance_manager.py | 4 ++ dvc/data_cloud.py | 97 ++--------------------------------- tests/test_data_cloud.py | 2 +- 5 files changed, 99 insertions(+), 95 deletions(-) create mode 100644 dvc/cloud/__init__.py create mode 100644 dvc/cloud/credentials_aws.py create mode 100644 dvc/cloud/instance_manager.py diff --git a/dvc/cloud/__init__.py b/dvc/cloud/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dvc/cloud/credentials_aws.py b/dvc/cloud/credentials_aws.py new file mode 100644 index 000000000..e60aaae9a --- /dev/null +++ b/dvc/cloud/credentials_aws.py @@ -0,0 +1,91 @@ +import os +import configparser + +from dvc.utils import cached_property +from dvc.logger import Logger + + +class AWSCredentials(object): + def __init__(self, cloud_config): + self._conf_credpath = cloud_config.get('CredentialPath', None) + self._conf_credsect = cloud_config.get('CredentialSection', 'default') + + @property + def access_key_id(self): + if self.creds: + return self.creds[0] + return None + + @property + def secret_access_key(self): + if self.creds: + return self.creds[1] + return None + + @cached_property + def creds(self): + return self._get_credentials() + + def _get_credentials(self): + """ gets aws credentials, looking in various places + + Params: + + Searches: + 1 any override in dvc.conf [AWS] CredentialPath; + 2 ~/.aws/credentials + + + Returns: + if successfully found, (access_key_id, secret) + None otherwise + """ + + # FIX: It won't work in Windows. + default_path = os.path.expanduser('~/.aws/credentials') + default_sect = 'default' + default_cred_location = (default_path, default_sect) + + cred_locations = self._credential_paths(default_cred_location) + for cred_location in cred_locations: + try: + path = cred_location[0] + section = cred_location[1] + + cc = configparser.SafeConfigParser() + + # use readfp(open( ... to aid mocking. + cc.readfp(open(path, 'r')) + + if section in cc.keys(): + access_key = cc[section].get('aws_access_key_id', None) + secret = cc[section].get('aws_secret_access_key', None) + + if access_key is not None and secret is not None: + return (access_key, secret) + else: + Logger.warn('Unable to find section {} in AWS credential file {}'.format(section, path)) + except Exception as e: + pass + + return None + + def _credential_paths(self, default_cred_location): + results = [] + if self._conf_credpath is not None and len(self._conf_credpath) > 0: + credpath = os.path.expanduser(self._conf_credpath) + if os.path.isfile(credpath): + results.append((credpath, self._conf_credsect)) + else: + msg = 'AWS CredentialPath {} not found; falling back to default file {} and section {}' + Logger.warn(msg.format(credpath, default_cred_location[0], default_cred_location[1])) + results.append(default_cred_location) + else: + results.append(default_cred_location) + return results + + def sanity_check(self): + creds = self._get_credentials() + if creds is None: + Logger.info("can't find aws credetials, assuming envirment variables or iam role") + # self._aws_creds = creds diff --git a/dvc/cloud/instance_manager.py b/dvc/cloud/instance_manager.py new file mode 100644 index 000000000..861693ae4 --- /dev/null +++ b/dvc/cloud/instance_manager.py @@ -0,0 +1,4 @@ +class CloudSettings(object): + def __init__(self, global_storage_path, cloud_config): + self.cloud_config = cloud_config + self.global_storage_path = global_storage_path diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index aa966ad58..42b10bf65 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -29,13 +29,15 @@ except ImportError: from urllib.parse import urlparse import dvc +from dvc.cloud.instance_manager import CloudSettings from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.config import ConfigError from dvc.progress import progress from dvc.utils import copyfile +from dvc.cloud.credentials_aws import AWSCredentials from dvc.system import System -from dvc.utils import map_progress, cached_property +from dvc.utils import map_progress STATUS_UNKNOWN = 0 STATUS_OK = 1 @@ -51,99 +53,6 @@ STATUS_MAP = { (False, True, None) : STATUS_DELETED, } - -class CloudSettings(object): - def __init__(self, global_storage_path, cloud_config): - self.cloud_config = cloud_config - self.global_storage_path = global_storage_path - - -class AWSCredentials(object): - def __init__(self, cloud_config): - self._conf_credpath = cloud_config.get('CredentialPath', None) - self._conf_credsect = cloud_config.get('CredentialSection', 'default') - - @property - def access_key_id(self): - if self.creds: - return self.creds[0] - return None - - @property - def secret_access_key(self): - if self.creds: - return self.creds[1] - return None - - @cached_property - def creds(self): - return self._get_credentials() - - def _get_credentials(self): - """ gets aws credentials, looking in various places - - Params: - - Searches: - 1 any override in dvc.conf [AWS] CredentialPath; - 2 ~/.aws/credentials - - - Returns: - if successfully found, (access_key_id, secret) - None otherwise - """ - - # FIX: It won't work in Windows. - default_path = os.path.expanduser('~/.aws/credentials') - default_sect = 'default' - default_cred_location = (default_path, default_sect) - - cred_locations = self._credential_paths(default_cred_location) - for cred_location in cred_locations: - try: - path = cred_location[0] - section = cred_location[1] - - cc = configparser.SafeConfigParser() - - # use readfp(open( ... to aid mocking. - cc.readfp(open(path, 'r')) - - if section in cc.keys(): - access_key = cc[section].get('aws_access_key_id', None) - secret = cc[section].get('aws_secret_access_key', None) - - if access_key is not None and secret is not None: - return (access_key, secret) - else: - Logger.warn('Unable to find section {} in AWS credential file {}'.format(section, path)) - except Exception as e: - pass - - return None - - def _credential_paths(self, default_cred_location): - results = [] - if self._conf_credpath is not None and len(self._conf_credpath) > 0: - credpath = os.path.expanduser(self._conf_credpath) - if os.path.isfile(credpath): - results.append((credpath, self._conf_credsect)) - else: - msg = 'AWS CredentialPath {} not found; falling back to default file {} and section {}' - Logger.warn(msg.format(credpath, default_cred_location[0], default_cred_location[1])) - results.append(default_cred_location) - else: - results.append(default_cred_location) - return results - - def sanity_check(self): - creds = self._get_credentials() - if creds is None: - Logger.info("can't find aws credetials, assuming envirment variables or iam role") - # self._aws_creds = creds - - class DataCloudError(DvcException): """ Data Cloud exception """ def __init__(self, msg): diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index cf04d6826..4cd41ddc2 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -4,7 +4,7 @@ import uuid from dvc.config import ConfigError from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL from dvc.data_cloud import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED -from dvc.data_cloud import CloudSettings +from dvc.cloud.instance_manager import CloudSettings from tests.basic_env import TestDvc