From ee0e012f86eecd5b2d22d51955ea2baaf187ea62 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Wed, 23 Aug 2017 00:38:28 -0700 Subject: [PATCH] Config redesign --- dvc/cloud/credentials_aws.py | 6 ++-- dvc/cloud/instance_aws.py | 6 ++-- dvc/cloud/instance_manager.py | 11 ++++-- dvc/command/init.py | 10 ++++-- dvc/config.py | 2 +- dvc/data_cloud.py | 66 +++++++++++++++++++---------------- 6 files changed, 59 insertions(+), 42 deletions(-) diff --git a/dvc/cloud/credentials_aws.py b/dvc/cloud/credentials_aws.py index c33c00bb1..e60aaae9a 100644 --- a/dvc/cloud/credentials_aws.py +++ b/dvc/cloud/credentials_aws.py @@ -6,9 +6,9 @@ from dvc.logger import Logger class AWSCredentials(object): - def __init__(self, conf_credpath, conf_credsect): - self._conf_credpath = conf_credpath - self._conf_credsect = conf_credsect + def __init__(self, cloud_config): + self._conf_credpath = cloud_config.get('CredentialPath', None) + self._conf_credsect = cloud_config.get('CredentialSection', 'default') @property def access_key_id(self): diff --git a/dvc/cloud/instance_aws.py b/dvc/cloud/instance_aws.py index 9196dd50c..08b661a5f 100644 --- a/dvc/cloud/instance_aws.py +++ b/dvc/cloud/instance_aws.py @@ -1,5 +1,6 @@ import sys, os, time, boto.ec2 +from dvc.cloud.credentials_aws import AWSCredentials from dvc.cloud.instance import Instance, InstanceError from dvc.logger import Logger @@ -11,11 +12,12 @@ class InstanceAws(Instance): def __init__(self, homedir, conf_file): super(InstanceAws).__init__(homedir, conf_file) + self._aws_creds = AWSCredentials(self._cloud_config) self._conn = boto.ec2.connect_to_region( self._region, - aws_access_key_id = self._aws_conf.get("AccessKeyID"), - aws_secret_access_key = self._aws_conf.get("SecretAccessKey")) + aws_access_key_id=self._aws_conf.get("AccessKeyID"), + aws_secret_access_key=self._aws_conf.get("SecretAccessKey")) def create(self): instance = None diff --git a/dvc/cloud/instance_manager.py b/dvc/cloud/instance_manager.py index 440bac5d9..48467b4cf 100644 --- a/dvc/cloud/instance_manager.py +++ b/dvc/cloud/instance_manager.py @@ -1,7 +1,14 @@ - +from dvc.cloud.instance_aws import InstanceAws from dvc.logger import Logger +class CloudSettings(object): + def __init__(self, path_factory, global_storage_path, cloud_config): + self.path_factory = path_factory + self.cloud_config = cloud_config + self.global_storage_path = global_storage_path + + class InstanceManager(object): def instances(self): return [] @@ -11,5 +18,5 @@ class InstanceManager(object): Logger.error('Instance with name {} is already exist'.format(name)) return 1 - + inst = InstanceAws() pass diff --git a/dvc/command/init.py b/dvc/command/init.py index f864d030e..4979de0ce 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -23,7 +23,10 @@ TargetFile = {} # Supported clouds: AWS, GCP Cloud = AWS -StoragePath = + +# This global storage path is going to be deprecated in the next version. +# Please use StoragePath from a specific cloud instead. +#StoragePath = # Log levels: Debug, Info, Warning and Error LogLevel = Info @@ -32,6 +35,8 @@ LogLevel = Info CredentialPath = ~/.aws/credentials CredentialSection = default +StoragePath = dvc/tutorial + # Default settings for AWS instances: Type = t2.nano Image = ami-2d39803a @@ -47,8 +52,7 @@ Region = us-east-1 Zone = us-east-1a SubnetId = -Storage = my-100gb-drive-io -#StoragePath = dvc/tutorial +Volume = my-100gb-drive-io Monitoring = false EbsOptimized = false diff --git a/dvc/config.py b/dvc/config.py index a008dbae2..198eb4d71 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -23,7 +23,7 @@ class ConfigI(object): self._cloud = None self.set(data_dir, cache_dir, state_dir, target_file, cloud, conf_parser) - def set(self, data_dir, cache_dir, state_dir, target_file, cloud, conf_parser): + def set(self, data_dir, cache_dir, state_dir, target_file, cloud=None, conf_parser=None): self._data_dir = data_dir self._cache_dir = cache_dir self._state_dir = state_dir diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 1481530cb..f290e36cb 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -9,6 +9,7 @@ from boto.s3.connection import S3Connection from google.cloud import storage as gc import dvc +from dvc.cloud.instance_manager import CloudSettings from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.config import ConfigError @@ -53,10 +54,8 @@ def file_md5(fname): class DataCloudBase(object): """ Base class for DataCloud """ - def __init__(self, settings, config, cloud_config): - self._settings = settings - self._config = config - self._cloud_config = cloud_config + def __init__(self, cloud_settings): + self._cloud_settings = cloud_settings self._lock = threading.Lock() @property @@ -66,11 +65,10 @@ class DataCloudBase(object): Precedence: Storage, then cloud specific """ - path = self._config['Global'].get('StoragePath', None) - if path: - return path + if self._cloud_settings.global_storage_path: + return self._cloud_settings.global_storage_path - path = self._cloud_config.get('StoragePath', None) + path = self._cloud_settings.cloud_config.get('StoragePath', None) if path is None: raise ConfigError('invalid StoragePath: not set for Data or cloud specific') @@ -104,7 +102,7 @@ class DataCloudBase(object): pass def sync(self, fname): - item = self._settings.path_factory.data_item(fname) + item = self._cloud_settings.path_factory.data_item(fname) if os.path.isfile(item.resolved_cache.dvc): self.sync_to_cloud(item) else: @@ -147,15 +145,9 @@ class DataCloudLOCAL(DataCloudBase): class DataCloudAWS(DataCloudBase): """ DataCloud class for Amazon Web Services """ - def __init__(self, settings, config, cloud_config): - super(DataCloudAWS, self).__init__(settings, config, cloud_config) - - credpath = self._cloud_config.get('CredentialPath', None) - credsect = self._cloud_config.get('CredentialSection', 'default') - - aws_creds = AWSCredentials(credpath, credsect) - self.aws_access_key_id = aws_creds.access_key_id - self.aws_secret_access_key = aws_creds.secret_access_key + def __init__(self, cloud_settings): # settings, config, cloud_config): + super(DataCloudAWS, self).__init__(cloud_settings) + self._aws_creds = AWSCredentials(cloud_settings.cloud_config) @property def aws_region_host(self): @@ -164,7 +156,7 @@ class DataCloudAWS(DataCloudBase): See notes http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region """ - region = self._cloud_config['Region'] + region = self._cloud_settings.cloud_config['Region'] if region is None or region == '': return 's3.amazonaws.com' if region == 'us-east-1': @@ -173,14 +165,16 @@ class DataCloudAWS(DataCloudBase): def _get_bucket_aws(self): """ get a bucket object, aws """ - if all([self.aws_access_key_id, self.aws_secret_access_key, self.aws_region_host]): - conn = S3Connection(self.aws_access_key_id, self.aws_secret_access_key, host=self.aws_region_host) + if all([self._aws_creds.access_key_id, self._aws_creds.secret_access_key, self.aws_region_host]): + conn = S3Connection(self._aws_creds.access_key_id, + self._aws_creds.secret_access_key, + host=self.aws_region_host) else: conn = S3Connection() bucket_name = self.storage_bucket bucket = conn.lookup(bucket_name) if bucket is None: - raise DataCloudError('Storage path is not setup correctly') + raise DataCloudError('Storage path {} is not setup correctly'.format(bucket_name)) return bucket def sync_from_cloud(self, item): @@ -244,10 +238,10 @@ class DataCloudAWS(DataCloudBase): Logger.debug(u'[Cmd-Remove] Remove from cloud {}.'.format(aws_file_name)) - if not self.aws_access_key_id or not self.aws_secret_access_key: + if not self._aws_creds.access_key_id or not self._aws_creds.secret_access_key: Logger.debug('[Cmd-Remove] Unable to check cache file in the cloud') return - conn = S3Connection(self.aws_access_key_id, self.aws_secret_access_key) + conn = S3Connection(self._aws_creds.access_key_id, self._aws_creds.secret_access_key) bucket_name = self.storage_bucket bucket = conn.lookup(bucket_name) if bucket: @@ -332,23 +326,33 @@ class DataCloud(object): #To handle ConfigI case if not hasattr(settings.config, '_config'): - self._cloud = DataCloudBase(None, None, None) + self._cloud = DataCloudBase(None) return self._settings = settings self._config = self._settings.config._config - self.typ = self._config['Global'].get('Cloud', '').strip().upper() - if self.typ not in self.CLOUD_MAP.keys(): - raise ConfigError('Wrong cloud type %s specified' % self.typ) + cloud_type = self._config['Global'].get('Cloud', '').strip().upper() + if cloud_type not in self.CLOUD_MAP.keys(): + raise ConfigError('Wrong cloud type %s specified' % cloud_type) - if self.typ not in self._config.keys(): - raise ConfigError('Can\'t find cloud section \'[%s]\' in config' % self.typ) + if cloud_type not in self._config.keys(): + raise ConfigError('Can\'t find cloud section \'[%s]\' in config' % cloud_type) - self._cloud = self.CLOUD_MAP[self.typ](self._settings, self._config, self._config[self.typ]) + cloud_settings = self.get_cloud_settings(self._config, cloud_type, self._settings.path_factory) + + self.typ = cloud_type + self._cloud = self.CLOUD_MAP[cloud_type](cloud_settings) self.sanity_check() + @staticmethod + def get_cloud_settings(config, cloud_type, path_factory): + cloud_config = config[cloud_type] + global_storage_path = config['Global'].get('StoragePath', None) + cloud_settings = CloudSettings(path_factory, global_storage_path, cloud_config) + return cloud_settings + def sanity_check(self): """ sanity check a config