Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-06 15:28:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: remove ec2 commands

Browse Source 
Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-01-10 11:37:16 +02:00
parent b752c600f1
commit 2b6d997fd3
11 changed files with 95 additions and 733 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cli.py
-79
View File
@@ -16,7 +16,6 @@ from dvc.command.lock import CmdLock
from dvc.command.gc import CmdGC
from dvc.command.add import CmdAdd
from dvc.command.show_workflow import CmdShowWorkflow
from dvc.command.instance_create import CmdInstanceCreate
from dvc.command.config import CmdConfig
from dvc.command.show_pipeline import CmdShowPipeline
from dvc.command.checkout import CmdCheckout
@@ -222,84 +221,6 @@ def parse_args(argv=None):
help='Collect garbage')
gc_parser.set_defaults(func=CmdGC)
# Cloud
ex_parser = subparsers.add_parser(
'ex',
parents=[parent_parser],
help='Experimental commands')
ex_subparsers = ex_parser.add_subparsers(
dest='cmd',
help='Use dvc cloud CMD --help for command-specific help')
cloud_parser = ex_subparsers.add_parser(
'cloud',
parents=[parent_parser],
help='Cloud manipulation')
cloud_subparsers = cloud_parser.add_subparsers(
dest='cmd',
help='Use dvc cloud CMD --help for command-specific help')
# Instance create
instance_create_parser = cloud_subparsers.add_parser(
'create',
help='Create cloud instance')
instance_create_parser.add_argument('name',
# metavar='',
nargs='?',
help='Instance name.')
instance_create_parser.add_argument('-c',
'--cloud',
# metavar='',
help='Cloud: AWS, GCP.')
instance_create_parser.add_argument('-t',
'--type',
# metavar='',
help='Instance type.')
instance_create_parser.add_argument('-i',
'--image',
# metavar='',
help='Instance image.')
instance_create_parser.add_argument('--spot-price',
metavar='PRICE',
help='Spot instance price in $ i.e. 1.54')
instance_create_parser.add_argument('--spot-timeout',
metavar='TIMEOUT',
type=int,
help='Spot instances waiting timeout in seconds.')
instance_create_parser.add_argument('--keypair-name',
metavar='KEYPAIR',
help='The name of key pair for instance launch')
instance_create_parser.add_argument('--keypair-dir',
metavar='DIR',
help='The directory of key pairs')
instance_create_parser.add_argument('--security-group',
metavar='GROUP',
help='Security group')
instance_create_parser.add_argument('--region',
help='Region')
instance_create_parser.add_argument('--zone',
help='Zone')
instance_create_parser.add_argument('--subnet-id',
metavar='SUBNET',
help='Subnet ID')
instance_create_parser.add_argument('--storage',
# metavar='',
help='The name of attachable storage volume.')
# WHERE EBS IS?
instance_create_parser.add_argument('--monitoring',
action='store_true',
default=False,
help='Enable EC2 instance monitoring')
instance_create_parser.add_argument('--ebs-optimized',
action='store_true',
default=False,
help='Enable EBS I\O optimization')
instance_create_parser.add_argument('--disks-to-ride0',
action='store_true',
default=False,
help='Detect all ephemeral disks and stripe together in raid-0')
instance_create_parser.set_defaults(func=CmdInstanceCreate)
# Config
config_parser = subparsers.add_parser(
'config',
dvc/cloud/__init__.py
View File
dvc/cloud/base_aws.py
-43
View File
@@ -1,43 +0,0 @@
import sys, os, time, configparser, argparse, boto.ec2
class AwsBase:
DEFAULT_REGION = "us-east-1"
DEFAULT_ZONE = "us-east-1a"
def __init__(self, homedir, conf_file):
self._homedir = homedir
config = configparser.ConfigParser()
config.read(conf_file)
self._aws_conf = config['AWS']
self._args = self.parse_args()
self._conn = boto.ec2.connect_to_region(
self._args.region,
aws_access_key_id = self._aws_conf.get("AccessKeyID"),
aws_secret_access_key = self._aws_conf.get("SecretAccessKey"))
pass
def get_command(self):
return (self._args.command, self._args.param)
def parse_args(self):
parser = argparse.ArgumentParser()
parser.add_argument('command')
parser.add_argument('param', nargs = '?', default = '')
parser.add_argument('--region', metavar = '',
default = self._aws_conf.get("Region", self.DEFAULT_REGION),
help = 'AWS region: us-east-2, ...')
parser.add_argument('--zone', metavar = '',
default = self._aws_conf.get("Zone", self.DEFAULT_ZONE),
help = 'Availability Zone. Examples: us-east-1a, us-east-1b, ...')
self.extend_args(parser)
args = parser.parse_args()
return args
def extend_args(self, parser):
raise NotImplementedError()
dvc/cloud/credentials_aws.py
-91
View File
@@ -1,91 +0,0 @@
import os
import configparser
from dvc.utils import cached_property
from dvc.logger import Logger
class AWSCredentials(object):
def __init__(self, cloud_config):
self._conf_credpath = cloud_config.get('CredentialPath', None)
self._conf_credsect = cloud_config.get('CredentialSection', 'default')
@property
def access_key_id(self):
if self.creds:
return self.creds[0]
return None
@property
def secret_access_key(self):
if self.creds:
return self.creds[1]
return None
@cached_property
def creds(self):
return self._get_credentials()
def _get_credentials(self):
""" gets aws credentials, looking in various places
Params:
Searches:
1 any override in dvc.conf [AWS] CredentialPath;
2 ~/.aws/credentials
Returns:
if successfully found, (access_key_id, secret)
None otherwise
"""
# FIX: It won't work in Windows.
default_path = os.path.expanduser('~/.aws/credentials')
default_sect = 'default'
default_cred_location = (default_path, default_sect)
cred_locations = self._credential_paths(default_cred_location)
for cred_location in cred_locations:
try:
path = cred_location[0]
section = cred_location[1]
cc = configparser.SafeConfigParser()
# use readfp(open( ... to aid mocking.
cc.readfp(open(path, 'r'))
if section in cc.keys():
access_key = cc[section].get('aws_access_key_id', None)
secret = cc[section].get('aws_secret_access_key', None)
if access_key is not None and secret is not None:
return (access_key, secret)
else:
Logger.warn('Unable to find section {} in AWS credential file {}'.format(section, path))
except Exception as e:
pass
return None
def _credential_paths(self, default_cred_location):
results = []
if self._conf_credpath is not None and len(self._conf_credpath) > 0:
credpath = os.path.expanduser(self._conf_credpath)
if os.path.isfile(credpath):
results.append((credpath, self._conf_credsect))
else:
msg = 'AWS CredentialPath {} not found; falling back to default file {} and section {}'
Logger.warn(msg.format(credpath, default_cred_location[0], default_cred_location[1]))
results.append(default_cred_location)
else:
results.append(default_cred_location)
return results
def sanity_check(self):
creds = self._get_credentials()
if creds is None:
Logger.info("can't find aws credetials, assuming envirment variables or iam role")
# self._aws_creds = creds
dvc/cloud/instance.py
-41
View File
@@ -1,41 +0,0 @@
from dvc.exceptions import DvcException
class InstanceError(DvcException):
def __init__(self, msg):
super(InstanceError).__init__(u"Instance error: " + msg)
class Instance(object):
INSTANCE_STATE_TAG = 'dvc-is-active'
VOLUME_TAG = 'dvc-volume'
def __init__(self, name, cloud, parsed_args, conf_parser):
self._name = name
self._cloud = cloud
self._type = parsed_args.type or conf_parser._config[cloud]['Type']
self._image = parsed_args.image or conf_parser._config[cloud]['Image']
self._spot_price = parsed_args.spot_price or conf_parser._config[cloud]['SpotPrice']
self._spot_timeout = parsed_args.spot_timeout or conf_parser._config[cloud]['SpotTimeout']
self._keypair_name = parsed_args.keypair_name or conf_parser._config[cloud]['KeyPairName']
self._keypair_dir = parsed_args.keypair_dir or conf_parser._config[cloud]['KeyPairDir']
self._security_group = parsed_args.security_group or conf_parser._config[cloud]['SecurityGroup']
self._region = parsed_args.region or conf_parser._config[cloud]['Region']
self._zone = parsed_args.zone or conf_parser._config[cloud]['Zone']
self._subnet_id = parsed_args.subnet_id or conf_parser._config[cloud]['SubnetId']
self._storage = parsed_args.storage or conf_parser._config[cloud]['Storage']
self._monitoring = parsed_args.monitoring or conf_parser._config[cloud]['Monitoring']
self._ebs_optimized = parsed_args.ebs_optimized or conf_parser._config[cloud]['EbsOptimized']
self._disks_to_raid0 = parsed_args.disks_to_ride0 or conf_parser._config[cloud]['DisksToRAID0']
pass
def toBool(self, value):
if value == None:
return False
return value.lower() in ("yes", "true", "t", "1")
dvc/cloud/instance_aws.py
-371
View File
@@ -1,371 +0,0 @@
import sys, os, time, boto.ec2
from dvc.cloud.credentials_aws import AWSCredentials
from dvc.cloud.instance import Instance, InstanceError
from dvc.logger import Logger
class InstanceAws(Instance):
TERMINATED_STATE = 'terminated'
INSTANCE_FORMAT = "{:8s}{:12s}{:12s}{:12s}{:16s}{:16s}{:16s}"
VOLUME_FORMAT = "{:20s}{:16s}{:12s}{:12s}{:12s}{:12s}{:16s}"
def __init__(self, homedir, conf_file):
super(InstanceAws).__init__(homedir, conf_file)
self._aws_creds = AWSCredentials(self._cloud_config)
self._conn = boto.ec2.connect_to_region(
self._region,
aws_access_key_id=self._aws_conf.get("AccessKeyID"),
aws_secret_access_key=self._aws_conf.get("SecretAccessKey"))
def create(self):
instance = None
if not self._spot_price:
reserv = self._conn.run_instances(self._image,
key_name=self.get_key_name(),
instance_type=self._type,
security_groups=self.get_security_groups(),
monitoring_enabled=self.toBool(self._monitoring),
#subnet_id = self._args.subnet_id,
placement=self._zone,
ebs_optimized=self.toBool(self._ebs_optimized))
instance = reserv.instances[0]
else:
instance = self.create_spot_instance()
return instance
def find_volume(self):
(volumes, rest_volumes) = self.all_volumes()
for volume in volumes:
if volume.tags[self.VOLUME_TAG] == self._storage:
return volume
msg = u'Cannot find storage volume {}. Verify that the volume was created.'.format(
self._storage)
raise InstanceError(msg)
def create_spot_instance(self):
# Create spot instance
req = self._conn.request_spot_instances(price = self._args.spot_price,
image_id = self._args.image,
key_name = self.get_key_name(),
instance_type = self._args.instance_type,
security_groups = self.get_security_groups(),
monitoring_enabled = self.toBool(self._args.monitoring),
#subnet_id = self._args.subnet_id,
placement = self._args.zone,
ebs_optimized = self.toBool(self._args.ebs_optimized))
job_instance_id = self._wait_for_an_instance(req)
if not job_instance_id:
self._conn.cancel_spot_instance_requests(req[0].id)
raise InstanceError(u'Unable to obtain {} spot instance in region {} for price ${}: {}'.format(
(self._args.instance_type, self._args.region, self._args.spot_price,
'the request was canceled')))
Logger.info(u'{} spot instance was created: {}'.format(self._type, job_instance_id))
reservations = self._conn.get_all_instances(instance_ids = job_instance_id)
instance = reservations[0].instances[0]
return instance
def _wait_for_an_instance(self, req):
job_instance_id = None
sec = 0
Logger.info(u'Waiting for a spot instance. Request {}.'.format(req[0].id))
while job_instance_id == None and sec < self._args.spot_timeout:
sys.stdout.write(".")
sys.stdout.flush()
job_sir_id = req[0].id
reqs = self._conn.get_all_spot_instance_requests()
for sir in reqs:
if sir.id == job_sir_id:
job_instance_id = sir.instance_id
break
time.sleep(1)
sec += 1
sys.stdout.write("\n")
return job_instance_id
def run_instance(self):
if not self._image:
raise InstanceError('Cannot run EC2 instance: image (AMI) is not defined')
# Remove active tag.
active_filter = {'tag-key': self.INSTANCE_STATE_TAG, 'tag-value': 'True'}
active_reserv = self._conn.get_all_instances(filters = active_filter)
active_instances = [i for r in active_reserv for i in r.instances]
if len(active_instances) > 0:
#active_insts = active_reserv.instances
if len(active_instances) > 1:
Logger.error('EC2 instances consistency error - more than one active EC2 instance')
for inst in active_instances:
inst.remove_tag(self.INSTANCE_STATE_TAG, 'True')
inst.add_tag(self.INSTANCE_STATE_TAG, 'False')
if inst.state != self.TERMINATED_STATE:
Logger.info('{} instance {} is not longer active'.format(inst.instance_type, inst.id))
self._run_instance()
pass
def _run_instance(self):
instance = self.create_instance()
# Assign the created instace as active.
instance.add_tag(self.INSTANCE_STATE_TAG, 'True')
Logger.info('New {} instance {} was selected as active'.format(instance.instance_type, instance.id))
Logger.info('Waiting for a running status')
while instance.state != 'running':
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(1)
instance.update()
sys.stdout.write("\n")
self._conn.attach_volume(self.find_volume().id, instance.id, "/dev/sdx")
def get_security_groups(self, ssh_port = 22):
group_name = self._security_group
# Check if the group exists and create one if does not.
try:
group = self._conn.get_all_security_groups(groupnames=[group_name])[0]
except self._conn.ResponseError as e:
if e.code == 'InvalidGroup.NotFound':
Logger.error('AWS Security Group {} does not exist: creating the group'.format(group_name))
group = self._conn.create_security_group(group_name, 'group with SSH access')
else:
raise
# Enable SSH access.
try:
group.authorize('tcp', ssh_port, ssh_port, '0.0.0.0/0')
except self._conn.ResponseError as e:
if e.code != 'InvalidPermission.Duplicate':
raise
return [group_name]
def get_key_name(self):
if not self._keypair_name:
raise InstanceError('AWS keypair cannot be created: KeyName is not specified in AWS section in the config file')
if not self._keypair_dir:
raise InstanceError('AWS keypair cannot be created: KeyDir is not specified in AWS section in the config file')
# Check if the key exists and create one if does not.
try:
key = self._conn.get_all_key_pairs(keynames=[self._keypair_name])[0]
except self._conn.ResponseError as e:
if e.code != 'InvalidKeyPair.NotFound':
raise
self.create_new_keypair()
return self._keypair_name
def create_new_keypair(self):
Logger.info('AWS key {} does not exist: creating the key'.format(self._keypair_name))
# Create an SSH key to use when logging into instances.
key = self._conn.create_key_pair(self._keypair_name)
Logger.info('AWS key was created: {}'.format(self._keypair_name))
# Expand key dir.
key_dir = os.path.expandvars(self._keypair_dir)
if not os.path.isdir(self._keypair_dir):
os.mkdir(self._keypair_dir, 0o700)
# Private key has to be stored locally.
key_file = os.path.join(key_dir, self._keypair_name + '.pem')
# key.save(key_file) # doesn't work in python3
fp = open(key_file, 'w')
fp.write(key.material)
fp.close()
os.chmod(key_file, 0o600)
Logger.info('AWS private key file was saved: {}'.format(key_file))
def all_instances(self, terminated = False):
# active = []
# not_active = []
# rest_inst = []
reserv = self._conn.get_all_instances()
instances = [i for r in reserv for i in r.instances]
active, not_active, rest_inst = self.split_inatances_by_types(instances, terminated)
if len(active) > 1:
Logger.error(u'Instance consistancy error - more than one instance is in active state')
if len(active) == 0 and len(not_active) > 0:
Logger.error(u'Instance consistancy error - no active instances')
return (active, not_active, rest_inst)
def split_inatances_by_types(self, instances, terminated):
active = []
not_active = []
rest_inst = []
for inst in instances:
if terminated == False and inst.state == self.TERMINATED_STATE:
continue
if self.INSTANCE_STATE_TAG in inst.tags:
if inst.tags[self.INSTANCE_STATE_TAG] == 'True':
active.append(inst)
else:
not_active.append(inst)
else:
rest_inst.append(inst)
return active, not_active, rest_inst
def describe_instance(self, inst, volumes, is_active):
ip = inst.ip_address
ip_private = inst.private_ip_address
if not ip:
ip = ''
if not ip_private:
ip_private = ''
activeFlag = ''
if is_active:
activeFlag = ' ***'
volume_name = self.get_volume_name(inst, volumes)
return self.INSTANCE_FORMAT.format(activeFlag,
inst.id[:12],
inst.instance_type[:12],
inst.state[:12],
volume_name,
ip[:16],
ip_private[:16])
def get_volume_name(self, inst, volumes):
volume_name = ''
bdm = inst.block_device_mapping
for device_type in bdm.values():
for vol in volumes:
if device_type.volume_id == vol.id:
if self.VOLUME_TAG in vol.tags:
volume_name = vol.tags[self.VOLUME_TAG]
else:
Logger.error('Instance {} storage volume does not have tag'.format(inst.id))
break
return volume_name
def describe_instances(self):
(active, not_active, rest_inst) = self.all_instances()
(volumes, rest_volumes) = self.all_volumes()
Logger.info(self.INSTANCE_FORMAT.format('Active', 'Id', 'Type', 'State', 'Storage',
'IP public', 'IP private'))
for inst in active:
Logger.info(self.describe_instance(inst, volumes, True))
for inst in not_active:
logger.info(self.describe_instance(inst, volumes, False))
if len(rest_inst) > 0:
Logger.error(u'{} not tracked instances were found: not in the list'.format(len(rest_inst)))
pass
def get_instance_id(self, instance):
if instance[:2] != 'i-' and len(instance) == 8:
instance = 'i-' + instance
return instance
def terminate_instances(self, instance):
instance = self.get_instance_id(instance)
if not instance:
Logger.error('Instance Id is not specified')
return
(active, not_active, rest_inst) = self.all_instances()
if instance == 'all':
target_in_active = active
target_in_not_active = not_active
else:
target_in_active = list(filter(lambda inst: inst.id == instance, active))
target_in_not_active = list(filter(lambda inst: inst.id == instance, not_active))
self._terminate(target_in_active, target_in_not_active)
if instance != 'all' and len(target_in_active) > 0 and len(not_active) > 0:
new_active_inst = not_active[0]
new_active_inst.remove_tag(self.INSTANCE_STATE_TAG, 'False')
new_active_inst.add_tag(self.INSTANCE_STATE_TAG, 'True')
randomly = ''
if len(not_active) > 1:
randomly = 'randomly '
Logger.error('{} instance {} was {} selected as active because of an active instance was terminated'.format(
new_active_inst.instance_type, new_active_inst.id, randomly))
pass
def _terminate(self, target_in_active, target_in_not_active):
if target_in_not_active:
self._conn.terminate_instances(instance_ids=[inst.id for inst in target_in_not_active])
for inst in target_in_active:
inst.remove_tag(self.INSTANCE_STATE_TAG, 'True')
inst.add_tag(self.INSTANCE_STATE_TAG, 'False')
self._conn.terminate_instances(instance_ids=[inst.id])
def set_active_instance(self, instance):
instance = self.get_instance_id(instance)
(active, not_active, rest_inst) = self.all_instances()
target_in_not_actives = InstanceAws._get_target_in_not_active(instance, active, not_active)
if not target_in_not_actives:
return
target_in_not_actives[0].remove_tag(self.INSTANCE_STATE_TAG, 'False')
target_in_not_actives[0].add_tag(self.INSTANCE_STATE_TAG, 'True')
for inst in active:
inst.remove_tag(self.INSTANCE_STATE_TAG, 'True')
inst.add_tag(self.INSTANCE_STATE_TAG, 'False')
pass
@staticmethod
def _get_target_in_not_active(instance, active, not_active):
if not instance:
Logger.error('Instance Id is not specified')
return None
target_in_active = list(filter(lambda inst: inst.id == instance, active))
if len(target_in_active) > 0:
Logger.error('The instance is already active')
return None
target_in_not_actives = list(filter(lambda inst: inst.id == instance, not_active))
if len(target_in_not_actives) == 0:
Logger.error('The instance is not running')
return None
if len(target_in_not_actives) > 1:
Logger.error('Instances consistancy error: more than one instance with the same id')
return None
return target_in_not_actives
def all_volumes(self):
rest_volumes = []
volumes = self._conn.get_all_volumes()
for vol in volumes:
if self.VOLUME_TAG in vol.tags:
volumes.append(vol)
else:
rest_volumes.append(vol)
return (volumes, rest_volumes)
def describe_storages(self):
Logger.info(self.VOLUME_FORMAT.format('Name', 'Id', 'Size(GB)',
'Status', 'Type', 'Encrypted', 'Zone'))
(volumes, rest_volumes) = self.all_volumes()
for vol in volumes:
Logger.info(self.VOLUME_FORMAT.format(vol.tags[self.VOLUME_TAG],
vol.id, str(vol.size), vol.status,
vol.type, str(vol.encrypted), vol.zone))
pass
dvc/cloud/instance_manager.py
-21
View File
@@ -1,21 +0,0 @@
from dvc.cloud.instance_aws import InstanceAws
from dvc.logger import Logger
class CloudSettings(object):
def __init__(self, global_storage_path, cloud_config):
self.cloud_config = cloud_config
self.global_storage_path = global_storage_path
class InstanceManager(object):
def instances(self):
return []
def create(self, name, cloud, parsed_args, conf_parser):
if name in set(map(lambda x: x.name, self.instances())):
Logger.error('Instance with name {} is already exist'.format(name))
return 1
inst = InstanceAws()
pass
dvc/cloud/storage_aws.py
-55
View File
@@ -1,55 +0,0 @@
import sys, os, argparse, boto.ec2
from dvc.cloud.base_aws import AwsBase #logger, Logger
class StorageTool(AwsBase):
def __init__(self, homedir, conf_file):
super().__init__(homedir, conf_file)
def extend_args(self, parser):
parser.add_argument('--size', metavar = '', type = int,
help = 'Storage volume size in GB. Min=1, Max=16384.')
parser.add_argument('--snapshot-id', metavar = '',
help = 'The snapshot id for the volumne creation.')
parser.add_argument('--type', metavar = '',
help = 'The volume type: standard, io1 - provisioned IOPS, gp2 - general purpose')
parser.add_argument('--iops', metavar = '',
help = 'Number of I\O per second for provisioned IOPS volumes.')
parser.add_argument('--encryption', metavar = '',
help = 'Enable encription for the volume.')
pass
def create(self, storage_name):
if not self._args.size:
logger.error("Storage volume size is not specified")
return
try:
vol = self._conn.create_volume(self._args.size,
self._args.zone,
self._args.snapshot_id,
self._args.type,
self._args.iops,
self.toBool(self._args.encryption))
vol.add_tag(self.VOLUME_TAG, storage_name)
except self._conn.ResponseError as e:
logger.error('cannot create a volume: %s' % e.message)
return
logger.log("%sGB volume %s was created" % (self._args.size, storage_name))
pass
if __name__ == "__main__":
HOMEDIR_VAR = "ONEBOXML_HOME"
if not (HOMEDIR_VAR in os.environ):
raise Exception(HOMEDIR_VAR + " variable is not defined")
homedir = os.environ[HOMEDIR_VAR]
conf_file = homedir + "/oneboxml.conf"
logger = Logger()
storage_tool = StorageTool(homedir, conf_file)
(command, param) = storage_tool.get_command()
if command == 'create':
storage_tool.create(param)
dvc/command/instance_create.py
-28
View File
@@ -1,28 +0,0 @@
from dvc.cloud.instance_manager import InstanceManager
from dvc.command.common.base import CmdBase
from dvc.exceptions import DvcException
from dvc.logger import Logger
class CmdInstanceCreate(CmdBase):
def __init__(self, settings):
super(CmdInstanceCreate, self).__init__(settings)
def run(self):
name = self.parsed_args.name
cloud = self.parsed_args.cloud or self.settings.config.cloud
# print('NAME: {}'.format(name))
# print('CLOUD: {}'.format(cloud))
if not name:
Logger.error('Instance name is not defined')
return 1
try:
InstanceManager().create(name, cloud, self.parsed_args, self.settings.config)
except DvcException as ex:
Logger.error('Instance creation error: {}'.format(ex))
return 1
return 0
dvc/data_cloud.py
+94 -3
View File
@@ -29,15 +29,13 @@ except ImportError:
from urllib.parse import urlparse
import dvc
from dvc.cloud.instance_manager import CloudSettings
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.config import ConfigError
from dvc.progress import progress
from dvc.utils import copyfile
from dvc.cloud.credentials_aws import AWSCredentials
from dvc.system import System
from dvc.utils import map_progress
from dvc.utils import map_progress, cached_property
STATUS_UNKNOWN = 0
STATUS_OK = 1
@@ -53,6 +51,99 @@ STATUS_MAP = {
(False, True, None) : STATUS_DELETED,
}
class CloudSettings(object):
def __init__(self, global_storage_path, cloud_config):
self.cloud_config = cloud_config
self.global_storage_path = global_storage_path
class AWSCredentials(object):
def __init__(self, cloud_config):
self._conf_credpath = cloud_config.get('CredentialPath', None)
self._conf_credsect = cloud_config.get('CredentialSection', 'default')
@property
def access_key_id(self):
if self.creds:
return self.creds[0]
return None
@property
def secret_access_key(self):
if self.creds:
return self.creds[1]
return None
@cached_property
def creds(self):
return self._get_credentials()
def _get_credentials(self):
""" gets aws credentials, looking in various places
Params:
Searches:
1 any override in dvc.conf [AWS] CredentialPath;
2 ~/.aws/credentials
Returns:
if successfully found, (access_key_id, secret)
None otherwise
"""
# FIX: It won't work in Windows.
default_path = os.path.expanduser('~/.aws/credentials')
default_sect = 'default'
default_cred_location = (default_path, default_sect)
cred_locations = self._credential_paths(default_cred_location)
for cred_location in cred_locations:
try:
path = cred_location[0]
section = cred_location[1]
cc = configparser.SafeConfigParser()
# use readfp(open( ... to aid mocking.
cc.readfp(open(path, 'r'))
if section in cc.keys():
access_key = cc[section].get('aws_access_key_id', None)
secret = cc[section].get('aws_secret_access_key', None)
if access_key is not None and secret is not None:
return (access_key, secret)
else:
Logger.warn('Unable to find section {} in AWS credential file {}'.format(section, path))
except Exception as e:
pass
return None
def _credential_paths(self, default_cred_location):
results = []
if self._conf_credpath is not None and len(self._conf_credpath) > 0:
credpath = os.path.expanduser(self._conf_credpath)
if os.path.isfile(credpath):
results.append((credpath, self._conf_credsect))
else:
msg = 'AWS CredentialPath {} not found; falling back to default file {} and section {}'
Logger.warn(msg.format(credpath, default_cred_location[0], default_cred_location[1]))
results.append(default_cred_location)
else:
results.append(default_cred_location)
return results
def sanity_check(self):
creds = self._get_credentials()
if creds is None:
Logger.info("can't find aws credetials, assuming envirment variables or iam role")
# self._aws_creds = creds
class DataCloudError(DvcException):
""" Data Cloud exception """
def __init__(self, msg):
tests/test_data_cloud.py
+1 -1
View File
@@ -4,7 +4,7 @@ import uuid
from dvc.config import ConfigError
from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL
from dvc.data_cloud import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED
from dvc.cloud.instance_manager import CloudSettings
from dvc.data_cloud import CloudSettings
from tests.basic_env import TestDvc