diff --git a/dvc/cli.py b/dvc/cli.py index af3d63484..4e4bae560 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -16,7 +16,6 @@ from dvc.command.lock import CmdLock from dvc.command.gc import CmdGC from dvc.command.add import CmdAdd from dvc.command.show_workflow import CmdShowWorkflow -from dvc.command.instance_create import CmdInstanceCreate from dvc.command.config import CmdConfig from dvc.command.show_pipeline import CmdShowPipeline from dvc.command.checkout import CmdCheckout @@ -222,84 +221,6 @@ def parse_args(argv=None): help='Collect garbage') gc_parser.set_defaults(func=CmdGC) - # Cloud - ex_parser = subparsers.add_parser( - 'ex', - parents=[parent_parser], - help='Experimental commands') - ex_subparsers = ex_parser.add_subparsers( - dest='cmd', - help='Use dvc cloud CMD --help for command-specific help') - - cloud_parser = ex_subparsers.add_parser( - 'cloud', - parents=[parent_parser], - help='Cloud manipulation') - cloud_subparsers = cloud_parser.add_subparsers( - dest='cmd', - help='Use dvc cloud CMD --help for command-specific help') - - # Instance create - instance_create_parser = cloud_subparsers.add_parser( - 'create', - help='Create cloud instance') - instance_create_parser.add_argument('name', - # metavar='', - nargs='?', - help='Instance name.') - instance_create_parser.add_argument('-c', - '--cloud', - # metavar='', - help='Cloud: AWS, GCP.') - instance_create_parser.add_argument('-t', - '--type', - # metavar='', - help='Instance type.') - instance_create_parser.add_argument('-i', - '--image', - # metavar='', - help='Instance image.') - instance_create_parser.add_argument('--spot-price', - metavar='PRICE', - help='Spot instance price in $ i.e. 1.54') - instance_create_parser.add_argument('--spot-timeout', - metavar='TIMEOUT', - type=int, - help='Spot instances waiting timeout in seconds.') - instance_create_parser.add_argument('--keypair-name', - metavar='KEYPAIR', - help='The name of key pair for instance launch') - instance_create_parser.add_argument('--keypair-dir', - metavar='DIR', - help='The directory of key pairs') - instance_create_parser.add_argument('--security-group', - metavar='GROUP', - help='Security group') - instance_create_parser.add_argument('--region', - help='Region') - instance_create_parser.add_argument('--zone', - help='Zone') - instance_create_parser.add_argument('--subnet-id', - metavar='SUBNET', - help='Subnet ID') - instance_create_parser.add_argument('--storage', - # metavar='', - help='The name of attachable storage volume.') - # WHERE EBS IS? - instance_create_parser.add_argument('--monitoring', - action='store_true', - default=False, - help='Enable EC2 instance monitoring') - instance_create_parser.add_argument('--ebs-optimized', - action='store_true', - default=False, - help='Enable EBS I\O optimization') - instance_create_parser.add_argument('--disks-to-ride0', - action='store_true', - default=False, - help='Detect all ephemeral disks and stripe together in raid-0') - instance_create_parser.set_defaults(func=CmdInstanceCreate) - # Config config_parser = subparsers.add_parser( 'config', diff --git a/dvc/cloud/__init__.py b/dvc/cloud/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/dvc/cloud/base_aws.py b/dvc/cloud/base_aws.py deleted file mode 100644 index a77ca6837..000000000 --- a/dvc/cloud/base_aws.py +++ /dev/null @@ -1,43 +0,0 @@ -import sys, os, time, configparser, argparse, boto.ec2 - - -class AwsBase: - DEFAULT_REGION = "us-east-1" - DEFAULT_ZONE = "us-east-1a" - - def __init__(self, homedir, conf_file): - self._homedir = homedir - - config = configparser.ConfigParser() - config.read(conf_file) - - self._aws_conf = config['AWS'] - self._args = self.parse_args() - - self._conn = boto.ec2.connect_to_region( - self._args.region, - aws_access_key_id = self._aws_conf.get("AccessKeyID"), - aws_secret_access_key = self._aws_conf.get("SecretAccessKey")) - pass - - def get_command(self): - return (self._args.command, self._args.param) - - def parse_args(self): - parser = argparse.ArgumentParser() - parser.add_argument('command') - parser.add_argument('param', nargs = '?', default = '') - parser.add_argument('--region', metavar = '', - default = self._aws_conf.get("Region", self.DEFAULT_REGION), - help = 'AWS region: us-east-2, ...') - parser.add_argument('--zone', metavar = '', - default = self._aws_conf.get("Zone", self.DEFAULT_ZONE), - help = 'Availability Zone. Examples: us-east-1a, us-east-1b, ...') - self.extend_args(parser) - - args = parser.parse_args() - return args - - def extend_args(self, parser): - raise NotImplementedError() - diff --git a/dvc/cloud/credentials_aws.py b/dvc/cloud/credentials_aws.py deleted file mode 100644 index e60aaae9a..000000000 --- a/dvc/cloud/credentials_aws.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import configparser - -from dvc.utils import cached_property -from dvc.logger import Logger - - -class AWSCredentials(object): - def __init__(self, cloud_config): - self._conf_credpath = cloud_config.get('CredentialPath', None) - self._conf_credsect = cloud_config.get('CredentialSection', 'default') - - @property - def access_key_id(self): - if self.creds: - return self.creds[0] - return None - - @property - def secret_access_key(self): - if self.creds: - return self.creds[1] - return None - - @cached_property - def creds(self): - return self._get_credentials() - - def _get_credentials(self): - """ gets aws credentials, looking in various places - - Params: - - Searches: - 1 any override in dvc.conf [AWS] CredentialPath; - 2 ~/.aws/credentials - - - Returns: - if successfully found, (access_key_id, secret) - None otherwise - """ - - # FIX: It won't work in Windows. - default_path = os.path.expanduser('~/.aws/credentials') - default_sect = 'default' - default_cred_location = (default_path, default_sect) - - cred_locations = self._credential_paths(default_cred_location) - for cred_location in cred_locations: - try: - path = cred_location[0] - section = cred_location[1] - - cc = configparser.SafeConfigParser() - - # use readfp(open( ... to aid mocking. - cc.readfp(open(path, 'r')) - - if section in cc.keys(): - access_key = cc[section].get('aws_access_key_id', None) - secret = cc[section].get('aws_secret_access_key', None) - - if access_key is not None and secret is not None: - return (access_key, secret) - else: - Logger.warn('Unable to find section {} in AWS credential file {}'.format(section, path)) - except Exception as e: - pass - - return None - - def _credential_paths(self, default_cred_location): - results = [] - if self._conf_credpath is not None and len(self._conf_credpath) > 0: - credpath = os.path.expanduser(self._conf_credpath) - if os.path.isfile(credpath): - results.append((credpath, self._conf_credsect)) - else: - msg = 'AWS CredentialPath {} not found; falling back to default file {} and section {}' - Logger.warn(msg.format(credpath, default_cred_location[0], default_cred_location[1])) - results.append(default_cred_location) - else: - results.append(default_cred_location) - return results - - def sanity_check(self): - creds = self._get_credentials() - if creds is None: - Logger.info("can't find aws credetials, assuming envirment variables or iam role") - # self._aws_creds = creds diff --git a/dvc/cloud/instance.py b/dvc/cloud/instance.py deleted file mode 100644 index a1733c09d..000000000 --- a/dvc/cloud/instance.py +++ /dev/null @@ -1,41 +0,0 @@ -from dvc.exceptions import DvcException - - -class InstanceError(DvcException): - def __init__(self, msg): - super(InstanceError).__init__(u"Instance error: " + msg) - - -class Instance(object): - INSTANCE_STATE_TAG = 'dvc-is-active' - VOLUME_TAG = 'dvc-volume' - - def __init__(self, name, cloud, parsed_args, conf_parser): - self._name = name - self._cloud = cloud - - self._type = parsed_args.type or conf_parser._config[cloud]['Type'] - self._image = parsed_args.image or conf_parser._config[cloud]['Image'] - - self._spot_price = parsed_args.spot_price or conf_parser._config[cloud]['SpotPrice'] - self._spot_timeout = parsed_args.spot_timeout or conf_parser._config[cloud]['SpotTimeout'] - - self._keypair_name = parsed_args.keypair_name or conf_parser._config[cloud]['KeyPairName'] - self._keypair_dir = parsed_args.keypair_dir or conf_parser._config[cloud]['KeyPairDir'] - self._security_group = parsed_args.security_group or conf_parser._config[cloud]['SecurityGroup'] - - self._region = parsed_args.region or conf_parser._config[cloud]['Region'] - self._zone = parsed_args.zone or conf_parser._config[cloud]['Zone'] - self._subnet_id = parsed_args.subnet_id or conf_parser._config[cloud]['SubnetId'] - - self._storage = parsed_args.storage or conf_parser._config[cloud]['Storage'] - - self._monitoring = parsed_args.monitoring or conf_parser._config[cloud]['Monitoring'] - self._ebs_optimized = parsed_args.ebs_optimized or conf_parser._config[cloud]['EbsOptimized'] - self._disks_to_raid0 = parsed_args.disks_to_ride0 or conf_parser._config[cloud]['DisksToRAID0'] - pass - - def toBool(self, value): - if value == None: - return False - return value.lower() in ("yes", "true", "t", "1") \ No newline at end of file diff --git a/dvc/cloud/instance_aws.py b/dvc/cloud/instance_aws.py deleted file mode 100644 index 80d9486e4..000000000 --- a/dvc/cloud/instance_aws.py +++ /dev/null @@ -1,371 +0,0 @@ -import sys, os, time, boto.ec2 - -from dvc.cloud.credentials_aws import AWSCredentials -from dvc.cloud.instance import Instance, InstanceError -from dvc.logger import Logger - - -class InstanceAws(Instance): - TERMINATED_STATE = 'terminated' - INSTANCE_FORMAT = "{:8s}{:12s}{:12s}{:12s}{:16s}{:16s}{:16s}" - VOLUME_FORMAT = "{:20s}{:16s}{:12s}{:12s}{:12s}{:12s}{:16s}" - - def __init__(self, homedir, conf_file): - super(InstanceAws).__init__(homedir, conf_file) - self._aws_creds = AWSCredentials(self._cloud_config) - - self._conn = boto.ec2.connect_to_region( - self._region, - aws_access_key_id=self._aws_conf.get("AccessKeyID"), - aws_secret_access_key=self._aws_conf.get("SecretAccessKey")) - - def create(self): - instance = None - - if not self._spot_price: - reserv = self._conn.run_instances(self._image, - key_name=self.get_key_name(), - instance_type=self._type, - security_groups=self.get_security_groups(), - monitoring_enabled=self.toBool(self._monitoring), - #subnet_id = self._args.subnet_id, - placement=self._zone, - ebs_optimized=self.toBool(self._ebs_optimized)) - instance = reserv.instances[0] - else: - instance = self.create_spot_instance() - - return instance - - def find_volume(self): - (volumes, rest_volumes) = self.all_volumes() - - for volume in volumes: - if volume.tags[self.VOLUME_TAG] == self._storage: - return volume - msg = u'Cannot find storage volume {}. Verify that the volume was created.'.format( - self._storage) - raise InstanceError(msg) - - def create_spot_instance(self): - # Create spot instance - req = self._conn.request_spot_instances(price = self._args.spot_price, - image_id = self._args.image, - key_name = self.get_key_name(), - instance_type = self._args.instance_type, - security_groups = self.get_security_groups(), - monitoring_enabled = self.toBool(self._args.monitoring), - #subnet_id = self._args.subnet_id, - placement = self._args.zone, - ebs_optimized = self.toBool(self._args.ebs_optimized)) - job_instance_id = self._wait_for_an_instance(req) - - if not job_instance_id: - self._conn.cancel_spot_instance_requests(req[0].id) - raise InstanceError(u'Unable to obtain {} spot instance in region {} for price ${}: {}'.format( - (self._args.instance_type, self._args.region, self._args.spot_price, - 'the request was canceled'))) - - Logger.info(u'{} spot instance was created: {}'.format(self._type, job_instance_id)) - reservations = self._conn.get_all_instances(instance_ids = job_instance_id) - instance = reservations[0].instances[0] - return instance - - def _wait_for_an_instance(self, req): - job_instance_id = None - sec = 0 - Logger.info(u'Waiting for a spot instance. Request {}.'.format(req[0].id)) - while job_instance_id == None and sec < self._args.spot_timeout: - sys.stdout.write(".") - sys.stdout.flush() - - job_sir_id = req[0].id - reqs = self._conn.get_all_spot_instance_requests() - for sir in reqs: - if sir.id == job_sir_id: - job_instance_id = sir.instance_id - break - time.sleep(1) - sec += 1 - sys.stdout.write("\n") - return job_instance_id - - def run_instance(self): - if not self._image: - raise InstanceError('Cannot run EC2 instance: image (AMI) is not defined') - - # Remove active tag. - active_filter = {'tag-key': self.INSTANCE_STATE_TAG, 'tag-value': 'True'} - active_reserv = self._conn.get_all_instances(filters = active_filter) - active_instances = [i for r in active_reserv for i in r.instances] - - if len(active_instances) > 0: - #active_insts = active_reserv.instances - if len(active_instances) > 1: - Logger.error('EC2 instances consistency error - more than one active EC2 instance') - for inst in active_instances: - inst.remove_tag(self.INSTANCE_STATE_TAG, 'True') - inst.add_tag(self.INSTANCE_STATE_TAG, 'False') - if inst.state != self.TERMINATED_STATE: - Logger.info('{} instance {} is not longer active'.format(inst.instance_type, inst.id)) - - self._run_instance() - pass - - def _run_instance(self): - instance = self.create_instance() - # Assign the created instace as active. - instance.add_tag(self.INSTANCE_STATE_TAG, 'True') - Logger.info('New {} instance {} was selected as active'.format(instance.instance_type, instance.id)) - Logger.info('Waiting for a running status') - while instance.state != 'running': - sys.stdout.write(".") - sys.stdout.flush() - time.sleep(1) - instance.update() - sys.stdout.write("\n") - self._conn.attach_volume(self.find_volume().id, instance.id, "/dev/sdx") - - def get_security_groups(self, ssh_port = 22): - group_name = self._security_group - - # Check if the group exists and create one if does not. - try: - group = self._conn.get_all_security_groups(groupnames=[group_name])[0] - except self._conn.ResponseError as e: - if e.code == 'InvalidGroup.NotFound': - Logger.error('AWS Security Group {} does not exist: creating the group'.format(group_name)) - group = self._conn.create_security_group(group_name, 'group with SSH access') - else: - raise - - # Enable SSH access. - try: - group.authorize('tcp', ssh_port, ssh_port, '0.0.0.0/0') - except self._conn.ResponseError as e: - if e.code != 'InvalidPermission.Duplicate': - raise - - return [group_name] - - def get_key_name(self): - if not self._keypair_name: - raise InstanceError('AWS keypair cannot be created: KeyName is not specified in AWS section in the config file') - if not self._keypair_dir: - raise InstanceError('AWS keypair cannot be created: KeyDir is not specified in AWS section in the config file') - - # Check if the key exists and create one if does not. - try: - key = self._conn.get_all_key_pairs(keynames=[self._keypair_name])[0] - except self._conn.ResponseError as e: - if e.code != 'InvalidKeyPair.NotFound': - raise - self.create_new_keypair() - - return self._keypair_name - - def create_new_keypair(self): - Logger.info('AWS key {} does not exist: creating the key'.format(self._keypair_name)) - # Create an SSH key to use when logging into instances. - key = self._conn.create_key_pair(self._keypair_name) - Logger.info('AWS key was created: {}'.format(self._keypair_name)) - # Expand key dir. - key_dir = os.path.expandvars(self._keypair_dir) - if not os.path.isdir(self._keypair_dir): - os.mkdir(self._keypair_dir, 0o700) - - # Private key has to be stored locally. - key_file = os.path.join(key_dir, self._keypair_name + '.pem') - # key.save(key_file) # doesn't work in python3 - fp = open(key_file, 'w') - fp.write(key.material) - fp.close() - os.chmod(key_file, 0o600) - Logger.info('AWS private key file was saved: {}'.format(key_file)) - - def all_instances(self, terminated = False): - # active = [] - # not_active = [] - # rest_inst = [] - - reserv = self._conn.get_all_instances() - instances = [i for r in reserv for i in r.instances] - - active, not_active, rest_inst = self.split_inatances_by_types(instances, terminated) - - if len(active) > 1: - Logger.error(u'Instance consistancy error - more than one instance is in active state') - if len(active) == 0 and len(not_active) > 0: - Logger.error(u'Instance consistancy error - no active instances') - return (active, not_active, rest_inst) - - def split_inatances_by_types(self, instances, terminated): - active = [] - not_active = [] - rest_inst = [] - for inst in instances: - if terminated == False and inst.state == self.TERMINATED_STATE: - continue - - if self.INSTANCE_STATE_TAG in inst.tags: - if inst.tags[self.INSTANCE_STATE_TAG] == 'True': - active.append(inst) - else: - not_active.append(inst) - else: - rest_inst.append(inst) - - return active, not_active, rest_inst - - def describe_instance(self, inst, volumes, is_active): - ip = inst.ip_address - ip_private = inst.private_ip_address - if not ip: - ip = '' - if not ip_private: - ip_private = '' - - activeFlag = '' - if is_active: - activeFlag = ' ***' - - volume_name = self.get_volume_name(inst, volumes) - - return self.INSTANCE_FORMAT.format(activeFlag, - inst.id[:12], - inst.instance_type[:12], - inst.state[:12], - volume_name, - ip[:16], - ip_private[:16]) - - def get_volume_name(self, inst, volumes): - volume_name = '' - bdm = inst.block_device_mapping - for device_type in bdm.values(): - for vol in volumes: - if device_type.volume_id == vol.id: - if self.VOLUME_TAG in vol.tags: - volume_name = vol.tags[self.VOLUME_TAG] - else: - Logger.error('Instance {} storage volume does not have tag'.format(inst.id)) - break - return volume_name - - def describe_instances(self): - (active, not_active, rest_inst) = self.all_instances() - (volumes, rest_volumes) = self.all_volumes() - - Logger.info(self.INSTANCE_FORMAT.format('Active', 'Id', 'Type', 'State', 'Storage', - 'IP public', 'IP private')) - - for inst in active: - Logger.info(self.describe_instance(inst, volumes, True)) - for inst in not_active: - logger.info(self.describe_instance(inst, volumes, False)) - - if len(rest_inst) > 0: - Logger.error(u'{} not tracked instances were found: not in the list'.format(len(rest_inst))) - pass - - def get_instance_id(self, instance): - if instance[:2] != 'i-' and len(instance) == 8: - instance = 'i-' + instance - return instance - - def terminate_instances(self, instance): - instance = self.get_instance_id(instance) - - if not instance: - Logger.error('Instance Id is not specified') - return - - (active, not_active, rest_inst) = self.all_instances() - - if instance == 'all': - target_in_active = active - target_in_not_active = not_active - else: - target_in_active = list(filter(lambda inst: inst.id == instance, active)) - target_in_not_active = list(filter(lambda inst: inst.id == instance, not_active)) - - self._terminate(target_in_active, target_in_not_active) - - if instance != 'all' and len(target_in_active) > 0 and len(not_active) > 0: - new_active_inst = not_active[0] - new_active_inst.remove_tag(self.INSTANCE_STATE_TAG, 'False') - new_active_inst.add_tag(self.INSTANCE_STATE_TAG, 'True') - randomly = '' - if len(not_active) > 1: - randomly = 'randomly ' - Logger.error('{} instance {} was {} selected as active because of an active instance was terminated'.format( - new_active_inst.instance_type, new_active_inst.id, randomly)) - pass - - def _terminate(self, target_in_active, target_in_not_active): - if target_in_not_active: - self._conn.terminate_instances(instance_ids=[inst.id for inst in target_in_not_active]) - for inst in target_in_active: - inst.remove_tag(self.INSTANCE_STATE_TAG, 'True') - inst.add_tag(self.INSTANCE_STATE_TAG, 'False') - self._conn.terminate_instances(instance_ids=[inst.id]) - - def set_active_instance(self, instance): - instance = self.get_instance_id(instance) - - (active, not_active, rest_inst) = self.all_instances() - - target_in_not_actives = InstanceAws._get_target_in_not_active(instance, active, not_active) - if not target_in_not_actives: - return - - target_in_not_actives[0].remove_tag(self.INSTANCE_STATE_TAG, 'False') - target_in_not_actives[0].add_tag(self.INSTANCE_STATE_TAG, 'True') - - for inst in active: - inst.remove_tag(self.INSTANCE_STATE_TAG, 'True') - inst.add_tag(self.INSTANCE_STATE_TAG, 'False') - pass - - @staticmethod - def _get_target_in_not_active(instance, active, not_active): - if not instance: - Logger.error('Instance Id is not specified') - return None - - target_in_active = list(filter(lambda inst: inst.id == instance, active)) - if len(target_in_active) > 0: - Logger.error('The instance is already active') - return None - - target_in_not_actives = list(filter(lambda inst: inst.id == instance, not_active)) - if len(target_in_not_actives) == 0: - Logger.error('The instance is not running') - return None - - if len(target_in_not_actives) > 1: - Logger.error('Instances consistancy error: more than one instance with the same id') - return None - - return target_in_not_actives - - - def all_volumes(self): - rest_volumes = [] - volumes = self._conn.get_all_volumes() - for vol in volumes: - if self.VOLUME_TAG in vol.tags: - volumes.append(vol) - else: - rest_volumes.append(vol) - return (volumes, rest_volumes) - - def describe_storages(self): - Logger.info(self.VOLUME_FORMAT.format('Name', 'Id', 'Size(GB)', - 'Status', 'Type', 'Encrypted', 'Zone')) - (volumes, rest_volumes) = self.all_volumes() - for vol in volumes: - Logger.info(self.VOLUME_FORMAT.format(vol.tags[self.VOLUME_TAG], - vol.id, str(vol.size), vol.status, - vol.type, str(vol.encrypted), vol.zone)) - pass diff --git a/dvc/cloud/instance_manager.py b/dvc/cloud/instance_manager.py deleted file mode 100644 index 9783674fa..000000000 --- a/dvc/cloud/instance_manager.py +++ /dev/null @@ -1,21 +0,0 @@ -from dvc.cloud.instance_aws import InstanceAws -from dvc.logger import Logger - - -class CloudSettings(object): - def __init__(self, global_storage_path, cloud_config): - self.cloud_config = cloud_config - self.global_storage_path = global_storage_path - - -class InstanceManager(object): - def instances(self): - return [] - - def create(self, name, cloud, parsed_args, conf_parser): - if name in set(map(lambda x: x.name, self.instances())): - Logger.error('Instance with name {} is already exist'.format(name)) - return 1 - - inst = InstanceAws() - pass diff --git a/dvc/cloud/storage_aws.py b/dvc/cloud/storage_aws.py deleted file mode 100644 index 3a9bb1299..000000000 --- a/dvc/cloud/storage_aws.py +++ /dev/null @@ -1,55 +0,0 @@ -import sys, os, argparse, boto.ec2 -from dvc.cloud.base_aws import AwsBase #logger, Logger - -class StorageTool(AwsBase): - - def __init__(self, homedir, conf_file): - super().__init__(homedir, conf_file) - - def extend_args(self, parser): - parser.add_argument('--size', metavar = '', type = int, - help = 'Storage volume size in GB. Min=1, Max=16384.') - parser.add_argument('--snapshot-id', metavar = '', - help = 'The snapshot id for the volumne creation.') - parser.add_argument('--type', metavar = '', - help = 'The volume type: standard, io1 - provisioned IOPS, gp2 - general purpose') - parser.add_argument('--iops', metavar = '', - help = 'Number of I\O per second for provisioned IOPS volumes.') - parser.add_argument('--encryption', metavar = '', - help = 'Enable encription for the volume.') - pass - - def create(self, storage_name): - if not self._args.size: - logger.error("Storage volume size is not specified") - return - - try: - vol = self._conn.create_volume(self._args.size, - self._args.zone, - self._args.snapshot_id, - self._args.type, - self._args.iops, - self.toBool(self._args.encryption)) - vol.add_tag(self.VOLUME_TAG, storage_name) - except self._conn.ResponseError as e: - logger.error('cannot create a volume: %s' % e.message) - return - - logger.log("%sGB volume %s was created" % (self._args.size, storage_name)) - pass - -if __name__ == "__main__": - HOMEDIR_VAR = "ONEBOXML_HOME" - if not (HOMEDIR_VAR in os.environ): - raise Exception(HOMEDIR_VAR + " variable is not defined") - homedir = os.environ[HOMEDIR_VAR] - conf_file = homedir + "/oneboxml.conf" - - logger = Logger() - - storage_tool = StorageTool(homedir, conf_file) - - (command, param) = storage_tool.get_command() - if command == 'create': - storage_tool.create(param) diff --git a/dvc/command/instance_create.py b/dvc/command/instance_create.py deleted file mode 100644 index 6697fa769..000000000 --- a/dvc/command/instance_create.py +++ /dev/null @@ -1,28 +0,0 @@ -from dvc.cloud.instance_manager import InstanceManager -from dvc.command.common.base import CmdBase -from dvc.exceptions import DvcException -from dvc.logger import Logger - - -class CmdInstanceCreate(CmdBase): - def __init__(self, settings): - super(CmdInstanceCreate, self).__init__(settings) - - def run(self): - name = self.parsed_args.name - cloud = self.parsed_args.cloud or self.settings.config.cloud - - # print('NAME: {}'.format(name)) - # print('CLOUD: {}'.format(cloud)) - - if not name: - Logger.error('Instance name is not defined') - return 1 - - try: - InstanceManager().create(name, cloud, self.parsed_args, self.settings.config) - except DvcException as ex: - Logger.error('Instance creation error: {}'.format(ex)) - return 1 - - return 0 diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 42b10bf65..aa966ad58 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -29,15 +29,13 @@ except ImportError: from urllib.parse import urlparse import dvc -from dvc.cloud.instance_manager import CloudSettings from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.config import ConfigError from dvc.progress import progress from dvc.utils import copyfile -from dvc.cloud.credentials_aws import AWSCredentials from dvc.system import System -from dvc.utils import map_progress +from dvc.utils import map_progress, cached_property STATUS_UNKNOWN = 0 STATUS_OK = 1 @@ -53,6 +51,99 @@ STATUS_MAP = { (False, True, None) : STATUS_DELETED, } + +class CloudSettings(object): + def __init__(self, global_storage_path, cloud_config): + self.cloud_config = cloud_config + self.global_storage_path = global_storage_path + + +class AWSCredentials(object): + def __init__(self, cloud_config): + self._conf_credpath = cloud_config.get('CredentialPath', None) + self._conf_credsect = cloud_config.get('CredentialSection', 'default') + + @property + def access_key_id(self): + if self.creds: + return self.creds[0] + return None + + @property + def secret_access_key(self): + if self.creds: + return self.creds[1] + return None + + @cached_property + def creds(self): + return self._get_credentials() + + def _get_credentials(self): + """ gets aws credentials, looking in various places + + Params: + + Searches: + 1 any override in dvc.conf [AWS] CredentialPath; + 2 ~/.aws/credentials + + + Returns: + if successfully found, (access_key_id, secret) + None otherwise + """ + + # FIX: It won't work in Windows. + default_path = os.path.expanduser('~/.aws/credentials') + default_sect = 'default' + default_cred_location = (default_path, default_sect) + + cred_locations = self._credential_paths(default_cred_location) + for cred_location in cred_locations: + try: + path = cred_location[0] + section = cred_location[1] + + cc = configparser.SafeConfigParser() + + # use readfp(open( ... to aid mocking. + cc.readfp(open(path, 'r')) + + if section in cc.keys(): + access_key = cc[section].get('aws_access_key_id', None) + secret = cc[section].get('aws_secret_access_key', None) + + if access_key is not None and secret is not None: + return (access_key, secret) + else: + Logger.warn('Unable to find section {} in AWS credential file {}'.format(section, path)) + except Exception as e: + pass + + return None + + def _credential_paths(self, default_cred_location): + results = [] + if self._conf_credpath is not None and len(self._conf_credpath) > 0: + credpath = os.path.expanduser(self._conf_credpath) + if os.path.isfile(credpath): + results.append((credpath, self._conf_credsect)) + else: + msg = 'AWS CredentialPath {} not found; falling back to default file {} and section {}' + Logger.warn(msg.format(credpath, default_cred_location[0], default_cred_location[1])) + results.append(default_cred_location) + else: + results.append(default_cred_location) + return results + + def sanity_check(self): + creds = self._get_credentials() + if creds is None: + Logger.info("can't find aws credetials, assuming envirment variables or iam role") + # self._aws_creds = creds + + class DataCloudError(DvcException): """ Data Cloud exception """ def __init__(self, msg): diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 4cd41ddc2..cf04d6826 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -4,7 +4,7 @@ import uuid from dvc.config import ConfigError from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL from dvc.data_cloud import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED -from dvc.cloud.instance_manager import CloudSettings +from dvc.data_cloud import CloudSettings from tests.basic_env import TestDvc