diff --git a/dvc/cli.py b/dvc/cli.py index 51bc22372..562d92bf8 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -1,9 +1,6 @@ from __future__ import print_function -import os -import sys import argparse -import configparser from multiprocessing import cpu_count from dvc.command.init import CmdInit @@ -15,7 +12,7 @@ from dvc.command.lock import CmdLock from dvc.command.gc import CmdGC from dvc.command.import_file import CmdImportFile from dvc.command.target import CmdTarget -from dvc.command.test import CmdTest +from dvc.command.instance_create import CmdInstanceCreate from dvc.config import Config from dvc import VERSION @@ -266,6 +263,67 @@ def parse_args(argv=None): help='Reset target.') target_parser.set_defaults(func=CmdTarget) + # Instance create + instance_create_parser = subparsers.add_parser('instance-create', + parents=[parent_parser], + help='Create cloud instance') + instance_create_parser.add_argument('name', + # metavar='', + nargs='?', + help='Instance name.') + instance_create_parser.add_argument('-c', + '--cloud', + # metavar='', + help='Cloud: AWS, GCP.') + instance_create_parser.add_argument('-t', + '--type', + # metavar='', + help='Instance type.') + instance_create_parser.add_argument('-i', + '--image', + # metavar='', + help='Instance image.') + instance_create_parser.add_argument('--spot-price', + metavar='PRICE', + help='Spot instance price in $ i.e. 1.54') + instance_create_parser.add_argument('--spot-timeout', + metavar='TIMEOUT', + type=int, + help='Spot instances waiting timeout in seconds.') + instance_create_parser.add_argument('--keypair-name', + metavar='KEYPAIR', + help='The name of key pair for instance launch') + instance_create_parser.add_argument('--keypair-dir', + metavar='DIR', + help='The directory of key pairs') + instance_create_parser.add_argument('--security-group', + metavar='GROUP', + help='Security group') + instance_create_parser.add_argument('--region', + help='Region') + instance_create_parser.add_argument('--zone', + help='Zone') + instance_create_parser.add_argument('--subnet-id', + metavar='SUBNET', + help='Subnet ID') + instance_create_parser.add_argument('--storage', + # metavar='', + help='The name of attachable storage volume.') + # WHERE EBS IS? + instance_create_parser.add_argument('--monitoring', + action='store_true', + default=False, + help='Enable EC2 instance monitoring') + instance_create_parser.add_argument('--ebs-optimized', + action='store_true', + default=False, + help='Enable EBS I\O optimization') + instance_create_parser.add_argument('--disks-to-ride0', + action='store_true', + default=False, + help='Detect all ephemeral disks and stripe together in raid-0') + instance_create_parser.set_defaults(func=CmdInstanceCreate) + if isinstance(argv, str): argv = argv.split() diff --git a/dvc/cloud/__init__.py b/dvc/cloud/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dvc/cloud/base_aws.py b/dvc/cloud/base_aws.py new file mode 100644 index 000000000..11ee846db --- /dev/null +++ b/dvc/cloud/base_aws.py @@ -0,0 +1,46 @@ +import sys, os, time, configparser, argparse, boto.ec2 + + +class AwsBase: + DEFAULT_REGION = "us-east-1" + DEFAULT_ZONE = "us-east-1a" + + INSTANCE_STATE_TAG = 'oneboxml-is-active' + VOLUME_TAG = 'oneboxml-volume' + + def __init__(self, homedir, conf_file): + self._homedir = homedir + + config = configparser.ConfigParser() + config.read(conf_file) + + self._aws_conf = config['AWS'] + self._args = self.parse_args() + + self._conn = boto.ec2.connect_to_region( + self._args.region, + aws_access_key_id = self._aws_conf.get("AccessKeyID"), + aws_secret_access_key = self._aws_conf.get("SecretAccessKey")) + pass + + def get_command(self): + return (self._args.command, self._args.param) + + def parse_args(self): + parser = argparse.ArgumentParser() + parser.add_argument('command') + parser.add_argument('param', nargs = '?', default = '') + parser.add_argument('--region', metavar = '', + default = self._aws_conf.get("Region", self.DEFAULT_REGION), + help = 'AWS region: us-east-2, ...') + parser.add_argument('--zone', metavar = '', + default = self._aws_conf.get("Zone", self.DEFAULT_ZONE), + help = 'Availability Zone. Examples: us-east-1a, us-east-1b, ...') + self.extend_args(parser) + + args = parser.parse_args() + return args + + def extend_args(self, parser): + raise NotImplementedError() + diff --git a/dvc/cloud/instance.py b/dvc/cloud/instance.py new file mode 100644 index 000000000..77c51bf38 --- /dev/null +++ b/dvc/cloud/instance.py @@ -0,0 +1,32 @@ + + +class Instance(object): + def __init__(self, name, cloud, parsed_args, conf_parser): + self._name = name + self._cloud = cloud + + self._type = parsed_args.type or conf_parser._config[cloud]['Type'] + self._image = parsed_args.image or conf_parser._config[cloud]['Image'] + + self._spot_price = parsed_args.spot_price or conf_parser._config[cloud]['SpotPrice'] + self._spot_timeout = parsed_args.spot_timeout or conf_parser._config[cloud]['SpotTimeout'] + + self._keypair_name = parsed_args.keypair_name or conf_parser._config[cloud]['KeyPairName'] + self._keypair_dir = parsed_args.keypair_dir or conf_parser._config[cloud]['KeyPairDir'] + self._security_group = parsed_args.security_group or conf_parser._config[cloud]['SecurityGroup'] + + self._region = parsed_args.region or conf_parser._config[cloud]['Region'] + self._zone = parsed_args.zone or conf_parser._config[cloud]['Zone'] + self._subnet_id = parsed_args.subnet_id or conf_parser._config[cloud]['SubnetId'] + + self._storage = parsed_args.storage or conf_parser._config[cloud]['Storage'] + + self._monitoring = parsed_args.monitoring or conf_parser._config[cloud]['Monitoring'] + self._ebs_optimized = parsed_args.ebs_optimized or conf_parser._config[cloud]['EbsOptimized'] + self._disks_to_raid0 = parsed_args.disks_to_ride0 or conf_parser._config[cloud]['DisksToRAID0'] + pass + + def toBool(self, value): + if value == None: + return False + return value.lower() in ("yes", "true", "t", "1") \ No newline at end of file diff --git a/dvc/cloud/instance_aws.py b/dvc/cloud/instance_aws.py new file mode 100644 index 000000000..44e33195e --- /dev/null +++ b/dvc/cloud/instance_aws.py @@ -0,0 +1,369 @@ +import sys, os, time, configparser, argparse, boto.ec2 +from base_aws import AwsBase + +from dvc.cloud.instance import Instance + + +class InstanceAws(Instance): + TERMINATED_STATE = 'terminated' + INSTANCE_FORMAT = "{:8s}{:12s}{:12s}{:12s}{:16s}{:16s}{:16s}" + VOLUME_FORMAT = "{:20s}{:16s}{:12s}{:12s}{:12s}{:12s}{:16s}" + + def __init__(self, homedir, conf_file): + super(InstanceAws).__init__(homedir, conf_file) + + self._conn = boto.ec2.connect_to_region( + self._region, + aws_access_key_id = self._aws_conf.get("AccessKeyID"), + aws_secret_access_key = self._aws_conf.get("SecretAccessKey")) + + def create(self): + instance = None + + if not self._spot_price: + reserv = self._conn.run_instances(self._image, + key_name=self.get_key_name(), + instance_type=self._type, + security_groups=self.get_security_groups(), + monitoring_enabled=self.toBool(self._monitoring), + #subnet_id = self._args.subnet_id, + placement=self._zone, + ebs_optimized=self.toBool(self._ebs_optimized)) + instance = reserv.instances[0] + else: + instance = self.create_spot_instance() + + return instance + + def find_volume(self): + (oneboxml_volume, rest_volumes) = self.all_volumes() + + for volume in oneboxml_volume: + if volume.tags[self.VOLUME_TAG] == self._args.storage: + return volume + raise ValueError("Cannot find OneBoxML storage volume '%s'. Verify that the volume was created." + % self._args.storage) + + def create_spot_instance(self): + # Create spot instance + req = self._conn.request_spot_instances(price = self._args.spot_price, + image_id = self._args.image, + key_name = self.get_key_name(), + instance_type = self._args.instance_type, + security_groups = self.get_security_groups(), + monitoring_enabled = self.toBool(self._args.monitoring), + #subnet_id = self._args.subnet_id, + placement = self._args.zone, + ebs_optimized = self.toBool(self._args.ebs_optimized)) + job_instance_id = None + sec = 0 + sys.stdout.write("Waiting for a spot instance. Request %s." % req[0].id) + while job_instance_id == None and sec < self._args.spot_timeout: + sys.stdout.write(".") + sys.stdout.flush() + + job_sir_id = req[0].id + reqs = self._conn.get_all_spot_instance_requests() + for sir in reqs: + if sir.id == job_sir_id: + job_instance_id = sir.instance_id + break + time.sleep(1) + sec += 1 + sys.stdout.write("\n") + + if not job_instance_id: + self._conn.cancel_spot_instance_requests(req[0].id) + msg = "the request was canceled" + raise Exception("Unable to obtain %s spot instance in region %s for price $%s: %s" % + (self._args.instance_type, self._args.region, self._args.spot_price, msg)) + + logger.log("%s spot instance was created: %s" % (self._args.instance_type, job_instance_id)) + reservations = self._conn.get_all_instances(instance_ids = job_instance_id) + instance = reservations[0].instances[0] + return instance + + def run_instance(self): + if self._args.image == None or self._args.image == '': + raise Exception("Cannot run EC2 instance: image (AMI) is not defined") + + instance = self.create_instance() + + # Remove oneboxml active tag. + active_filter = {'tag-key': self.INSTANCE_STATE_TAG, 'tag-value': 'True'} + active_reserv = self._conn.get_all_instances(filters = active_filter) + active_instances = [i for r in active_reserv for i in r.instances] + + if len(active_instances) > 0: + #active_insts = active_reserv.instances + if len(active_instances) > 1: + logger.error("EC2 instances consistency error - more than one active EC2 instance") + for inst in active_instances: + inst.remove_tag(self.INSTANCE_STATE_TAG, 'True') + inst.add_tag(self.INSTANCE_STATE_TAG, 'False') + if inst.state != self.TERMINATED_STATE: + logger.log("%s instance %s is not longer active" % (inst.instance_type, inst.id)) + + # Assign the created instace as active. + instance.add_tag(self.INSTANCE_STATE_TAG, 'True') + logger.log("New %s instance %s was selected as active"%(instance.instance_type, instance.id)) + + sys.stdout.write("Waiting for a running status") + while instance.state != 'running': + sys.stdout.write(".") + sys.stdout.flush() + time.sleep(1) + instance.update() + sys.stdout.write("\n") + + self._conn.attach_volume(self.find_volume().id, instance.id, "/dev/sdx") + pass + + def get_security_groups(self, ssh_port = 22): + group_name = self._security_group + + # Check if the group exists and create one if does not. + try: + group = self._conn.get_all_security_groups(groupnames=[group_name])[0] + except self._conn.ResponseError as e: + if e.code == 'InvalidGroup.NotFound': + logger.log('AWS Security Group %s does not exist: creating the group' % group_name) + group = self._conn.create_security_group(group_name, 'OneBoxML group with SSH access') + else: + raise + + # Enable SSH access. + try: + group.authorize('tcp', ssh_port, ssh_port, '0.0.0.0/0') + except self._conn.ResponseError as e: + if e.code != 'InvalidPermission.Duplicate': + raise + + return [group_name] + + def get_key_name(self): + if not self._keypair_name: + raise Exception("AWS keypair cannot be created: KeyName is not specified in AWS section in the config file") + if not self._keypair_dir: + raise Exception("AWS keypair cannot be created: KeyDir is not specified in AWS section in the config file") + + # Check if the key exists and create one if does not. + try: + key = self._conn.get_all_key_pairs(keynames = [self._keypair_name])[0] + except self._conn.ResponseError as e: + if e.code == 'InvalidKeyPair.NotFound': + logger.log('AWS key %s does not exist: creating the key' % self._keypair_name) + # Create an SSH key to use when logging into instances. + key = self._conn.create_key_pair(self._keypair_name) + logger.log("AWS key was created: " + self._keypair_name) + + # Expand key dir. + key_dir = os.path.expandvars(self._keypair_dir) + if not os.path.isdir(self._keypair_dir): + os.mkdir(self._keypair_dir, 0o700) + + # Private key has to be stored locally. + key_file = os.path.join(key_dir, self._keypair_name + '.pem') + #key.save(key_file) # doesn't work in python3 + fp = open(key_file, 'w') + fp.write(key.material) + fp.close() + os.chmod(key_file, 0o600) + logger.log("AWS private key file was saved: " + key_file) + else: + raise + + return self._keypair_name + + def all_instances(self, terminated = False): + oneboxml_active = [] + oneboxml_not_active = [] + rest_inst = [] + + reserv = self._conn.get_all_instances() + instances = [i for r in reserv for i in r.instances] + for inst in instances: + if terminated == False and inst.state == self.TERMINATED_STATE: + continue + + if self.INSTANCE_STATE_TAG in inst.tags: + if inst.tags[self.INSTANCE_STATE_TAG] == 'True': + oneboxml_active.append(inst) + else: + oneboxml_not_active.append(inst) + else: + rest_inst.append(inst) + + if len(oneboxml_active) > 1: + logger.error("Instance consistancy error - more than one instance is in active state") + if len(oneboxml_active) == 0 and len(oneboxml_not_active) > 0: + logger.error("Instance consistancy error - no active instances") + return (oneboxml_active, oneboxml_not_active, rest_inst) + + def describe_instance(self, inst, volumes, is_active): + ip = inst.ip_address + ip_private = inst.private_ip_address + if not ip: + ip = "" + if not ip_private: + ip_private = "" + + activeFlag = '' + if is_active: + activeFlag = ' ***' + + + volume_id = None + volume_name = '' + bdm = inst.block_device_mapping + for device_type in bdm.values(): + for vol in volumes: + if device_type.volume_id == vol.id: + if self.VOLUME_TAG in vol.tags: + volume_name = vol.tags[self.VOLUME_TAG] + else: + log.error("Instance %s storage volume doesn't have tag", inst.id) + break + + return self.INSTANCE_FORMAT.format(activeFlag, + inst.id[:12], + inst.instance_type[:12], + inst.state[:12], + volume_name, + ip[:16], + ip_private[:16]) + + def describe_instances(self): + (oneboxml_active, oneboxml_not_active, rest_inst) = self.all_instances() + (oneboxml_volumes, rest_volumes) = self.all_volumes() + + logger.log(self.INSTANCE_FORMAT.format("Active", "Id", "Type", "State", "Storage", + "IP public", "IP private")) + + + for inst in oneboxml_active: + logger.log(self.describe_instance(inst, oneboxml_volumes, True)) + for inst in oneboxml_not_active: + logger.log(self.describe_instance(inst, oneboxml_volumes, False)) + + if len(rest_inst) > 0: + logger.warn("%s addition not OneBoxML instances were found: not in the list"\ + % len(rest_inst)) + pass + + def get_instance_id(self, instance): + if instance[:2] != 'i-' and len(instance) == 8: + instance = 'i-' + instance + return instance + + def terminate_instances(self, instance): + instance = self.get_instance_id(instance) + + if not instance: + logger.error("Instance Id is not specified") + return + + (oneboxml_active, oneboxml_not_active, rest_inst) = self.all_instances() + + if instance == 'all': + target_in_active = oneboxml_active + target_in_not_active = oneboxml_not_active + else: + target_in_active = list(filter(lambda inst: inst.id == instance, oneboxml_active)) + target_in_not_active = list(filter(lambda inst: inst.id == instance, oneboxml_not_active)) + + if target_in_not_active: + self._conn.terminate_instances(instance_ids = [inst.id for inst in target_in_not_active]) + + for inst in target_in_active: + inst.remove_tag(self.INSTANCE_STATE_TAG, 'True') + inst.add_tag(self.INSTANCE_STATE_TAG, 'False') + self._conn.terminate_instances(instance_ids = [inst.id]) + + if instance != 'all' and len(target_in_active) > 0 and len(oneboxml_not_active) > 0: + new_active_inst = oneboxml_not_active[0] + new_active_inst.remove_tag(self.INSTANCE_STATE_TAG, 'False') + new_active_inst.add_tag(self.INSTANCE_STATE_TAG, 'True') + randomly = '' + if len(oneboxml_not_active) > 1: + randomly = 'randomly ' + logger.warn("%s instance %s was %sselected as active because of an active instance was terminated" % + (new_active_inst.instance_type, new_active_inst.id, randomly)) + pass + + def set_active_instance(self, instance): + instance = self.get_instance_id(instance) + + if not instance: + logger.error("Instance Id is not specified") + return + + (oneboxml_active, oneboxml_not_active, rest_inst) = self.all_instances() + + target_in_active = list(filter(lambda inst: inst.id == instance, oneboxml_active)) + if len(target_in_active) > 0: + logger.error("The instance is already active") + return + + target_in_not_actives = list(filter(lambda inst: inst.id == instance, oneboxml_not_active)) + if len(target_in_not_actives) == 0: + logger.error("The instance is not running") + return + + if len(target_in_not_actives) > 1: + logger.error("Instances consistancy error: more than one instance with the same id") + return + + target_in_not_actives[0].remove_tag(self.INSTANCE_STATE_TAG, 'False') + target_in_not_actives[0].add_tag(self.INSTANCE_STATE_TAG, 'True') + + for inst in oneboxml_active: + inst.remove_tag(self.INSTANCE_STATE_TAG, 'True') + inst.add_tag(self.INSTANCE_STATE_TAG, 'False') + pass + + def all_volumes(self): + oneboxml_volumes = [] + rest_volumes = [] + volumes = self._conn.get_all_volumes() + for vol in volumes: + if self.VOLUME_TAG in vol.tags: + oneboxml_volumes.append(vol) + else: + rest_volumes.append(vol) + return (oneboxml_volumes, rest_volumes) + + + def describe_storages(self): + logger.log(self.VOLUME_FORMAT.format("Name", "Id", "Size(GB)", + "Status", "Type", "Encrypted", "Zone")) + (oneboxml_volume, rest_volumes) = self.all_volumes() + for vol in oneboxml_volume: + logger.log(self.VOLUME_FORMAT.format(vol.tags[self.VOLUME_TAG], + vol.id, str(vol.size), vol.status, + vol.type, str(vol.encrypted), vol.zone)) + pass + +if __name__ == "__main__": + HOMEDIR_VAR = "ONEBOXML_HOME" + if not (HOMEDIR_VAR in os.environ): + raise Exception(HOMEDIR_VAR + " variable is not defined") + homedir = os.environ[HOMEDIR_VAR] + conf_file = homedir + "/oneboxml.conf" + + logger = Logger() + + instance_tool = InstanceAws(homedir, conf_file) + + (command, param) = instance_tool.get_command() + if command == 'run': + instance_tool.run_instance() + if command == 'describe-instances': + instance_tool.describe_instances() + if command == 'terminate-instances': + instance_tool.terminate_instances(param) + if command == 'set-active-instance': + instance_tool.set_active_instance(param) + + if command == 'describe-storages': + instance_tool.describe_storages() diff --git a/dvc/cloud/instance_manager.py b/dvc/cloud/instance_manager.py new file mode 100644 index 000000000..440bac5d9 --- /dev/null +++ b/dvc/cloud/instance_manager.py @@ -0,0 +1,15 @@ + +from dvc.logger import Logger + + +class InstanceManager(object): + def instances(self): + return [] + + def create(self, name, cloud, parsed_args, conf_parser): + if name in set(map(lambda x: x.name, self.instances())): + Logger.error('Instance with name {} is already exist'.format(name)) + return 1 + + + pass diff --git a/dvc/cloud/storage_aws.py b/dvc/cloud/storage_aws.py new file mode 100644 index 000000000..200bffc98 --- /dev/null +++ b/dvc/cloud/storage_aws.py @@ -0,0 +1,55 @@ +import sys, os, argparse, boto.ec2 +from base_aws import AwsBase #logger, Logger + +class StorageTool(AwsBase): + + def __init__(self, homedir, conf_file): + super().__init__(homedir, conf_file) + + def extend_args(self, parser): + parser.add_argument('--size', metavar = '', type = int, + help = 'Storage volume size in GB. Min=1, Max=16384.') + parser.add_argument('--snapshot-id', metavar = '', + help = 'The snapshot id for the volumne creation.') + parser.add_argument('--type', metavar = '', + help = 'The volume type: standard, io1 - provisioned IOPS, gp2 - general purpose') + parser.add_argument('--iops', metavar = '', + help = 'Number of I\O per second for provisioned IOPS volumes.') + parser.add_argument('--encryption', metavar = '', + help = 'Enable encription for the volume.') + pass + + def create(self, storage_name): + if not self._args.size: + logger.error("Storage volume size is not specified") + return + + try: + vol = self._conn.create_volume(self._args.size, + self._args.zone, + self._args.snapshot_id, + self._args.type, + self._args.iops, + self.toBool(self._args.encryption)) + vol.add_tag(self.VOLUME_TAG, storage_name) + except self._conn.ResponseError as e: + logger.error('cannot create a volume: %s' % e.message) + return + + logger.log("%sGB volume %s was created" % (self._args.size, storage_name)) + pass + +if __name__ == "__main__": + HOMEDIR_VAR = "ONEBOXML_HOME" + if not (HOMEDIR_VAR in os.environ): + raise Exception(HOMEDIR_VAR + " variable is not defined") + homedir = os.environ[HOMEDIR_VAR] + conf_file = homedir + "/oneboxml.conf" + + logger = Logger() + + storage_tool = StorageTool(homedir, conf_file) + + (command, param) = storage_tool.get_command() + if command == 'create': + storage_tool.create(param) diff --git a/dvc/command/init.py b/dvc/command/init.py index f7560426b..265a3e000 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from dvc.command.base import CmdBase from dvc.logger import Logger @@ -28,26 +29,29 @@ StoragePath = LogLevel = Info [AWS] -StoragePath = dvc/tutorial - CredentialPath = -Region = us-east-1 -Zone = us-east-1a - +# Default settings for AWS instances: +Type = t2.nano Image = ami-2d39803a -InstanceType = t2.nano - SpotPrice = SpotTimeout = 300 +KeyPairName = dvc-key +KeyPairDir = ~/.ssh +SecurityGroup = dvc-sg + +Region = us-east-1 +Zone = us-east-1a +SubnetId = + Storage = my-100gb-drive-io +#StoragePath = dvc/tutorial -KeyDir = ~/.ssh -KeyName = dvc-key - -SecurityGroup = dvc-group +Monitoring = false +EbsOptimized = false +AllDisksAsRAID0 = false [GCP] StoragePath = @@ -61,10 +65,10 @@ ProjectName = super(CmdInit, self).__init__(settings) def get_not_existing_path(self, dir): - path = os.path.join(self.git.git_dir, dir) - if os.path.exists(path): - raise InitError('Path "{}" already exist'.format(path)) - return os.path.basename(path), path + path = Path(os.path.join(self.git.git_dir, dir)) + if path.exists(): + raise InitError('Path "{}" already exist'.format(path.name)) + return path def get_not_existing_conf_file_name(self): file_name = os.path.join(self.git.git_dir, Config.CONFIG) @@ -82,10 +86,10 @@ ProjectName = )) return 1 - data_dir, data_dir_path = self.get_not_existing_path(self.parsed_args.data_dir) - cache_dir, cache_dir_path = self.get_not_existing_path(self.parsed_args.cache_dir) - state_dir, state_dir_path = self.get_not_existing_path(self.parsed_args.state_dir) - target_file, target_file_path = self.get_not_existing_path(self.parsed_args.target_file) + data_dir_path = self.get_not_existing_path(self.parsed_args.data_dir) + cache_dir_path = self.get_not_existing_path(self.parsed_args.cache_dir) + state_dir_path = self.get_not_existing_path(self.parsed_args.state_dir) + target_file_path = self.get_not_existing_path(self.parsed_args.target_file) self.settings.config.set(self.parsed_args.data_dir, self.parsed_args.cache_dir, @@ -94,34 +98,34 @@ ProjectName = conf_file_name = self.get_not_existing_conf_file_name() - os.mkdir(data_dir_path) - os.mkdir(cache_dir_path) - os.mkdir(state_dir_path) - open(target_file_path, 'w').close() + data_dir_path.mkdir() + cache_dir_path.mkdir() + state_dir_path.mkdir() + target_file_path.touch() Logger.info('Directories {}/, {}/, {}/ and target file {} were created'.format( - data_dir, - cache_dir, - state_dir, - target_file)) + data_dir_path.name, + cache_dir_path.name, + state_dir_path.name, + target_file_path.name)) self.create_empty_file() conf_file = open(conf_file_name, 'wt') - conf_file.write(self.CONFIG_TEMPLATE.format(data_dir, - cache_dir, - state_dir, - target_file)) + conf_file.write(self.CONFIG_TEMPLATE.format(data_dir_path.name, + cache_dir_path.name, + state_dir_path.name, + target_file_path.name)) conf_file.close() message = 'DVC init. data dir {}, cache dir {}, state dir {}, '.format( - data_dir, - cache_dir, - state_dir + data_dir_path.name, + cache_dir_path.name, + state_dir_path.name ) if self.commit_if_needed(message) == 1: return 1 - self.modify_gitignore(cache_dir) + self.modify_gitignore(cache_dir_path.name) return self.commit_if_needed('DVC init. Commit .gitignore file') def create_empty_file(self): diff --git a/dvc/command/instance_create.py b/dvc/command/instance_create.py new file mode 100644 index 000000000..35237b152 --- /dev/null +++ b/dvc/command/instance_create.py @@ -0,0 +1,29 @@ +from dvc.cloud.instance_manager import InstanceManager +from dvc.command.base import CmdBase, DvcLock +from dvc.exceptions import DvcException +from dvc.logger import Logger + + +class CmdInstanceCreate(CmdBase): + def __init__(self, settings): + super(CmdInstanceCreate, self).__init__(settings) + + def run(self): + with DvcLock(self.is_locker, self.git): + name = self.parsed_args.name + cloud = self.parsed_args.cloud or self.settings.config.cloud + + # print('NAME: {}'.format(name)) + # print('CLOUD: {}'.format(cloud)) + + if not name: + Logger.error('Instance name is not defined') + return 1 + + try: + InstanceManager().create(name, cloud, self.parsed_args, self.settings.config) + except DvcException as ex: + Logger.error('Instance creation error: {}'.format(ex)) + return 1 + + return 0 diff --git a/dvc/config.py b/dvc/config.py index ccfcfb7c0..a008dbae2 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -13,18 +13,23 @@ class ConfigError(DvcException): class ConfigI(object): TARGET_FILE_DEFAULT = '.target' - def __init__(self, data_dir=None, cache_dir=None, state_dir=None, target_file=None): + def __init__(self, data_dir=None, cache_dir=None, state_dir=None, target_file=None, + cloud=None, conf_parser=None): self._data_dir = None self._cache_dir = None self._state_dir = None self._target_file = None - self.set(data_dir, cache_dir, state_dir, target_file) + self._conf_parser = None + self._cloud = None + self.set(data_dir, cache_dir, state_dir, target_file, cloud, conf_parser) - def set(self, data_dir, cache_dir, state_dir, target_file): + def set(self, data_dir, cache_dir, state_dir, target_file, cloud, conf_parser): self._data_dir = data_dir self._cache_dir = cache_dir self._state_dir = state_dir self._target_file = target_file + self._cloud = cloud + self._conf_parser = conf_parser @property def data_dir(self): @@ -42,6 +47,14 @@ class ConfigI(object): def target_file(self): return self._target_file + @property + def cloud(self): + return self._cloud + + @property + def conf_parser(self): + return self._conf_parser + class Config(ConfigI): CONFIG = 'dvc.conf' @@ -68,7 +81,9 @@ class Config(ConfigI): super(Config, self).__init__(self._config['Global']['DataDir'], self._config['Global']['CacheDir'], self._config['Global']['StateDir'], - self._config['Global'].get('TargetFile', Config.TARGET_FILE_DEFAULT)) + self._config['Global'].get('TargetFile', Config.TARGET_FILE_DEFAULT), + self._config['Global']['Cloud'], + self._config) pass @property