mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-09 23:10:24 +01:00
Cloud basic 2: init
This commit is contained in:
Dmitry Petrov
+62
-4
@@ -1,9 +1,6 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import configparser
|
||||
from multiprocessing import cpu_count
|
||||
|
||||
from dvc.command.init import CmdInit
|
||||
@@ -15,7 +12,7 @@ from dvc.command.lock import CmdLock
|
||||
from dvc.command.gc import CmdGC
|
||||
from dvc.command.import_file import CmdImportFile
|
||||
from dvc.command.target import CmdTarget
|
||||
from dvc.command.test import CmdTest
|
||||
from dvc.command.instance_create import CmdInstanceCreate
|
||||
from dvc.config import Config
|
||||
from dvc import VERSION
|
||||
|
||||
@@ -266,6 +263,67 @@ def parse_args(argv=None):
|
||||
help='Reset target.')
|
||||
target_parser.set_defaults(func=CmdTarget)
|
||||
|
||||
# Instance create
|
||||
instance_create_parser = subparsers.add_parser('instance-create',
|
||||
parents=[parent_parser],
|
||||
help='Create cloud instance')
|
||||
instance_create_parser.add_argument('name',
|
||||
# metavar='',
|
||||
nargs='?',
|
||||
help='Instance name.')
|
||||
instance_create_parser.add_argument('-c',
|
||||
'--cloud',
|
||||
# metavar='',
|
||||
help='Cloud: AWS, GCP.')
|
||||
instance_create_parser.add_argument('-t',
|
||||
'--type',
|
||||
# metavar='',
|
||||
help='Instance type.')
|
||||
instance_create_parser.add_argument('-i',
|
||||
'--image',
|
||||
# metavar='',
|
||||
help='Instance image.')
|
||||
instance_create_parser.add_argument('--spot-price',
|
||||
metavar='PRICE',
|
||||
help='Spot instance price in $ i.e. 1.54')
|
||||
instance_create_parser.add_argument('--spot-timeout',
|
||||
metavar='TIMEOUT',
|
||||
type=int,
|
||||
help='Spot instances waiting timeout in seconds.')
|
||||
instance_create_parser.add_argument('--keypair-name',
|
||||
metavar='KEYPAIR',
|
||||
help='The name of key pair for instance launch')
|
||||
instance_create_parser.add_argument('--keypair-dir',
|
||||
metavar='DIR',
|
||||
help='The directory of key pairs')
|
||||
instance_create_parser.add_argument('--security-group',
|
||||
metavar='GROUP',
|
||||
help='Security group')
|
||||
instance_create_parser.add_argument('--region',
|
||||
help='Region')
|
||||
instance_create_parser.add_argument('--zone',
|
||||
help='Zone')
|
||||
instance_create_parser.add_argument('--subnet-id',
|
||||
metavar='SUBNET',
|
||||
help='Subnet ID')
|
||||
instance_create_parser.add_argument('--storage',
|
||||
# metavar='',
|
||||
help='The name of attachable storage volume.')
|
||||
# WHERE EBS IS?
|
||||
instance_create_parser.add_argument('--monitoring',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Enable EC2 instance monitoring')
|
||||
instance_create_parser.add_argument('--ebs-optimized',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Enable EBS I\O optimization')
|
||||
instance_create_parser.add_argument('--disks-to-ride0',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Detect all ephemeral disks and stripe together in raid-0')
|
||||
instance_create_parser.set_defaults(func=CmdInstanceCreate)
|
||||
|
||||
if isinstance(argv, str):
|
||||
argv = argv.split()
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import sys, os, time, configparser, argparse, boto.ec2
|
||||
|
||||
|
||||
class AwsBase:
|
||||
DEFAULT_REGION = "us-east-1"
|
||||
DEFAULT_ZONE = "us-east-1a"
|
||||
|
||||
INSTANCE_STATE_TAG = 'oneboxml-is-active'
|
||||
VOLUME_TAG = 'oneboxml-volume'
|
||||
|
||||
def __init__(self, homedir, conf_file):
|
||||
self._homedir = homedir
|
||||
|
||||
config = configparser.ConfigParser()
|
||||
config.read(conf_file)
|
||||
|
||||
self._aws_conf = config['AWS']
|
||||
self._args = self.parse_args()
|
||||
|
||||
self._conn = boto.ec2.connect_to_region(
|
||||
self._args.region,
|
||||
aws_access_key_id = self._aws_conf.get("AccessKeyID"),
|
||||
aws_secret_access_key = self._aws_conf.get("SecretAccessKey"))
|
||||
pass
|
||||
|
||||
def get_command(self):
|
||||
return (self._args.command, self._args.param)
|
||||
|
||||
def parse_args(self):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('command')
|
||||
parser.add_argument('param', nargs = '?', default = '')
|
||||
parser.add_argument('--region', metavar = '',
|
||||
default = self._aws_conf.get("Region", self.DEFAULT_REGION),
|
||||
help = 'AWS region: us-east-2, ...')
|
||||
parser.add_argument('--zone', metavar = '',
|
||||
default = self._aws_conf.get("Zone", self.DEFAULT_ZONE),
|
||||
help = 'Availability Zone. Examples: us-east-1a, us-east-1b, ...')
|
||||
self.extend_args(parser)
|
||||
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
def extend_args(self, parser):
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
|
||||
|
||||
class Instance(object):
|
||||
def __init__(self, name, cloud, parsed_args, conf_parser):
|
||||
self._name = name
|
||||
self._cloud = cloud
|
||||
|
||||
self._type = parsed_args.type or conf_parser._config[cloud]['Type']
|
||||
self._image = parsed_args.image or conf_parser._config[cloud]['Image']
|
||||
|
||||
self._spot_price = parsed_args.spot_price or conf_parser._config[cloud]['SpotPrice']
|
||||
self._spot_timeout = parsed_args.spot_timeout or conf_parser._config[cloud]['SpotTimeout']
|
||||
|
||||
self._keypair_name = parsed_args.keypair_name or conf_parser._config[cloud]['KeyPairName']
|
||||
self._keypair_dir = parsed_args.keypair_dir or conf_parser._config[cloud]['KeyPairDir']
|
||||
self._security_group = parsed_args.security_group or conf_parser._config[cloud]['SecurityGroup']
|
||||
|
||||
self._region = parsed_args.region or conf_parser._config[cloud]['Region']
|
||||
self._zone = parsed_args.zone or conf_parser._config[cloud]['Zone']
|
||||
self._subnet_id = parsed_args.subnet_id or conf_parser._config[cloud]['SubnetId']
|
||||
|
||||
self._storage = parsed_args.storage or conf_parser._config[cloud]['Storage']
|
||||
|
||||
self._monitoring = parsed_args.monitoring or conf_parser._config[cloud]['Monitoring']
|
||||
self._ebs_optimized = parsed_args.ebs_optimized or conf_parser._config[cloud]['EbsOptimized']
|
||||
self._disks_to_raid0 = parsed_args.disks_to_ride0 or conf_parser._config[cloud]['DisksToRAID0']
|
||||
pass
|
||||
|
||||
def toBool(self, value):
|
||||
if value == None:
|
||||
return False
|
||||
return value.lower() in ("yes", "true", "t", "1")
|
||||
@@ -0,0 +1,369 @@
|
||||
import sys, os, time, configparser, argparse, boto.ec2
|
||||
from base_aws import AwsBase
|
||||
|
||||
from dvc.cloud.instance import Instance
|
||||
|
||||
|
||||
class InstanceAws(Instance):
|
||||
TERMINATED_STATE = 'terminated'
|
||||
INSTANCE_FORMAT = "{:8s}{:12s}{:12s}{:12s}{:16s}{:16s}{:16s}"
|
||||
VOLUME_FORMAT = "{:20s}{:16s}{:12s}{:12s}{:12s}{:12s}{:16s}"
|
||||
|
||||
def __init__(self, homedir, conf_file):
|
||||
super(InstanceAws).__init__(homedir, conf_file)
|
||||
|
||||
self._conn = boto.ec2.connect_to_region(
|
||||
self._region,
|
||||
aws_access_key_id = self._aws_conf.get("AccessKeyID"),
|
||||
aws_secret_access_key = self._aws_conf.get("SecretAccessKey"))
|
||||
|
||||
def create(self):
|
||||
instance = None
|
||||
|
||||
if not self._spot_price:
|
||||
reserv = self._conn.run_instances(self._image,
|
||||
key_name=self.get_key_name(),
|
||||
instance_type=self._type,
|
||||
security_groups=self.get_security_groups(),
|
||||
monitoring_enabled=self.toBool(self._monitoring),
|
||||
#subnet_id = self._args.subnet_id,
|
||||
placement=self._zone,
|
||||
ebs_optimized=self.toBool(self._ebs_optimized))
|
||||
instance = reserv.instances[0]
|
||||
else:
|
||||
instance = self.create_spot_instance()
|
||||
|
||||
return instance
|
||||
|
||||
def find_volume(self):
|
||||
(oneboxml_volume, rest_volumes) = self.all_volumes()
|
||||
|
||||
for volume in oneboxml_volume:
|
||||
if volume.tags[self.VOLUME_TAG] == self._args.storage:
|
||||
return volume
|
||||
raise ValueError("Cannot find OneBoxML storage volume '%s'. Verify that the volume was created."
|
||||
% self._args.storage)
|
||||
|
||||
def create_spot_instance(self):
|
||||
# Create spot instance
|
||||
req = self._conn.request_spot_instances(price = self._args.spot_price,
|
||||
image_id = self._args.image,
|
||||
key_name = self.get_key_name(),
|
||||
instance_type = self._args.instance_type,
|
||||
security_groups = self.get_security_groups(),
|
||||
monitoring_enabled = self.toBool(self._args.monitoring),
|
||||
#subnet_id = self._args.subnet_id,
|
||||
placement = self._args.zone,
|
||||
ebs_optimized = self.toBool(self._args.ebs_optimized))
|
||||
job_instance_id = None
|
||||
sec = 0
|
||||
sys.stdout.write("Waiting for a spot instance. Request %s." % req[0].id)
|
||||
while job_instance_id == None and sec < self._args.spot_timeout:
|
||||
sys.stdout.write(".")
|
||||
sys.stdout.flush()
|
||||
|
||||
job_sir_id = req[0].id
|
||||
reqs = self._conn.get_all_spot_instance_requests()
|
||||
for sir in reqs:
|
||||
if sir.id == job_sir_id:
|
||||
job_instance_id = sir.instance_id
|
||||
break
|
||||
time.sleep(1)
|
||||
sec += 1
|
||||
sys.stdout.write("\n")
|
||||
|
||||
if not job_instance_id:
|
||||
self._conn.cancel_spot_instance_requests(req[0].id)
|
||||
msg = "the request was canceled"
|
||||
raise Exception("Unable to obtain %s spot instance in region %s for price $%s: %s" %
|
||||
(self._args.instance_type, self._args.region, self._args.spot_price, msg))
|
||||
|
||||
logger.log("%s spot instance was created: %s" % (self._args.instance_type, job_instance_id))
|
||||
reservations = self._conn.get_all_instances(instance_ids = job_instance_id)
|
||||
instance = reservations[0].instances[0]
|
||||
return instance
|
||||
|
||||
def run_instance(self):
|
||||
if self._args.image == None or self._args.image == '':
|
||||
raise Exception("Cannot run EC2 instance: image (AMI) is not defined")
|
||||
|
||||
instance = self.create_instance()
|
||||
|
||||
# Remove oneboxml active tag.
|
||||
active_filter = {'tag-key': self.INSTANCE_STATE_TAG, 'tag-value': 'True'}
|
||||
active_reserv = self._conn.get_all_instances(filters = active_filter)
|
||||
active_instances = [i for r in active_reserv for i in r.instances]
|
||||
|
||||
if len(active_instances) > 0:
|
||||
#active_insts = active_reserv.instances
|
||||
if len(active_instances) > 1:
|
||||
logger.error("EC2 instances consistency error - more than one active EC2 instance")
|
||||
for inst in active_instances:
|
||||
inst.remove_tag(self.INSTANCE_STATE_TAG, 'True')
|
||||
inst.add_tag(self.INSTANCE_STATE_TAG, 'False')
|
||||
if inst.state != self.TERMINATED_STATE:
|
||||
logger.log("%s instance %s is not longer active" % (inst.instance_type, inst.id))
|
||||
|
||||
# Assign the created instace as active.
|
||||
instance.add_tag(self.INSTANCE_STATE_TAG, 'True')
|
||||
logger.log("New %s instance %s was selected as active"%(instance.instance_type, instance.id))
|
||||
|
||||
sys.stdout.write("Waiting for a running status")
|
||||
while instance.state != 'running':
|
||||
sys.stdout.write(".")
|
||||
sys.stdout.flush()
|
||||
time.sleep(1)
|
||||
instance.update()
|
||||
sys.stdout.write("\n")
|
||||
|
||||
self._conn.attach_volume(self.find_volume().id, instance.id, "/dev/sdx")
|
||||
pass
|
||||
|
||||
def get_security_groups(self, ssh_port = 22):
|
||||
group_name = self._security_group
|
||||
|
||||
# Check if the group exists and create one if does not.
|
||||
try:
|
||||
group = self._conn.get_all_security_groups(groupnames=[group_name])[0]
|
||||
except self._conn.ResponseError as e:
|
||||
if e.code == 'InvalidGroup.NotFound':
|
||||
logger.log('AWS Security Group %s does not exist: creating the group' % group_name)
|
||||
group = self._conn.create_security_group(group_name, 'OneBoxML group with SSH access')
|
||||
else:
|
||||
raise
|
||||
|
||||
# Enable SSH access.
|
||||
try:
|
||||
group.authorize('tcp', ssh_port, ssh_port, '0.0.0.0/0')
|
||||
except self._conn.ResponseError as e:
|
||||
if e.code != 'InvalidPermission.Duplicate':
|
||||
raise
|
||||
|
||||
return [group_name]
|
||||
|
||||
def get_key_name(self):
|
||||
if not self._keypair_name:
|
||||
raise Exception("AWS keypair cannot be created: KeyName is not specified in AWS section in the config file")
|
||||
if not self._keypair_dir:
|
||||
raise Exception("AWS keypair cannot be created: KeyDir is not specified in AWS section in the config file")
|
||||
|
||||
# Check if the key exists and create one if does not.
|
||||
try:
|
||||
key = self._conn.get_all_key_pairs(keynames = [self._keypair_name])[0]
|
||||
except self._conn.ResponseError as e:
|
||||
if e.code == 'InvalidKeyPair.NotFound':
|
||||
logger.log('AWS key %s does not exist: creating the key' % self._keypair_name)
|
||||
# Create an SSH key to use when logging into instances.
|
||||
key = self._conn.create_key_pair(self._keypair_name)
|
||||
logger.log("AWS key was created: " + self._keypair_name)
|
||||
|
||||
# Expand key dir.
|
||||
key_dir = os.path.expandvars(self._keypair_dir)
|
||||
if not os.path.isdir(self._keypair_dir):
|
||||
os.mkdir(self._keypair_dir, 0o700)
|
||||
|
||||
# Private key has to be stored locally.
|
||||
key_file = os.path.join(key_dir, self._keypair_name + '.pem')
|
||||
#key.save(key_file) # doesn't work in python3
|
||||
fp = open(key_file, 'w')
|
||||
fp.write(key.material)
|
||||
fp.close()
|
||||
os.chmod(key_file, 0o600)
|
||||
logger.log("AWS private key file was saved: " + key_file)
|
||||
else:
|
||||
raise
|
||||
|
||||
return self._keypair_name
|
||||
|
||||
def all_instances(self, terminated = False):
|
||||
oneboxml_active = []
|
||||
oneboxml_not_active = []
|
||||
rest_inst = []
|
||||
|
||||
reserv = self._conn.get_all_instances()
|
||||
instances = [i for r in reserv for i in r.instances]
|
||||
for inst in instances:
|
||||
if terminated == False and inst.state == self.TERMINATED_STATE:
|
||||
continue
|
||||
|
||||
if self.INSTANCE_STATE_TAG in inst.tags:
|
||||
if inst.tags[self.INSTANCE_STATE_TAG] == 'True':
|
||||
oneboxml_active.append(inst)
|
||||
else:
|
||||
oneboxml_not_active.append(inst)
|
||||
else:
|
||||
rest_inst.append(inst)
|
||||
|
||||
if len(oneboxml_active) > 1:
|
||||
logger.error("Instance consistancy error - more than one instance is in active state")
|
||||
if len(oneboxml_active) == 0 and len(oneboxml_not_active) > 0:
|
||||
logger.error("Instance consistancy error - no active instances")
|
||||
return (oneboxml_active, oneboxml_not_active, rest_inst)
|
||||
|
||||
def describe_instance(self, inst, volumes, is_active):
|
||||
ip = inst.ip_address
|
||||
ip_private = inst.private_ip_address
|
||||
if not ip:
|
||||
ip = ""
|
||||
if not ip_private:
|
||||
ip_private = ""
|
||||
|
||||
activeFlag = ''
|
||||
if is_active:
|
||||
activeFlag = ' ***'
|
||||
|
||||
|
||||
volume_id = None
|
||||
volume_name = ''
|
||||
bdm = inst.block_device_mapping
|
||||
for device_type in bdm.values():
|
||||
for vol in volumes:
|
||||
if device_type.volume_id == vol.id:
|
||||
if self.VOLUME_TAG in vol.tags:
|
||||
volume_name = vol.tags[self.VOLUME_TAG]
|
||||
else:
|
||||
log.error("Instance %s storage volume doesn't have tag", inst.id)
|
||||
break
|
||||
|
||||
return self.INSTANCE_FORMAT.format(activeFlag,
|
||||
inst.id[:12],
|
||||
inst.instance_type[:12],
|
||||
inst.state[:12],
|
||||
volume_name,
|
||||
ip[:16],
|
||||
ip_private[:16])
|
||||
|
||||
def describe_instances(self):
|
||||
(oneboxml_active, oneboxml_not_active, rest_inst) = self.all_instances()
|
||||
(oneboxml_volumes, rest_volumes) = self.all_volumes()
|
||||
|
||||
logger.log(self.INSTANCE_FORMAT.format("Active", "Id", "Type", "State", "Storage",
|
||||
"IP public", "IP private"))
|
||||
|
||||
|
||||
for inst in oneboxml_active:
|
||||
logger.log(self.describe_instance(inst, oneboxml_volumes, True))
|
||||
for inst in oneboxml_not_active:
|
||||
logger.log(self.describe_instance(inst, oneboxml_volumes, False))
|
||||
|
||||
if len(rest_inst) > 0:
|
||||
logger.warn("%s addition not OneBoxML instances were found: not in the list"\
|
||||
% len(rest_inst))
|
||||
pass
|
||||
|
||||
def get_instance_id(self, instance):
|
||||
if instance[:2] != 'i-' and len(instance) == 8:
|
||||
instance = 'i-' + instance
|
||||
return instance
|
||||
|
||||
def terminate_instances(self, instance):
|
||||
instance = self.get_instance_id(instance)
|
||||
|
||||
if not instance:
|
||||
logger.error("Instance Id is not specified")
|
||||
return
|
||||
|
||||
(oneboxml_active, oneboxml_not_active, rest_inst) = self.all_instances()
|
||||
|
||||
if instance == 'all':
|
||||
target_in_active = oneboxml_active
|
||||
target_in_not_active = oneboxml_not_active
|
||||
else:
|
||||
target_in_active = list(filter(lambda inst: inst.id == instance, oneboxml_active))
|
||||
target_in_not_active = list(filter(lambda inst: inst.id == instance, oneboxml_not_active))
|
||||
|
||||
if target_in_not_active:
|
||||
self._conn.terminate_instances(instance_ids = [inst.id for inst in target_in_not_active])
|
||||
|
||||
for inst in target_in_active:
|
||||
inst.remove_tag(self.INSTANCE_STATE_TAG, 'True')
|
||||
inst.add_tag(self.INSTANCE_STATE_TAG, 'False')
|
||||
self._conn.terminate_instances(instance_ids = [inst.id])
|
||||
|
||||
if instance != 'all' and len(target_in_active) > 0 and len(oneboxml_not_active) > 0:
|
||||
new_active_inst = oneboxml_not_active[0]
|
||||
new_active_inst.remove_tag(self.INSTANCE_STATE_TAG, 'False')
|
||||
new_active_inst.add_tag(self.INSTANCE_STATE_TAG, 'True')
|
||||
randomly = ''
|
||||
if len(oneboxml_not_active) > 1:
|
||||
randomly = 'randomly '
|
||||
logger.warn("%s instance %s was %sselected as active because of an active instance was terminated" %
|
||||
(new_active_inst.instance_type, new_active_inst.id, randomly))
|
||||
pass
|
||||
|
||||
def set_active_instance(self, instance):
|
||||
instance = self.get_instance_id(instance)
|
||||
|
||||
if not instance:
|
||||
logger.error("Instance Id is not specified")
|
||||
return
|
||||
|
||||
(oneboxml_active, oneboxml_not_active, rest_inst) = self.all_instances()
|
||||
|
||||
target_in_active = list(filter(lambda inst: inst.id == instance, oneboxml_active))
|
||||
if len(target_in_active) > 0:
|
||||
logger.error("The instance is already active")
|
||||
return
|
||||
|
||||
target_in_not_actives = list(filter(lambda inst: inst.id == instance, oneboxml_not_active))
|
||||
if len(target_in_not_actives) == 0:
|
||||
logger.error("The instance is not running")
|
||||
return
|
||||
|
||||
if len(target_in_not_actives) > 1:
|
||||
logger.error("Instances consistancy error: more than one instance with the same id")
|
||||
return
|
||||
|
||||
target_in_not_actives[0].remove_tag(self.INSTANCE_STATE_TAG, 'False')
|
||||
target_in_not_actives[0].add_tag(self.INSTANCE_STATE_TAG, 'True')
|
||||
|
||||
for inst in oneboxml_active:
|
||||
inst.remove_tag(self.INSTANCE_STATE_TAG, 'True')
|
||||
inst.add_tag(self.INSTANCE_STATE_TAG, 'False')
|
||||
pass
|
||||
|
||||
def all_volumes(self):
|
||||
oneboxml_volumes = []
|
||||
rest_volumes = []
|
||||
volumes = self._conn.get_all_volumes()
|
||||
for vol in volumes:
|
||||
if self.VOLUME_TAG in vol.tags:
|
||||
oneboxml_volumes.append(vol)
|
||||
else:
|
||||
rest_volumes.append(vol)
|
||||
return (oneboxml_volumes, rest_volumes)
|
||||
|
||||
|
||||
def describe_storages(self):
|
||||
logger.log(self.VOLUME_FORMAT.format("Name", "Id", "Size(GB)",
|
||||
"Status", "Type", "Encrypted", "Zone"))
|
||||
(oneboxml_volume, rest_volumes) = self.all_volumes()
|
||||
for vol in oneboxml_volume:
|
||||
logger.log(self.VOLUME_FORMAT.format(vol.tags[self.VOLUME_TAG],
|
||||
vol.id, str(vol.size), vol.status,
|
||||
vol.type, str(vol.encrypted), vol.zone))
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
HOMEDIR_VAR = "ONEBOXML_HOME"
|
||||
if not (HOMEDIR_VAR in os.environ):
|
||||
raise Exception(HOMEDIR_VAR + " variable is not defined")
|
||||
homedir = os.environ[HOMEDIR_VAR]
|
||||
conf_file = homedir + "/oneboxml.conf"
|
||||
|
||||
logger = Logger()
|
||||
|
||||
instance_tool = InstanceAws(homedir, conf_file)
|
||||
|
||||
(command, param) = instance_tool.get_command()
|
||||
if command == 'run':
|
||||
instance_tool.run_instance()
|
||||
if command == 'describe-instances':
|
||||
instance_tool.describe_instances()
|
||||
if command == 'terminate-instances':
|
||||
instance_tool.terminate_instances(param)
|
||||
if command == 'set-active-instance':
|
||||
instance_tool.set_active_instance(param)
|
||||
|
||||
if command == 'describe-storages':
|
||||
instance_tool.describe_storages()
|
||||
@@ -0,0 +1,15 @@
|
||||
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
class InstanceManager(object):
|
||||
def instances(self):
|
||||
return []
|
||||
|
||||
def create(self, name, cloud, parsed_args, conf_parser):
|
||||
if name in set(map(lambda x: x.name, self.instances())):
|
||||
Logger.error('Instance with name {} is already exist'.format(name))
|
||||
return 1
|
||||
|
||||
|
||||
pass
|
||||
@@ -0,0 +1,55 @@
|
||||
import sys, os, argparse, boto.ec2
|
||||
from base_aws import AwsBase #logger, Logger
|
||||
|
||||
class StorageTool(AwsBase):
|
||||
|
||||
def __init__(self, homedir, conf_file):
|
||||
super().__init__(homedir, conf_file)
|
||||
|
||||
def extend_args(self, parser):
|
||||
parser.add_argument('--size', metavar = '', type = int,
|
||||
help = 'Storage volume size in GB. Min=1, Max=16384.')
|
||||
parser.add_argument('--snapshot-id', metavar = '',
|
||||
help = 'The snapshot id for the volumne creation.')
|
||||
parser.add_argument('--type', metavar = '',
|
||||
help = 'The volume type: standard, io1 - provisioned IOPS, gp2 - general purpose')
|
||||
parser.add_argument('--iops', metavar = '',
|
||||
help = 'Number of I\O per second for provisioned IOPS volumes.')
|
||||
parser.add_argument('--encryption', metavar = '',
|
||||
help = 'Enable encription for the volume.')
|
||||
pass
|
||||
|
||||
def create(self, storage_name):
|
||||
if not self._args.size:
|
||||
logger.error("Storage volume size is not specified")
|
||||
return
|
||||
|
||||
try:
|
||||
vol = self._conn.create_volume(self._args.size,
|
||||
self._args.zone,
|
||||
self._args.snapshot_id,
|
||||
self._args.type,
|
||||
self._args.iops,
|
||||
self.toBool(self._args.encryption))
|
||||
vol.add_tag(self.VOLUME_TAG, storage_name)
|
||||
except self._conn.ResponseError as e:
|
||||
logger.error('cannot create a volume: %s' % e.message)
|
||||
return
|
||||
|
||||
logger.log("%sGB volume %s was created" % (self._args.size, storage_name))
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
HOMEDIR_VAR = "ONEBOXML_HOME"
|
||||
if not (HOMEDIR_VAR in os.environ):
|
||||
raise Exception(HOMEDIR_VAR + " variable is not defined")
|
||||
homedir = os.environ[HOMEDIR_VAR]
|
||||
conf_file = homedir + "/oneboxml.conf"
|
||||
|
||||
logger = Logger()
|
||||
|
||||
storage_tool = StorageTool(homedir, conf_file)
|
||||
|
||||
(command, param) = storage_tool.get_command()
|
||||
if command == 'create':
|
||||
storage_tool.create(param)
|
||||
+39
-35
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.logger import Logger
|
||||
@@ -28,26 +29,29 @@ StoragePath =
|
||||
LogLevel = Info
|
||||
|
||||
[AWS]
|
||||
StoragePath = dvc/tutorial
|
||||
|
||||
CredentialPath =
|
||||
|
||||
Region = us-east-1
|
||||
Zone = us-east-1a
|
||||
|
||||
# Default settings for AWS instances:
|
||||
Type = t2.nano
|
||||
Image = ami-2d39803a
|
||||
|
||||
InstanceType = t2.nano
|
||||
|
||||
SpotPrice =
|
||||
SpotTimeout = 300
|
||||
|
||||
KeyPairName = dvc-key
|
||||
KeyPairDir = ~/.ssh
|
||||
SecurityGroup = dvc-sg
|
||||
|
||||
Region = us-east-1
|
||||
Zone = us-east-1a
|
||||
SubnetId =
|
||||
|
||||
Storage = my-100gb-drive-io
|
||||
#StoragePath = dvc/tutorial
|
||||
|
||||
KeyDir = ~/.ssh
|
||||
KeyName = dvc-key
|
||||
|
||||
SecurityGroup = dvc-group
|
||||
Monitoring = false
|
||||
EbsOptimized = false
|
||||
AllDisksAsRAID0 = false
|
||||
|
||||
[GCP]
|
||||
StoragePath =
|
||||
@@ -61,10 +65,10 @@ ProjectName =
|
||||
super(CmdInit, self).__init__(settings)
|
||||
|
||||
def get_not_existing_path(self, dir):
|
||||
path = os.path.join(self.git.git_dir, dir)
|
||||
if os.path.exists(path):
|
||||
raise InitError('Path "{}" already exist'.format(path))
|
||||
return os.path.basename(path), path
|
||||
path = Path(os.path.join(self.git.git_dir, dir))
|
||||
if path.exists():
|
||||
raise InitError('Path "{}" already exist'.format(path.name))
|
||||
return path
|
||||
|
||||
def get_not_existing_conf_file_name(self):
|
||||
file_name = os.path.join(self.git.git_dir, Config.CONFIG)
|
||||
@@ -82,10 +86,10 @@ ProjectName =
|
||||
))
|
||||
return 1
|
||||
|
||||
data_dir, data_dir_path = self.get_not_existing_path(self.parsed_args.data_dir)
|
||||
cache_dir, cache_dir_path = self.get_not_existing_path(self.parsed_args.cache_dir)
|
||||
state_dir, state_dir_path = self.get_not_existing_path(self.parsed_args.state_dir)
|
||||
target_file, target_file_path = self.get_not_existing_path(self.parsed_args.target_file)
|
||||
data_dir_path = self.get_not_existing_path(self.parsed_args.data_dir)
|
||||
cache_dir_path = self.get_not_existing_path(self.parsed_args.cache_dir)
|
||||
state_dir_path = self.get_not_existing_path(self.parsed_args.state_dir)
|
||||
target_file_path = self.get_not_existing_path(self.parsed_args.target_file)
|
||||
|
||||
self.settings.config.set(self.parsed_args.data_dir,
|
||||
self.parsed_args.cache_dir,
|
||||
@@ -94,34 +98,34 @@ ProjectName =
|
||||
|
||||
conf_file_name = self.get_not_existing_conf_file_name()
|
||||
|
||||
os.mkdir(data_dir_path)
|
||||
os.mkdir(cache_dir_path)
|
||||
os.mkdir(state_dir_path)
|
||||
open(target_file_path, 'w').close()
|
||||
data_dir_path.mkdir()
|
||||
cache_dir_path.mkdir()
|
||||
state_dir_path.mkdir()
|
||||
target_file_path.touch()
|
||||
Logger.info('Directories {}/, {}/, {}/ and target file {} were created'.format(
|
||||
data_dir,
|
||||
cache_dir,
|
||||
state_dir,
|
||||
target_file))
|
||||
data_dir_path.name,
|
||||
cache_dir_path.name,
|
||||
state_dir_path.name,
|
||||
target_file_path.name))
|
||||
|
||||
self.create_empty_file()
|
||||
|
||||
conf_file = open(conf_file_name, 'wt')
|
||||
conf_file.write(self.CONFIG_TEMPLATE.format(data_dir,
|
||||
cache_dir,
|
||||
state_dir,
|
||||
target_file))
|
||||
conf_file.write(self.CONFIG_TEMPLATE.format(data_dir_path.name,
|
||||
cache_dir_path.name,
|
||||
state_dir_path.name,
|
||||
target_file_path.name))
|
||||
conf_file.close()
|
||||
|
||||
message = 'DVC init. data dir {}, cache dir {}, state dir {}, '.format(
|
||||
data_dir,
|
||||
cache_dir,
|
||||
state_dir
|
||||
data_dir_path.name,
|
||||
cache_dir_path.name,
|
||||
state_dir_path.name
|
||||
)
|
||||
if self.commit_if_needed(message) == 1:
|
||||
return 1
|
||||
|
||||
self.modify_gitignore(cache_dir)
|
||||
self.modify_gitignore(cache_dir_path.name)
|
||||
return self.commit_if_needed('DVC init. Commit .gitignore file')
|
||||
|
||||
def create_empty_file(self):
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
from dvc.cloud.instance_manager import InstanceManager
|
||||
from dvc.command.base import CmdBase, DvcLock
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
class CmdInstanceCreate(CmdBase):
|
||||
def __init__(self, settings):
|
||||
super(CmdInstanceCreate, self).__init__(settings)
|
||||
|
||||
def run(self):
|
||||
with DvcLock(self.is_locker, self.git):
|
||||
name = self.parsed_args.name
|
||||
cloud = self.parsed_args.cloud or self.settings.config.cloud
|
||||
|
||||
# print('NAME: {}'.format(name))
|
||||
# print('CLOUD: {}'.format(cloud))
|
||||
|
||||
if not name:
|
||||
Logger.error('Instance name is not defined')
|
||||
return 1
|
||||
|
||||
try:
|
||||
InstanceManager().create(name, cloud, self.parsed_args, self.settings.config)
|
||||
except DvcException as ex:
|
||||
Logger.error('Instance creation error: {}'.format(ex))
|
||||
return 1
|
||||
|
||||
return 0
|
||||
+19
-4
@@ -13,18 +13,23 @@ class ConfigError(DvcException):
|
||||
class ConfigI(object):
|
||||
TARGET_FILE_DEFAULT = '.target'
|
||||
|
||||
def __init__(self, data_dir=None, cache_dir=None, state_dir=None, target_file=None):
|
||||
def __init__(self, data_dir=None, cache_dir=None, state_dir=None, target_file=None,
|
||||
cloud=None, conf_parser=None):
|
||||
self._data_dir = None
|
||||
self._cache_dir = None
|
||||
self._state_dir = None
|
||||
self._target_file = None
|
||||
self.set(data_dir, cache_dir, state_dir, target_file)
|
||||
self._conf_parser = None
|
||||
self._cloud = None
|
||||
self.set(data_dir, cache_dir, state_dir, target_file, cloud, conf_parser)
|
||||
|
||||
def set(self, data_dir, cache_dir, state_dir, target_file):
|
||||
def set(self, data_dir, cache_dir, state_dir, target_file, cloud, conf_parser):
|
||||
self._data_dir = data_dir
|
||||
self._cache_dir = cache_dir
|
||||
self._state_dir = state_dir
|
||||
self._target_file = target_file
|
||||
self._cloud = cloud
|
||||
self._conf_parser = conf_parser
|
||||
|
||||
@property
|
||||
def data_dir(self):
|
||||
@@ -42,6 +47,14 @@ class ConfigI(object):
|
||||
def target_file(self):
|
||||
return self._target_file
|
||||
|
||||
@property
|
||||
def cloud(self):
|
||||
return self._cloud
|
||||
|
||||
@property
|
||||
def conf_parser(self):
|
||||
return self._conf_parser
|
||||
|
||||
|
||||
class Config(ConfigI):
|
||||
CONFIG = 'dvc.conf'
|
||||
@@ -68,7 +81,9 @@ class Config(ConfigI):
|
||||
super(Config, self).__init__(self._config['Global']['DataDir'],
|
||||
self._config['Global']['CacheDir'],
|
||||
self._config['Global']['StateDir'],
|
||||
self._config['Global'].get('TargetFile', Config.TARGET_FILE_DEFAULT))
|
||||
self._config['Global'].get('TargetFile', Config.TARGET_FILE_DEFAULT),
|
||||
self._config['Global']['Cloud'],
|
||||
self._config)
|
||||
pass
|
||||
|
||||
@property
|
||||
|
||||
Reference in New Issue
Block a user