diff --git a/dvc/cli.py b/dvc/cli.py index 457180da4..61953d265 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -5,6 +5,7 @@ import sys import argparse from multiprocessing import cpu_count +from dvc.logger import Logger from dvc.command.init import CmdInit from dvc.command.remove import CmdRemove from dvc.command.run import CmdRun @@ -15,6 +16,7 @@ from dvc.command.gc import CmdGC from dvc.command.add import CmdAdd from dvc.command.config import CmdConfig from dvc.command.checkout import CmdCheckout +from dvc.command.remote import CmdRemoteAdd, CmdRemoteRemove from dvc.stage import Stage from dvc import VERSION @@ -142,6 +144,9 @@ def parse_args(argv=None): 'pull', parents=[parent_cache_parser], help='Pull data files from the cloud') + pull_parser.add_argument('-r', + '--remote', + help='Remote repository to pull from') pull_parser.set_defaults(func=CmdDataPull) # Push @@ -149,6 +154,9 @@ def parse_args(argv=None): 'push', parents=[parent_cache_parser], help='Push data files to the cloud') + push_parser.add_argument('-r', + '--remote', + help='Remote repository to push to') push_parser.set_defaults(func=CmdDataPush) # Fetch @@ -156,6 +164,9 @@ def parse_args(argv=None): 'fetch', parents=[parent_cache_parser], help='Fetch data files from the cloud') + fetch_parser.add_argument('-r', + '--remote', + help='Remote repository to fetch from') fetch_parser.set_defaults(func=CmdDataFetch) # Status @@ -167,7 +178,10 @@ def parse_args(argv=None): '--cloud', action='store_true', default=False, - help='Show status of a local cache compared to a cloud') + help='Show status of a local cache compared to a remote repository') + status_parser.add_argument('-r', + '--remote', + help='Remote repository to compare local cache to') status_parser.set_defaults(func=CmdDataStatus) # Repro @@ -227,4 +241,45 @@ def parse_args(argv=None): help='Option value') config_parser.set_defaults(func=CmdConfig) - return parser.parse_args(argv) + + # Remote + remote_parser = subparsers.add_parser( + 'remote', + parents=[parent_parser], + help='Manage set of tracked repositories') + + remote_subparsers = remote_parser.add_subparsers( + dest='cmd', + help='Use dvc remote CMD --help for command-specific help') + + remote_add_parser = remote_subparsers.add_parser( + 'add', + parents=[parent_parser], + help='Add remote') + remote_add_parser.add_argument( + 'name', + help='Name') + remote_add_parser.add_argument( + 'url', + help='Url') + remote_add_parser.set_defaults(func=CmdRemoteAdd) + + + remote_remove_parser = remote_subparsers.add_parser( + 'remove', + parents=[parent_parser], + help='Remove remote') + remote_remove_parser.add_argument( + 'name', + help='Name') + remote_remove_parser.set_defaults(func=CmdRemoteRemove) + + + args = parser.parse_args(argv) + + if args.quiet and not args.verbose: + Logger.be_quiet() + elif not args.quiet and args.verbose: + Logger.be_verbose() + + return args diff --git a/dvc/cloud/aws.py b/dvc/cloud/aws.py index 90cb46b79..bb1c517b1 100644 --- a/dvc/cloud/aws.py +++ b/dvc/cloud/aws.py @@ -12,6 +12,7 @@ except ImportError: sys.modules['httplib'] = httplib from boto.s3.resumable_download_handler import ResumableDownloadHandler +from dvc.config import Config from dvc.logger import Logger from dvc.progress import progress from dvc.cloud.credentials_aws import AWSCredentials @@ -57,7 +58,7 @@ class DataCloudAWS(DataCloudBase): See notes http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region """ - region = self._cloud_settings.cloud_config.get('Region', None) + region = self._cloud_settings.cloud_config.get(Config.SECTION_AWS_REGION, None) if region is None or region == '': return 's3.amazonaws.com' if region == 'us-east-1': @@ -69,7 +70,7 @@ class DataCloudAWS(DataCloudBase): Try obtaining path to aws credentials from config file. """ paths = [] - credpath = self._cloud_settings.cloud_config.get('CredentialPath', None) + credpath = self._cloud_settings.cloud_config.get(Config.SECTION_AWS_CREDENTIALPATH, None) if credpath is not None and len(credpath) > 0: credpath = os.path.expanduser(credpath) if os.path.isfile(credpath): diff --git a/dvc/cloud/base.py b/dvc/cloud/base.py index 3a23a23e6..20f352e98 100644 --- a/dvc/cloud/base.py +++ b/dvc/cloud/base.py @@ -1,10 +1,11 @@ import os +import re import tempfile from checksumdir import dirhash from dvc.logger import Logger from dvc.exceptions import DvcException -from dvc.config import ConfigError +from dvc.config import Config, ConfigError from dvc.stage import Output @@ -45,9 +46,14 @@ class DataCloudBase(object): if self._cloud_settings.global_storage_path: return self._cloud_settings.global_storage_path - path = self._cloud_settings.cloud_config.get('StoragePath', None) - if path is None: - raise ConfigError('invalid StoragePath: not set for Data or cloud specific') + config = self._cloud_settings.cloud_config + url = config.get(Config.SECTION_REMOTE_URL, None) + if not url: + path = config.get(Config.SECTION_CORE_STORAGEPATH, None) + if path: + Logger.warn('Using obsoleted config format. Consider updating.') + else: + path = re.match(Config.SECTION_REMOTE_URL_REGEX, url).group(2) return path diff --git a/dvc/cloud/credentials_aws.py b/dvc/cloud/credentials_aws.py index e28083ba6..1885a0ea1 100644 --- a/dvc/cloud/credentials_aws.py +++ b/dvc/cloud/credentials_aws.py @@ -1,14 +1,15 @@ import os import configparser +from dvc.config import Config from dvc.utils import cached_property from dvc.logger import Logger class AWSCredentials(object): def __init__(self, cloud_config): - self._conf_credpath = cloud_config.get('CredentialPath', None) - self._conf_credsect = cloud_config.get('Profile', 'default') + self._conf_credpath = cloud_config.get(Config.SECTION_AWS_CREDENTIALPATH, None) + self._conf_credsect = cloud_config.get(Config.SECTION_AWS_PROFILE, 'default') @property def access_key_id(self): diff --git a/dvc/cloud/data_cloud.py b/dvc/cloud/data_cloud.py index 3080da018..ded861a63 100644 --- a/dvc/cloud/data_cloud.py +++ b/dvc/cloud/data_cloud.py @@ -1,3 +1,10 @@ +import re + +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse + from multiprocessing.pool import ThreadPool from dvc.cloud.instance_manager import CloudSettings @@ -10,118 +17,137 @@ from dvc.config import Config from dvc.cloud.aws import DataCloudAWS from dvc.cloud.gcp import DataCloudGCP from dvc.cloud.local import DataCloudLOCAL +from dvc.cloud.base import DataCloudBase class DataCloud(object): """ Generic class to do initial config parsing and redirect to proper DataCloud methods """ CLOUD_MAP = { - 'AWS' : DataCloudAWS, - 'GCP' : DataCloudGCP, - 'LOCAL' : DataCloudLOCAL, + 'aws' : DataCloudAWS, + 'gcp' : DataCloudGCP, + 'local' : DataCloudLOCAL, } SCHEME_MAP = { - 's3' : 'AWS', - 'gs' : 'GCP', - '' : 'LOCAL', + 's3' : 'aws', + 'gs' : 'gcp', + '' : 'local', } def __init__(self, cache, config): + self._cache = cache self._config = config - cloud_type = self._config[Config.SECTION_CORE].get('Cloud', '').strip().upper() - if cloud_type not in self.CLOUD_MAP.keys(): + remote = self._config[Config.SECTION_CORE].get(Config.SECTION_CORE_REMOTE, '') + if remote == '': + if config[Config.SECTION_CORE].get(Config.SECTION_CORE_CLOUD, None): + # backward compatibility + Logger.warn('Using obsoleted config format. Consider updating.') + self._cloud = self.__init__compat() + else: + self._cloud = None + return + + self._cloud = self._init_remote(remote) + + def _init_remote(self, remote): + section = Config.SECTION_REMOTE_FMT.format(remote) + cloud_config = self._config.get(section, None) + if not cloud_config: + raise ConfigError("Can't find remote section '{}' in config".format(section)) + + url = cloud_config[Config.SECTION_REMOTE_URL] + scheme = urlparse(url).scheme + cloud_type = self.SCHEME_MAP.get(scheme, None) + if not cloud_type: + raise ConfigError("Unsupported scheme '{}' in '{}'".format(scheme, url)) + + return self._init_cloud(self._cache, cloud_config, cloud_type) + + def __init__compat(self): + cloud_type = self._config[Config.SECTION_CORE].get(Config.SECTION_CORE_CLOUD, '').strip().lower() + if cloud_type == '': + self._cloud = None + return + elif cloud_type not in self.CLOUD_MAP.keys(): raise ConfigError('Wrong cloud type %s specified' % cloud_type) - if cloud_type not in self._config.keys(): + cloud_config = self._config.get(cloud_type, None) + if not cloud_config: raise ConfigError('Can\'t find cloud section \'[%s]\' in config' % cloud_type) + return self._init_cloud(self._cache, cloud_config, cloud_type) + + def _init_cloud(self, cache, cloud_config, cloud_type): cloud_settings = self.get_cloud_settings(cache, self._config, - cloud_type) + cloud_config) - self.typ = cloud_type - self._cloud = self.CLOUD_MAP[cloud_type](cloud_settings) - - self.sanity_check() + cloud = self.CLOUD_MAP[cloud_type](cloud_settings) + cloud.sanity_check() + return cloud @staticmethod - def get_cloud_settings(cache, config, cloud_type): + def get_cloud_settings(cache, config, cloud_config): """ Obtain cloud settings from config. """ - if cloud_type not in config.keys(): - cloud_config = None - else: - cloud_config = config[cloud_type] - global_storage_path = config[Config.SECTION_CORE].get('StoragePath', None) + global_storage_path = config[Config.SECTION_CORE].get(Config.SECTION_CORE_STORAGEPATH, None) + if global_storage_path: + Logger.warn('Using obsoleted config format. Consider updating.') + cloud_settings = CloudSettings(cache, global_storage_path, cloud_config) return cloud_settings - def sanity_check(self): - """ sanity check a config - - check that we have a cloud and storagePath - if aws, check can read credentials - if google, check ProjectName - - Returns: - (T,) if good - (F, issues) if bad - """ - key = 'Cloud' - core = self._config[Config.SECTION_CORE] - if key.lower() not in [k.lower() for k in core.keys()] or len(core[key]) < 1: - raise ConfigError('Please set {} in section {} in config file'.format(key, Config.SECTION_CORE)) - - # now that a cloud is chosen, can check StoragePath - storage_path = self._cloud.storage_path - if storage_path is None or len(storage_path) == 0: - raise ConfigError('Please set StoragePath = bucket/{optional path} ' - 'in config file in a cloud specific section') - - self._cloud.sanity_check() - - def _collect(self, targets, jobs, local): + def _collect(self, cloud, targets, jobs, local): collected = set() pool = ThreadPool(processes=jobs) args = zip(targets, [local]*len(targets)) - ret = pool.map(self._cloud.collect, args) + ret = pool.map(cloud.collect, args) for r in ret: collected |= set(r) return collected - def _map_targets(self, func, targets, jobs, collect_local=False, collect_cloud=False): + def _map_targets(self, func, targets, jobs, collect_local=False, collect_cloud=False, remote=None): """ Process targets as data items in parallel. """ - self._cloud.connect() + + if not remote: + cloud = self._cloud + else: + cloud = self._init_remote(remote) + + if not cloud: + return + + cloud.connect() collected = set() if collect_local: - collected |= self._collect(targets, jobs, True) + collected |= self._collect(cloud, targets, jobs, True) if collect_cloud: - collected |= self._collect(targets, jobs, False) + collected |= self._collect(cloud, targets, jobs, False) - return map_progress(func, list(collected), jobs) + return map_progress(getattr(cloud, func), list(collected), jobs) - def push(self, targets, jobs=1): + def push(self, targets, jobs=1, remote=None): """ Push data items in a cloud-agnostic way. """ - return self._map_targets(self._cloud.push, targets, jobs, collect_local=True) + return self._map_targets('push', targets, jobs, collect_local=True, remote=remote) - def pull(self, targets, jobs=1): + def pull(self, targets, jobs=1, remote=None): """ Pull data items in a cloud-agnostic way. """ - return self._map_targets(self._cloud.pull, targets, jobs, collect_cloud=True) + return self._map_targets('pull', targets, jobs, collect_cloud=True, remote=remote) - def status(self, targets, jobs=1): + def status(self, targets, jobs=1, remote=None): """ Check status of data items in a cloud-agnostic way. """ - return self._map_targets(self._cloud.status, targets, jobs, True, True) + return self._map_targets('status', targets, jobs, True, True, remote=remote) diff --git a/dvc/cloud/gcp.py b/dvc/cloud/gcp.py index 1a8ae4917..50f5e2aac 100644 --- a/dvc/cloud/gcp.py +++ b/dvc/cloud/gcp.py @@ -4,7 +4,7 @@ import os from google.cloud import storage as gc from dvc.logger import Logger -from dvc.config import ConfigError +from dvc.config import Config, ConfigError from dvc.cloud.base import DataCloudError, DataCloudBase from dvc.utils import file_md5 from dvc.progress import progress @@ -17,13 +17,7 @@ class DataCloudGCP(DataCloudBase): """ Get project name from config. """ - return self._cloud_settings.cloud_config.get('ProjectName', None) - - def sanity_check(self): - project = self.gc_project_name - if project is None or len(project) < 1: - raise ConfigError('can\'t read google cloud project name. ' - 'Please set ProjectName in section GC.') + return self._cloud_settings.cloud_config.get(Config.SECTION_GCP_PROJECTNAME, None) def connect(self): client = gc.Client(project=self.gc_project_name) diff --git a/dvc/command/common/base.py b/dvc/command/common/base.py index d04d3763a..b0ded05c1 100644 --- a/dvc/command/common/base.py +++ b/dvc/command/common/base.py @@ -10,14 +10,6 @@ class CmdBase(object): def __init__(self, args): self.project = Project(self._find_root()) self.args = args - self._set_loglevel(args) - - @staticmethod - def _set_loglevel(args): - if args.quiet and not args.verbose: - Logger.be_quiet() - elif not args.quiet and args.verbose: - Logger.be_verbose() def _find_root(self): root = os.getcwd() diff --git a/dvc/command/config.py b/dvc/command/config.py index 86ee10fb5..4bb216d80 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -12,7 +12,9 @@ class CmdConfig(CmdBase): self.args = args root_dir = self._find_root() self.config_file = os.path.join(root_dir, Project.DVC_DIR, Config.CONFIG) - self._set_loglevel(args) + # Using configobj because it doesn't + # drop comments like configparser does. + self.configobj = configobj.ConfigObj(self.config_file) def run_cmd(self): return self.run() @@ -28,31 +30,39 @@ class CmdConfig(CmdBase): return None - def unset(self): + def save(self): try: - del self.configobj[self.section][self.opt] self.configobj.write() except Exception as exc: - Logger.error('Failed to unset \'{}\': {}'.format(self.args.name, exc)) + Logger.error("Failed to write config '{}'".format(self.configobj.filename), exc) + return 1 + return 0 + + def unset(self, section, opt): + if section not in self.configobj.keys(): + Logger.error("Section '{}' doesn't exist".format(section)) return 1 - return 0 + if opt not in self.configobj[section].keys(): + return 0 + + del self.configobj[section][opt] + if len(self.configobj[section]) == 0: + del self.configobj[section] + + return self.save() def show(self): Logger.info(self.configobj[self.section][self.opt]) return 0 - def set(self): - try: - self.configobj[self.section][self.opt] = self.args.value - self.configobj.write() - except Exception as exc: - Logger.error('Failed to set \'{}\' to \'{}\': {}'.format(self.args.name, - self.args.value, - exc)) - return 1 + def set(self, section, opt, value): + if section not in self.configobj.keys(): + self.configobj[section] = {} - return 0 + self.configobj[section][opt] = value + + return self.save() def check_opt(self): _section, _opt = self.args.name.strip().split('.', 1) @@ -75,17 +85,13 @@ class CmdConfig(CmdBase): return 0 def run(self): - # Using configobj because it doesn't - # drop comments like configparser does. - self.configobj = configobj.ConfigObj(self.config_file, write_empty_values=True) - if self.check_opt() != 0: return 1 if self.args.unset: - return self.unset() + return self.unset(self.section, self.opt) if self.args.value is None: return self.show() - return self.set() + return self.set(self.section, self.opt, self.args.value) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index f174c3b63..e03f8eb5d 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -23,7 +23,7 @@ class CmdDataBase(CmdBase): class CmdDataPull(CmdDataBase): def do_run(self, target=None): try: - self.project.pull(target=target, jobs=self.args.jobs) + self.project.pull(target=target, jobs=self.args.jobs, remote=self.args.remote) except Exception as exc: self.project.logger.error('Failed to pull data from the cloud', exc) return 1 @@ -33,7 +33,7 @@ class CmdDataPull(CmdDataBase): class CmdDataPush(CmdDataBase): def do_run(self, target=None): try: - self.project.push(target=target, jobs=self.args.jobs) + self.project.push(target=target, jobs=self.args.jobs, remote=self.args.remote) except Exception as exc: self.project.logger.error('Failed to push data to the cloud', exc) return 1 @@ -43,7 +43,7 @@ class CmdDataPush(CmdDataBase): class CmdDataFetch(CmdDataBase): def do_run(self, target=None): try: - self.project.fetch(target=target, jobs=self.args.jobs) + self.project.fetch(target=target, jobs=self.args.jobs, remote=self.args.remote) except Exception as exc: self.project.logger.error('Failed to fetch data from the cloud', exc) return 1 diff --git a/dvc/command/remote.py b/dvc/command/remote.py new file mode 100644 index 000000000..a822b36e4 --- /dev/null +++ b/dvc/command/remote.py @@ -0,0 +1,15 @@ +from dvc.command.common.base import CmdBase +from dvc.config import Config +from dvc.command.config import CmdConfig + + +class CmdRemoteAdd(CmdConfig): + def run(self): + section = Config.SECTION_REMOTE_FMT.format(self.args.name) + return self.set(section, Config.SECTION_REMOTE_URL, self.args.url) + + +class CmdRemoteRemove(CmdConfig): + def run(self): + section = Config.SECTION_REMOTE_FMT.format(self.args.name) + return self.unset(section, Config.SECTION_REMOTE_URL) diff --git a/dvc/command/status.py b/dvc/command/status.py index dd3c63c49..712e65181 100644 --- a/dvc/command/status.py +++ b/dvc/command/status.py @@ -1,7 +1,5 @@ import os -import dvc.cloud.base as cloud - from dvc.command.data_sync import CmdDataBase @@ -28,7 +26,7 @@ class CmdDataStatus(CmdDataBase): def do_run(self, target=None): indent = 1 if self.args.cloud else 0 try: - status = self.project.status(target=target, jobs=self.args.jobs, cloud=self.args.cloud) + status = self.project.status(target=target, jobs=self.args.jobs, cloud=self.args.cloud, remote=self.args.remote) self._show(status, indent) except Exception as exc: self.project.logger.error('Failed to obtain data status', exc) diff --git a/dvc/config.py b/dvc/config.py index c9632c1e6..8b9b222bb 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -2,56 +2,122 @@ DVC config objects. """ import os -import configparser +import schema +import configobj from dvc.exceptions import DvcException class ConfigError(DvcException): """ DVC config exception """ - def __init__(self, msg): - DvcException.__init__(self, 'Config file error: {}'.format(msg)) + def __init__(self, ex=None): + super(ConfigError, self).__init__('Config file error', ex) class Config(object): CONFIG = 'config' - SECTION_CORE = 'Core' - CONFIG_TEMPLATE = ''' -[Core] -# Supported clouds: AWS, GCP -Cloud = AWS -# Log levels: Debug, Info, Warning and Error -LogLevel = Info + SECTION_CORE = 'core' + SECTION_CORE_LOGLEVEL = 'loglevel' + SECTION_CORE_LOGLEVEL_SCHEMA = schema.And(schema.Use(str.lower), lambda l: l in ('info', 'debug', 'warning', 'error')) + SECTION_CORE_REMOTE = 'remote' -[AWS] -CredentialPath = ~/.aws/credentials -Profile = default + # backward compatibility + SECTION_CORE_CLOUD = 'cloud' + SECTION_CORE_CLOUD_SCHEMA = schema.And(schema.Use(str.lower), lambda c: c in ('aws', 'gcp', 'local', '')) + SECTION_CORE_STORAGEPATH = 'storagepath' -StoragePath = dvc/tutorial + SECTION_CORE_SCHEMA = { + schema.Optional(SECTION_CORE_LOGLEVEL, default='info'): SECTION_CORE_LOGLEVEL_SCHEMA, + schema.Optional(SECTION_CORE_REMOTE, default=''): schema.And(str, schema.Use(str.lower)), -[GCP] -StoragePath = -ProjectName = + # backward compatibility + schema.Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA, + schema.Optional(SECTION_CORE_STORAGEPATH, default=''): str, + } -''' + # backward compatibility + SECTION_AWS = 'aws' + SECTION_AWS_STORAGEPATH = 'storagepath' + SECTION_AWS_CREDENTIALPATH = 'credentialpath' + SECTION_AWS_REGION = 'region' + SECTION_AWS_PROFILE = 'profile' + SECTION_AWS_SCHEMA = { + SECTION_AWS_STORAGEPATH: str, + schema.Optional(SECTION_AWS_REGION): str, + schema.Optional(SECTION_AWS_PROFILE, default='default'): str, + schema.Optional(SECTION_AWS_CREDENTIALPATH, default = ''): str, + } + + # backward compatibility + SECTION_GCP = 'gcp' + SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH + SECTION_GCP_PROJECTNAME = 'projectname' + SECTION_GCP_SCHEMA = { + SECTION_GCP_STORAGEPATH: str, + schema.Optional(SECTION_GCP_PROJECTNAME): str, + } + + # backward compatibility + SECTION_LOCAL = 'local' + SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH + SECTION_LOCAL_SCHEMA = { + SECTION_LOCAL_STORAGEPATH: str, + } + + + SECTION_REMOTE_REGEX = r'^\s*remote\s*"(.*)"\s*$' + SECTION_REMOTE_FMT = 'remote "{}"' + SECTION_REMOTE_URL = 'url' + SECTION_REMOTE_URL_REGEX = r'^\s*(s3://|gs://|/)?(.*)\s*$' + SECTION_REMOTE_SCHEMA = { + SECTION_REMOTE_URL: schema.Regex(SECTION_REMOTE_URL_REGEX), + schema.Optional(SECTION_AWS_REGION): str, + schema.Optional(SECTION_AWS_PROFILE, default='default'): str, + schema.Optional(SECTION_AWS_CREDENTIALPATH, default = ''): str, + schema.Optional(SECTION_GCP_PROJECTNAME): str, + } + + SCHEMA = { + schema.Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA, + schema.Optional(schema.Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA, + + # backward compatibility + schema.Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA, + schema.Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA, + schema.Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA, + } def __init__(self, dvc_dir): self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) self.config_file = os.path.join(dvc_dir, self.CONFIG) - self._config = configparser.SafeConfigParser() - try: - self._config.read(self.config_file) - except configparser.Error as ex: - raise ConfigError(ex.message) + self._config = configobj.ConfigObj(self.config_file) - if not self._config.has_section(self.SECTION_CORE): - raise ConfigError(u'section \'{}\' was not found'.format(self.SECTION_CORE)) + # NOTE: schema doesn't support ConfigObj.Section validation, so we + # need to convert our config to dict before passing it to schema. + self._config = self._lower(self._config) + self._config = schema.Schema(self.SCHEMA).validate(self._config) + + # NOTE: now converting back to ConfigObj + self._config = configobj.ConfigObj(self._config, write_empty_values=True) + self._config.filename = self.config_file + except Exception as ex: + raise ConfigError(ex) + + @staticmethod + def _lower(config): + new_config = {} + for s_key, s_value in config.items(): + new_s = {} + for key, value in s_value.items(): + new_s[key.lower()] = value + new_config[s_key.lower()] = new_s + return new_config @staticmethod def init(dvc_dir): config_file = os.path.join(dvc_dir, Config.CONFIG) - open(config_file, 'w').write(Config.CONFIG_TEMPLATE) + open(config_file, 'w+').close() return Config(dvc_dir) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index ea55d451d..1bf79dbc9 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -4,5 +4,10 @@ import traceback class DvcException(Exception): def __init__(self, msg, cause=None): self.cause = cause - self.cause_tb = traceback.format_exc() if cause else None + self.case_tb = None + if cause: + try: + self.cause_tb = traceback.format_exc() + except Exception: + pass super(DvcException, self).__init__(msg) diff --git a/dvc/logger.py b/dvc/logger.py index d4a7bf990..40bcad2f4 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -76,7 +76,8 @@ class Logger(object): str_exc = ': {}'.format(str(exc)) if exc else "" if exc and hasattr(exc, 'cause') and exc.cause: - cause_str_exc, cause_str_tb = Logger.parse_exc(exc.cause, exc.cause_tb) + cause_tb = exc.cause_tb if hasattr(exc, 'cause_tb') else None + cause_str_exc, cause_str_tb = Logger.parse_exc(exc.cause, cause_tb) str_tb = cause_str_tb str_exc = '{}{}'.format(str_exc, cause_str_exc) diff --git a/dvc/project.py b/dvc/project.py index 27f5287d1..e8ecdbb8e 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -199,14 +199,14 @@ class Project(object): os.unlink(cache) self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache))) - def push(self, target=None, jobs=1): - return self.cloud.push(self._used_cache(target), jobs) + def push(self, target=None, jobs=1, remote=None): + return self.cloud.push(self._used_cache(target), jobs, remote=remote) - def fetch(self, target=None, jobs=1): - return self.cloud.pull(self._used_cache(target), jobs) + def fetch(self, target=None, jobs=1, remote=None): + return self.cloud.pull(self._used_cache(target), jobs, remote=remote) - def pull(self, target=None, jobs=1): - ret = self.fetch(target, jobs) + def pull(self, target=None, jobs=1, remote=None): + ret = self.fetch(target, jobs, remote=remote) self.checkout() return ret @@ -223,9 +223,9 @@ class Project(object): return status - def _cloud_status(self, target=None, jobs=1): + def _cloud_status(self, target=None, jobs=1, remote=None): status = {} - for target, ret in self.cloud.status(self._used_cache(target), jobs): + for target, ret in self.cloud.status(self._used_cache(target), jobs, remote=remote): if ret == cloud.STATUS_UNKNOWN or ret == cloud.STATUS_OK: continue @@ -241,9 +241,9 @@ class Project(object): return status - def status(self, target=None, jobs=1, cloud=False): + def status(self, target=None, jobs=1, cloud=False, remote=None): if cloud: - return self._cloud_status(target, jobs) + return self._cloud_status(target, jobs, remote=remote) return self._local_status(target) def graph(self): diff --git a/tests/test_config.py b/tests/test_config.py index 1fb84aed4..2acd4673c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,4 @@ import os -import configparser import configobj from unittest import TestCase import mock @@ -18,57 +17,53 @@ except ImportError: builtin_module_name = 'builtins' from dvc.cloud.data_cloud import DataCloud +from dvc.cloud.aws import DataCloudAWS +from dvc.cloud.gcp import DataCloudGCP from dvc.main import main from tests.basic_env import TestDvc + class TestConfigTest(TestCase): def setUp(self): pass def _conf(self, s): - c = configparser.SafeConfigParser() - c.readfp(s) + c = configobj.ConfigObj(s) return c def storage_path_hierarchy_test(self): """ StoragePath should be read first from global and then from cloud section """ - c = ("[Core]", - "LogLevel =", - "DataDir = ", - "CacheDir = ", - "StateDir = ", - "Cloud = aws", - "StoragePath = globalsb/global_storage_path", - "[AWS]", - "StoragePath = awssb/aws_storage_path", - "CredentialPath =", - "[GCP]", - "StoragePath = googlesb/google_storage_path" + c = ("[core]", + "loglevel =", + "cloud = aws", + "storagepath = globalsb/global_storage_path", + "[aws]", + "storagepath = awssb/aws_storage_path", + "credentialpath =", + "[gcp]", + "storagepath = googlesb/google_storage_path" ) s = StringIO('\n'.join(c)) conf = self._conf(s) cloud = DataCloud(None, conf) - self.assertEqual(cloud.typ, 'AWS') + self.assertIsInstance(cloud._cloud, DataCloudAWS) self.assertEqual(cloud._cloud.storage_bucket, 'globalsb') self.assertEqual(cloud._cloud.storage_prefix, 'global_storage_path') - c = ("[Core]", - "LogLevel =", - "DataDir = ", - "CacheDir = ", - "StateDir = ", - "Cloud = Aws", - "[AWS]", - "StoragePath = awssb/aws_storage_path", - "CredentialPath =", - "[GCP]", - "StoragePath = googlesb/google_storage_path" + c = ("[core]", + "loglevel =", + "cloud = Aws", + "[aws]", + "storagepath = awssb/aws_storage_path", + "credentialpath =", + "[gcp]", + "storagepath = googlesb/google_storage_path" ) s = StringIO('\n'.join(c)) conf = self._conf(s) cloud = DataCloud(None, conf) - self.assertEqual(cloud.typ, 'AWS') + self.assertIsInstance(cloud._cloud, DataCloudAWS) self.assertEqual(cloud._cloud.storage_bucket, 'awssb') self.assertEqual(cloud._cloud.storage_prefix, 'aws_storage_path') @@ -90,15 +85,12 @@ class TestConfigTest(TestCase): """ in absence of [AWS] -> CredentialPath, aws creds should be read from ~/.aws/credentials """ default_path = os.path.expanduser('~/.aws/credentials') - c = ("[Core]", - "LogLevel =", - "DataDir = ", - "CacheDir = ", - "StateDir = ", - "Cloud = aws", - "[AWS]", - "StoragePath = awssb/aws_storage_path", - "CredentialPath =", + c = ("[core]", + "loglevel =", + "cloud = aws", + "[aws]", + "storagepath = awssb/aws_storage_path", + "credentialpath =", ) s = StringIO('\n'.join(c)) @@ -121,15 +113,12 @@ class TestConfigTest(TestCase): def aws_credentials_specified_tes_t_DISABLED(self, isfile_function): """ in presence of [AWS] -> CredentialPath, use those credentials """ - c = ("[Core]", - "LogLevel =", - "DataDir = ", - "CacheDir = ", - "StateDir = ", - "Cloud = aws", - "[AWS]", - "StoragePath = awssb/aws_storage_path", - "CredentialPath = some_credential_path", + c = ("[core]", + "loglevel =", + "cloud = aws", + "[aws]", + "storagePath = awssb/aws_storage_path", + "credentialpath = some_credential_path", ) s = StringIO('\n'.join(c)) @@ -160,7 +149,7 @@ class TestConfigCLI(TestDvc): return True def test(self): - section = 'SetSection' + section = 'setsection' field = 'setfield' section_field = '{}.{}'.format(section, field) value = 'setvalue' diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index b1e907e16..4df0cafe3 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -12,17 +12,28 @@ from dvc.cloud.instance_manager import CloudSettings from tests.basic_env import TestDvc -class TestDataCloud(TestDvc): - def test(self): - for k,v in DataCloud.CLOUD_MAP.items(): - config = {Config.SECTION_CORE: {'Cloud': k}, - k: {'StoragePath': 'a/b', - 'ProjectName': 'name'}} - cloud = DataCloud(self.dvc.cache, config) - self.assertIsInstance(cloud._cloud, v) +TEST_REMOTE = 'upstream' +TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE) +TEST_CONFIG = {Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE}, + TEST_SECTION: {Config.SECTION_REMOTE_URL: ''}} + +class TestDataCloud(TestDvc): + def _test_cloud(self, config, cl): + cloud = DataCloud(self.dvc.cache, config) + self.assertIsInstance(cloud._cloud, cl) + + def test(self): + config = TEST_CONFIG + + for scheme, cl in [('s3://', DataCloudAWS), ('gs://', DataCloudGCP), ('/', DataCloudLOCAL)]: + config[TEST_SECTION][Config.SECTION_REMOTE_URL] = scheme + 'a/b' + self._test_cloud(config, cl) + + def test_unsupported(self): with self.assertRaises(ConfigError) as cx: - config = {Config.SECTION_CORE: {'Cloud': 'not_supported_type'}} + config = TEST_CONFIG + config[TEST_SECTION][Config.SECTION_REMOTE_URL] = 'notsupportedscheme://a/b' DataCloud(self.dvc.cache, config) @@ -114,12 +125,13 @@ class TestDataCloudAWS(TestDataCloudBase): if not self._should_test(): return - repo = self.TEST_REPO_BUCKET + '/' + str(uuid.uuid4()) + repo = 's3://' + self.TEST_REPO_BUCKET + '/' + str(uuid.uuid4()) # Setup cloud - config = {'StoragePath': repo, - 'Region': self.TEST_REPO_REGION} - cloud_settings = CloudSettings(self.dvc.cache, None, config) + config = TEST_CONFIG + config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo + config[TEST_SECTION][Config.SECTION_AWS_REGION] = self.TEST_REPO_REGION + cloud_settings = CloudSettings(self.dvc.cache, None, config[TEST_SECTION]) self.cloud = DataCloudAWS(cloud_settings) def test(self): @@ -129,7 +141,6 @@ class TestDataCloudAWS(TestDataCloudBase): class TestDataCloudGCP(TestDataCloudBase): TEST_REPO_GCP_BUCKET = 'dvc-test' - TEST_REPO_GCP_PROJECT='dvc-project' def _should_test(self): if os.getenv("DVC_TEST_GCP") == "true": @@ -147,12 +158,12 @@ class TestDataCloudGCP(TestDataCloudBase): if os.getenv("GOOGLE_APPLICATION_CREDENTIALS") and os.getenv("GCP_CREDS"): shutil.copyfile(self.GCP_CREDS_FILE, os.getenv("GOOGLE_APPLICATION_CREDENTIALS")) - repo = self.TEST_REPO_GCP_BUCKET + '/' + str(uuid.uuid4()) + repo = 'gs://' + self.TEST_REPO_GCP_BUCKET + '/' + str(uuid.uuid4()) # Setup cloud - config = {'StoragePath': repo, - 'Region': self.TEST_REPO_GCP_PROJECT} - cloud_settings = CloudSettings(self.dvc.cache, None, config) + config = TEST_CONFIG + config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo + cloud_settings = CloudSettings(self.dvc.cache, None, config[TEST_SECTION]) self.cloud = DataCloudGCP(cloud_settings) def test(self): @@ -167,8 +178,10 @@ class TestDataCloudLOCAL(TestDataCloudBase): def _setup_cloud(self): self.dname = 'cloud' os.mkdir(self.dname) - config = {'StoragePath': self.dname} - cloud_settings = CloudSettings(self.dvc.cache, None, config) + + config = TEST_CONFIG + config[TEST_SECTION][Config.SECTION_REMOTE_URL] = self.dname + cloud_settings = CloudSettings(self.dvc.cache, None, config[TEST_SECTION]) self.cloud = DataCloudLOCAL(cloud_settings) def test(self): @@ -186,8 +199,8 @@ class TestDataCloudLocalCli(TestDvc): dname = 'cloud' os.mkdir(dname) - self.main(['config', 'Core.cloud', 'LOCAL']) - self.main(['config', 'LOCAL.StoragePath', dname]) + self.main(['config', 'core.cloud', 'local']) + self.main(['config', 'local.storagepath', dname]) stage = self.dvc.add(self.FOO) cache = stage.outs[0].cache