Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #617 from efiop/master

Browse Source 
config: Introduce 'remotes' instead of 'cloud's
This commit is contained in:
Ruslan Kuprieiev
2018-04-06 00:37:14 +03:00
committed by GitHub
parent bb9aafe090 b1135e7edc
commit 2bddeb5993
17 changed files with 385 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cli.py
+57 -2
View File
@@ -5,6 +5,7 @@ import sys
import argparse
from multiprocessing import cpu_count
from dvc.logger import Logger
from dvc.command.init import CmdInit
from dvc.command.remove import CmdRemove
from dvc.command.run import CmdRun
@@ -15,6 +16,7 @@ from dvc.command.gc import CmdGC
from dvc.command.add import CmdAdd
from dvc.command.config import CmdConfig
from dvc.command.checkout import CmdCheckout
from dvc.command.remote import CmdRemoteAdd, CmdRemoteRemove
from dvc.stage import Stage
from dvc import VERSION
@@ -142,6 +144,9 @@ def parse_args(argv=None):
'pull',
parents=[parent_cache_parser],
help='Pull data files from the cloud')
pull_parser.add_argument('-r',
'--remote',
help='Remote repository to pull from')
pull_parser.set_defaults(func=CmdDataPull)
# Push
@@ -149,6 +154,9 @@ def parse_args(argv=None):
'push',
parents=[parent_cache_parser],
help='Push data files to the cloud')
push_parser.add_argument('-r',
'--remote',
help='Remote repository to push to')
push_parser.set_defaults(func=CmdDataPush)
# Fetch
@@ -156,6 +164,9 @@ def parse_args(argv=None):
'fetch',
parents=[parent_cache_parser],
help='Fetch data files from the cloud')
fetch_parser.add_argument('-r',
'--remote',
help='Remote repository to fetch from')
fetch_parser.set_defaults(func=CmdDataFetch)
# Status
@@ -167,7 +178,10 @@ def parse_args(argv=None):
'--cloud',
action='store_true',
default=False,
help='Show status of a local cache compared to a cloud')
help='Show status of a local cache compared to a remote repository')
status_parser.add_argument('-r',
'--remote',
help='Remote repository to compare local cache to')
status_parser.set_defaults(func=CmdDataStatus)
# Repro
@@ -227,4 +241,45 @@ def parse_args(argv=None):
help='Option value')
config_parser.set_defaults(func=CmdConfig)
return parser.parse_args(argv)
# Remote
remote_parser = subparsers.add_parser(
'remote',
parents=[parent_parser],
help='Manage set of tracked repositories')
remote_subparsers = remote_parser.add_subparsers(
dest='cmd',
help='Use dvc remote CMD --help for command-specific help')
remote_add_parser = remote_subparsers.add_parser(
'add',
parents=[parent_parser],
help='Add remote')
remote_add_parser.add_argument(
'name',
help='Name')
remote_add_parser.add_argument(
'url',
help='Url')
remote_add_parser.set_defaults(func=CmdRemoteAdd)
remote_remove_parser = remote_subparsers.add_parser(
'remove',
parents=[parent_parser],
help='Remove remote')
remote_remove_parser.add_argument(
'name',
help='Name')
remote_remove_parser.set_defaults(func=CmdRemoteRemove)
args = parser.parse_args(argv)
if args.quiet and not args.verbose:
Logger.be_quiet()
elif not args.quiet and args.verbose:
Logger.be_verbose()
return args
dvc/cloud/aws.py
+3 -2
View File
@@ -12,6 +12,7 @@ except ImportError:
sys.modules['httplib'] = httplib
from boto.s3.resumable_download_handler import ResumableDownloadHandler
from dvc.config import Config
from dvc.logger import Logger
from dvc.progress import progress
from dvc.cloud.credentials_aws import AWSCredentials
@@ -57,7 +58,7 @@ class DataCloudAWS(DataCloudBase):
See notes http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
"""
region = self._cloud_settings.cloud_config.get('Region', None)
region = self._cloud_settings.cloud_config.get(Config.SECTION_AWS_REGION, None)
if region is None or region == '':
return 's3.amazonaws.com'
if region == 'us-east-1':
@@ -69,7 +70,7 @@ class DataCloudAWS(DataCloudBase):
Try obtaining path to aws credentials from config file.
"""
paths = []
credpath = self._cloud_settings.cloud_config.get('CredentialPath', None)
credpath = self._cloud_settings.cloud_config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
if credpath is not None and len(credpath) > 0:
credpath = os.path.expanduser(credpath)
if os.path.isfile(credpath):
dvc/cloud/base.py
+10 -4
View File
@@ -1,10 +1,11 @@
import os
import re
import tempfile
from checksumdir import dirhash
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.config import ConfigError
from dvc.config import Config, ConfigError
from dvc.stage import Output
@@ -45,9 +46,14 @@ class DataCloudBase(object):
if self._cloud_settings.global_storage_path:
return self._cloud_settings.global_storage_path
path = self._cloud_settings.cloud_config.get('StoragePath', None)
if path is None:
raise ConfigError('invalid StoragePath: not set for Data or cloud specific')
config = self._cloud_settings.cloud_config
url = config.get(Config.SECTION_REMOTE_URL, None)
if not url:
path = config.get(Config.SECTION_CORE_STORAGEPATH, None)
if path:
Logger.warn('Using obsoleted config format. Consider updating.')
else:
path = re.match(Config.SECTION_REMOTE_URL_REGEX, url).group(2)
return path
dvc/cloud/credentials_aws.py
+3 -2
View File
@@ -1,14 +1,15 @@
import os
import configparser
from dvc.config import Config
from dvc.utils import cached_property
from dvc.logger import Logger
class AWSCredentials(object):
def __init__(self, cloud_config):
self._conf_credpath = cloud_config.get('CredentialPath', None)
self._conf_credsect = cloud_config.get('Profile', 'default')
self._conf_credpath = cloud_config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
self._conf_credsect = cloud_config.get(Config.SECTION_AWS_PROFILE, 'default')
@property
def access_key_id(self):
dvc/cloud/data_cloud.py
+83 -57
View File
@@ -1,3 +1,10 @@
import re
try:
from urlparse import urlparse
except ImportError:
from urllib.parse import urlparse
from multiprocessing.pool import ThreadPool
from dvc.cloud.instance_manager import CloudSettings
@@ -10,118 +17,137 @@ from dvc.config import Config
from dvc.cloud.aws import DataCloudAWS
from dvc.cloud.gcp import DataCloudGCP
from dvc.cloud.local import DataCloudLOCAL
from dvc.cloud.base import DataCloudBase
class DataCloud(object):
""" Generic class to do initial config parsing and redirect to proper DataCloud methods """
CLOUD_MAP = {
'AWS' : DataCloudAWS,
'GCP' : DataCloudGCP,
'LOCAL' : DataCloudLOCAL,
'aws' : DataCloudAWS,
'gcp' : DataCloudGCP,
'local' : DataCloudLOCAL,
}
SCHEME_MAP = {
's3' : 'AWS',
'gs' : 'GCP',
'' : 'LOCAL',
's3' : 'aws',
'gs' : 'gcp',
'' : 'local',
}
def __init__(self, cache, config):
self._cache = cache
self._config = config
cloud_type = self._config[Config.SECTION_CORE].get('Cloud', '').strip().upper()
if cloud_type not in self.CLOUD_MAP.keys():
remote = self._config[Config.SECTION_CORE].get(Config.SECTION_CORE_REMOTE, '')
if remote == '':
if config[Config.SECTION_CORE].get(Config.SECTION_CORE_CLOUD, None):
# backward compatibility
Logger.warn('Using obsoleted config format. Consider updating.')
self._cloud = self.__init__compat()
else:
self._cloud = None
return
self._cloud = self._init_remote(remote)
def _init_remote(self, remote):
section = Config.SECTION_REMOTE_FMT.format(remote)
cloud_config = self._config.get(section, None)
if not cloud_config:
raise ConfigError("Can't find remote section '{}' in config".format(section))
url = cloud_config[Config.SECTION_REMOTE_URL]
scheme = urlparse(url).scheme
cloud_type = self.SCHEME_MAP.get(scheme, None)
if not cloud_type:
raise ConfigError("Unsupported scheme '{}' in '{}'".format(scheme, url))
return self._init_cloud(self._cache, cloud_config, cloud_type)
def __init__compat(self):
cloud_type = self._config[Config.SECTION_CORE].get(Config.SECTION_CORE_CLOUD, '').strip().lower()
if cloud_type == '':
self._cloud = None
return
elif cloud_type not in self.CLOUD_MAP.keys():
raise ConfigError('Wrong cloud type %s specified' % cloud_type)
if cloud_type not in self._config.keys():
cloud_config = self._config.get(cloud_type, None)
if not cloud_config:
raise ConfigError('Can\'t find cloud section \'[%s]\' in config' % cloud_type)
return self._init_cloud(self._cache, cloud_config, cloud_type)
def _init_cloud(self, cache, cloud_config, cloud_type):
cloud_settings = self.get_cloud_settings(cache,
self._config,
cloud_type)
cloud_config)
self.typ = cloud_type
self._cloud = self.CLOUD_MAP[cloud_type](cloud_settings)
self.sanity_check()
cloud = self.CLOUD_MAP[cloud_type](cloud_settings)
cloud.sanity_check()
return cloud
@staticmethod
def get_cloud_settings(cache, config, cloud_type):
def get_cloud_settings(cache, config, cloud_config):
"""
Obtain cloud settings from config.
"""
if cloud_type not in config.keys():
cloud_config = None
else:
cloud_config = config[cloud_type]
global_storage_path = config[Config.SECTION_CORE].get('StoragePath', None)
global_storage_path = config[Config.SECTION_CORE].get(Config.SECTION_CORE_STORAGEPATH, None)
if global_storage_path:
Logger.warn('Using obsoleted config format. Consider updating.')
cloud_settings = CloudSettings(cache, global_storage_path, cloud_config)
return cloud_settings
def sanity_check(self):
""" sanity check a config
check that we have a cloud and storagePath
if aws, check can read credentials
if google, check ProjectName
Returns:
(T,) if good
(F, issues) if bad
"""
key = 'Cloud'
core = self._config[Config.SECTION_CORE]
if key.lower() not in [k.lower() for k in core.keys()] or len(core[key]) < 1:
raise ConfigError('Please set {} in section {} in config file'.format(key, Config.SECTION_CORE))
# now that a cloud is chosen, can check StoragePath
storage_path = self._cloud.storage_path
if storage_path is None or len(storage_path) == 0:
raise ConfigError('Please set StoragePath = bucket/{optional path} '
'in config file in a cloud specific section')
self._cloud.sanity_check()
def _collect(self, targets, jobs, local):
def _collect(self, cloud, targets, jobs, local):
collected = set()
pool = ThreadPool(processes=jobs)
args = zip(targets, [local]*len(targets))
ret = pool.map(self._cloud.collect, args)
ret = pool.map(cloud.collect, args)
for r in ret:
collected |= set(r)
return collected
def _map_targets(self, func, targets, jobs, collect_local=False, collect_cloud=False):
def _map_targets(self, func, targets, jobs, collect_local=False, collect_cloud=False, remote=None):
"""
Process targets as data items in parallel.
"""
self._cloud.connect()
if not remote:
cloud = self._cloud
else:
cloud = self._init_remote(remote)
if not cloud:
return
cloud.connect()
collected = set()
if collect_local:
collected |= self._collect(targets, jobs, True)
collected |= self._collect(cloud, targets, jobs, True)
if collect_cloud:
collected |= self._collect(targets, jobs, False)
collected |= self._collect(cloud, targets, jobs, False)
return map_progress(func, list(collected), jobs)
return map_progress(getattr(cloud, func), list(collected), jobs)
def push(self, targets, jobs=1):
def push(self, targets, jobs=1, remote=None):
"""
Push data items in a cloud-agnostic way.
"""
return self._map_targets(self._cloud.push, targets, jobs, collect_local=True)
return self._map_targets('push', targets, jobs, collect_local=True, remote=remote)
def pull(self, targets, jobs=1):
def pull(self, targets, jobs=1, remote=None):
"""
Pull data items in a cloud-agnostic way.
"""
return self._map_targets(self._cloud.pull, targets, jobs, collect_cloud=True)
return self._map_targets('pull', targets, jobs, collect_cloud=True, remote=remote)
def status(self, targets, jobs=1):
def status(self, targets, jobs=1, remote=None):
"""
Check status of data items in a cloud-agnostic way.
"""
return self._map_targets(self._cloud.status, targets, jobs, True, True)
return self._map_targets('status', targets, jobs, True, True, remote=remote)
dvc/cloud/gcp.py
+2 -8
View File
@@ -4,7 +4,7 @@ import os
from google.cloud import storage as gc
from dvc.logger import Logger
from dvc.config import ConfigError
from dvc.config import Config, ConfigError
from dvc.cloud.base import DataCloudError, DataCloudBase
from dvc.utils import file_md5
from dvc.progress import progress
@@ -17,13 +17,7 @@ class DataCloudGCP(DataCloudBase):
"""
Get project name from config.
"""
return self._cloud_settings.cloud_config.get('ProjectName', None)
def sanity_check(self):
project = self.gc_project_name
if project is None or len(project) < 1:
raise ConfigError('can\'t read google cloud project name. '
'Please set ProjectName in section GC.')
return self._cloud_settings.cloud_config.get(Config.SECTION_GCP_PROJECTNAME, None)
def connect(self):
client = gc.Client(project=self.gc_project_name)
dvc/command/common/base.py
-8
View File
@@ -10,14 +10,6 @@ class CmdBase(object):
def __init__(self, args):
self.project = Project(self._find_root())
self.args = args
self._set_loglevel(args)
@staticmethod
def _set_loglevel(args):
if args.quiet and not args.verbose:
Logger.be_quiet()
elif not args.quiet and args.verbose:
Logger.be_verbose()
def _find_root(self):
root = os.getcwd()
dvc/command/config.py
+27 -21
View File
@@ -12,7 +12,9 @@ class CmdConfig(CmdBase):
self.args = args
root_dir = self._find_root()
self.config_file = os.path.join(root_dir, Project.DVC_DIR, Config.CONFIG)
self._set_loglevel(args)
# Using configobj because it doesn't
# drop comments like configparser does.
self.configobj = configobj.ConfigObj(self.config_file)
def run_cmd(self):
return self.run()
@@ -28,31 +30,39 @@ class CmdConfig(CmdBase):
return None
def unset(self):
def save(self):
try:
del self.configobj[self.section][self.opt]
self.configobj.write()
except Exception as exc:
Logger.error('Failed to unset \'{}\': {}'.format(self.args.name, exc))
Logger.error("Failed to write config '{}'".format(self.configobj.filename), exc)
return 1
return 0
def unset(self, section, opt):
if section not in self.configobj.keys():
Logger.error("Section '{}' doesn't exist".format(section))
return 1
return 0
if opt not in self.configobj[section].keys():
return 0
del self.configobj[section][opt]
if len(self.configobj[section]) == 0:
del self.configobj[section]
return self.save()
def show(self):
Logger.info(self.configobj[self.section][self.opt])
return 0
def set(self):
try:
self.configobj[self.section][self.opt] = self.args.value
self.configobj.write()
except Exception as exc:
Logger.error('Failed to set \'{}\' to \'{}\': {}'.format(self.args.name,
self.args.value,
exc))
return 1
def set(self, section, opt, value):
if section not in self.configobj.keys():
self.configobj[section] = {}
return 0
self.configobj[section][opt] = value
return self.save()
def check_opt(self):
_section, _opt = self.args.name.strip().split('.', 1)
@@ -75,17 +85,13 @@ class CmdConfig(CmdBase):
return 0
def run(self):
# Using configobj because it doesn't
# drop comments like configparser does.
self.configobj = configobj.ConfigObj(self.config_file, write_empty_values=True)
if self.check_opt() != 0:
return 1
if self.args.unset:
return self.unset()
return self.unset(self.section, self.opt)
if self.args.value is None:
return self.show()
return self.set()
return self.set(self.section, self.opt, self.args.value)
dvc/command/data_sync.py
+3 -3
View File
@@ -23,7 +23,7 @@ class CmdDataBase(CmdBase):
class CmdDataPull(CmdDataBase):
def do_run(self, target=None):
try:
self.project.pull(target=target, jobs=self.args.jobs)
self.project.pull(target=target, jobs=self.args.jobs, remote=self.args.remote)
except Exception as exc:
self.project.logger.error('Failed to pull data from the cloud', exc)
return 1
@@ -33,7 +33,7 @@ class CmdDataPull(CmdDataBase):
class CmdDataPush(CmdDataBase):
def do_run(self, target=None):
try:
self.project.push(target=target, jobs=self.args.jobs)
self.project.push(target=target, jobs=self.args.jobs, remote=self.args.remote)
except Exception as exc:
self.project.logger.error('Failed to push data to the cloud', exc)
return 1
@@ -43,7 +43,7 @@ class CmdDataPush(CmdDataBase):
class CmdDataFetch(CmdDataBase):
def do_run(self, target=None):
try:
self.project.fetch(target=target, jobs=self.args.jobs)
self.project.fetch(target=target, jobs=self.args.jobs, remote=self.args.remote)
except Exception as exc:
self.project.logger.error('Failed to fetch data from the cloud', exc)
return 1
dvc/command/remote.py
+15
View File
@@ -0,0 +1,15 @@
from dvc.command.common.base import CmdBase
from dvc.config import Config
from dvc.command.config import CmdConfig
class CmdRemoteAdd(CmdConfig):
def run(self):
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
return self.set(section, Config.SECTION_REMOTE_URL, self.args.url)
class CmdRemoteRemove(CmdConfig):
def run(self):
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
return self.unset(section, Config.SECTION_REMOTE_URL)
dvc/command/status.py
+1 -3
View File
@@ -1,7 +1,5 @@
import os
import dvc.cloud.base as cloud
from dvc.command.data_sync import CmdDataBase
@@ -28,7 +26,7 @@ class CmdDataStatus(CmdDataBase):
def do_run(self, target=None):
indent = 1 if self.args.cloud else 0
try:
status = self.project.status(target=target, jobs=self.args.jobs, cloud=self.args.cloud)
status = self.project.status(target=target, jobs=self.args.jobs, cloud=self.args.cloud, remote=self.args.remote)
self._show(status, indent)
except Exception as exc:
self.project.logger.error('Failed to obtain data status', exc)
dvc/config.py
+92 -26
View File
@@ -2,56 +2,122 @@
DVC config objects.
"""
import os
import configparser
import schema
import configobj
from dvc.exceptions import DvcException
class ConfigError(DvcException):
""" DVC config exception """
def __init__(self, msg):
DvcException.__init__(self, 'Config file error: {}'.format(msg))
def __init__(self, ex=None):
super(ConfigError, self).__init__('Config file error', ex)
class Config(object):
CONFIG = 'config'
SECTION_CORE = 'Core'
CONFIG_TEMPLATE = '''
[Core]
# Supported clouds: AWS, GCP
Cloud = AWS
# Log levels: Debug, Info, Warning and Error
LogLevel = Info
SECTION_CORE = 'core'
SECTION_CORE_LOGLEVEL = 'loglevel'
SECTION_CORE_LOGLEVEL_SCHEMA = schema.And(schema.Use(str.lower), lambda l: l in ('info', 'debug', 'warning', 'error'))
SECTION_CORE_REMOTE = 'remote'
[AWS]
CredentialPath = ~/.aws/credentials
Profile = default
# backward compatibility
SECTION_CORE_CLOUD = 'cloud'
SECTION_CORE_CLOUD_SCHEMA = schema.And(schema.Use(str.lower), lambda c: c in ('aws', 'gcp', 'local', ''))
SECTION_CORE_STORAGEPATH = 'storagepath'
StoragePath = dvc/tutorial
SECTION_CORE_SCHEMA = {
schema.Optional(SECTION_CORE_LOGLEVEL, default='info'): SECTION_CORE_LOGLEVEL_SCHEMA,
schema.Optional(SECTION_CORE_REMOTE, default=''): schema.And(str, schema.Use(str.lower)),
[GCP]
StoragePath =
ProjectName =
# backward compatibility
schema.Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
schema.Optional(SECTION_CORE_STORAGEPATH, default=''): str,
}
'''
# backward compatibility
SECTION_AWS = 'aws'
SECTION_AWS_STORAGEPATH = 'storagepath'
SECTION_AWS_CREDENTIALPATH = 'credentialpath'
SECTION_AWS_REGION = 'region'
SECTION_AWS_PROFILE = 'profile'
SECTION_AWS_SCHEMA = {
SECTION_AWS_STORAGEPATH: str,
schema.Optional(SECTION_AWS_REGION): str,
schema.Optional(SECTION_AWS_PROFILE, default='default'): str,
schema.Optional(SECTION_AWS_CREDENTIALPATH, default = ''): str,
}
# backward compatibility
SECTION_GCP = 'gcp'
SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_GCP_PROJECTNAME = 'projectname'
SECTION_GCP_SCHEMA = {
SECTION_GCP_STORAGEPATH: str,
schema.Optional(SECTION_GCP_PROJECTNAME): str,
}
# backward compatibility
SECTION_LOCAL = 'local'
SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_LOCAL_SCHEMA = {
SECTION_LOCAL_STORAGEPATH: str,
}
SECTION_REMOTE_REGEX = r'^\s*remote\s*"(.*)"\s*$'
SECTION_REMOTE_FMT = 'remote "{}"'
SECTION_REMOTE_URL = 'url'
SECTION_REMOTE_URL_REGEX = r'^\s*(s3://|gs://|/)?(.*)\s*$'
SECTION_REMOTE_SCHEMA = {
SECTION_REMOTE_URL: schema.Regex(SECTION_REMOTE_URL_REGEX),
schema.Optional(SECTION_AWS_REGION): str,
schema.Optional(SECTION_AWS_PROFILE, default='default'): str,
schema.Optional(SECTION_AWS_CREDENTIALPATH, default = ''): str,
schema.Optional(SECTION_GCP_PROJECTNAME): str,
}
SCHEMA = {
schema.Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
schema.Optional(schema.Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
# backward compatibility
schema.Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
schema.Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
schema.Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
}
def __init__(self, dvc_dir):
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.config_file = os.path.join(dvc_dir, self.CONFIG)
self._config = configparser.SafeConfigParser()
try:
self._config.read(self.config_file)
except configparser.Error as ex:
raise ConfigError(ex.message)
self._config = configobj.ConfigObj(self.config_file)
if not self._config.has_section(self.SECTION_CORE):
raise ConfigError(u'section \'{}\' was not found'.format(self.SECTION_CORE))
# NOTE: schema doesn't support ConfigObj.Section validation, so we
# need to convert our config to dict before passing it to schema.
self._config = self._lower(self._config)
self._config = schema.Schema(self.SCHEMA).validate(self._config)
# NOTE: now converting back to ConfigObj
self._config = configobj.ConfigObj(self._config, write_empty_values=True)
self._config.filename = self.config_file
except Exception as ex:
raise ConfigError(ex)
@staticmethod
def _lower(config):
new_config = {}
for s_key, s_value in config.items():
new_s = {}
for key, value in s_value.items():
new_s[key.lower()] = value
new_config[s_key.lower()] = new_s
return new_config
@staticmethod
def init(dvc_dir):
config_file = os.path.join(dvc_dir, Config.CONFIG)
open(config_file, 'w').write(Config.CONFIG_TEMPLATE)
open(config_file, 'w+').close()
return Config(dvc_dir)
dvc/exceptions.py
+6 -1
View File
@@ -4,5 +4,10 @@ import traceback
class DvcException(Exception):
def __init__(self, msg, cause=None):
self.cause = cause
self.cause_tb = traceback.format_exc() if cause else None
self.case_tb = None
if cause:
try:
self.cause_tb = traceback.format_exc()
except Exception:
pass
super(DvcException, self).__init__(msg)
dvc/logger.py
+2 -1
View File
@@ -76,7 +76,8 @@ class Logger(object):
str_exc = ': {}'.format(str(exc)) if exc else ""
if exc and hasattr(exc, 'cause') and exc.cause:
cause_str_exc, cause_str_tb = Logger.parse_exc(exc.cause, exc.cause_tb)
cause_tb = exc.cause_tb if hasattr(exc, 'cause_tb') else None
cause_str_exc, cause_str_tb = Logger.parse_exc(exc.cause, cause_tb)
str_tb = cause_str_tb
str_exc = '{}{}'.format(str_exc, cause_str_exc)
dvc/project.py
+10 -10
View File
@@ -199,14 +199,14 @@ class Project(object):
os.unlink(cache)
self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache)))
def push(self, target=None, jobs=1):
return self.cloud.push(self._used_cache(target), jobs)
def push(self, target=None, jobs=1, remote=None):
return self.cloud.push(self._used_cache(target), jobs, remote=remote)
def fetch(self, target=None, jobs=1):
return self.cloud.pull(self._used_cache(target), jobs)
def fetch(self, target=None, jobs=1, remote=None):
return self.cloud.pull(self._used_cache(target), jobs, remote=remote)
def pull(self, target=None, jobs=1):
ret = self.fetch(target, jobs)
def pull(self, target=None, jobs=1, remote=None):
ret = self.fetch(target, jobs, remote=remote)
self.checkout()
return ret
@@ -223,9 +223,9 @@ class Project(object):
return status
def _cloud_status(self, target=None, jobs=1):
def _cloud_status(self, target=None, jobs=1, remote=None):
status = {}
for target, ret in self.cloud.status(self._used_cache(target), jobs):
for target, ret in self.cloud.status(self._used_cache(target), jobs, remote=remote):
if ret == cloud.STATUS_UNKNOWN or ret == cloud.STATUS_OK:
continue
@@ -241,9 +241,9 @@ class Project(object):
return status
def status(self, target=None, jobs=1, cloud=False):
def status(self, target=None, jobs=1, cloud=False, remote=None):
if cloud:
return self._cloud_status(target, jobs)
return self._cloud_status(target, jobs, remote=remote)
return self._local_status(target)
def graph(self):
tests/test_config.py
+36 -47
View File
@@ -1,5 +1,4 @@
import os
import configparser
import configobj
from unittest import TestCase
import mock
@@ -18,57 +17,53 @@ except ImportError:
builtin_module_name = 'builtins'
from dvc.cloud.data_cloud import DataCloud
from dvc.cloud.aws import DataCloudAWS
from dvc.cloud.gcp import DataCloudGCP
from dvc.main import main
from tests.basic_env import TestDvc
class TestConfigTest(TestCase):
def setUp(self):
pass
def _conf(self, s):
c = configparser.SafeConfigParser()
c.readfp(s)
c = configobj.ConfigObj(s)
return c
def storage_path_hierarchy_test(self):
""" StoragePath should be read first from global and then from cloud section """
c = ("[Core]",
"LogLevel =",
"DataDir = ",
"CacheDir = ",
"StateDir = ",
"Cloud = aws",
"StoragePath = globalsb/global_storage_path",
"[AWS]",
"StoragePath = awssb/aws_storage_path",
"CredentialPath =",
"[GCP]",
"StoragePath = googlesb/google_storage_path"
c = ("[core]",
"loglevel =",
"cloud = aws",
"storagepath = globalsb/global_storage_path",
"[aws]",
"storagepath = awssb/aws_storage_path",
"credentialpath =",
"[gcp]",
"storagepath = googlesb/google_storage_path"
)
s = StringIO('\n'.join(c))
conf = self._conf(s)
cloud = DataCloud(None, conf)
self.assertEqual(cloud.typ, 'AWS')
self.assertIsInstance(cloud._cloud, DataCloudAWS)
self.assertEqual(cloud._cloud.storage_bucket, 'globalsb')
self.assertEqual(cloud._cloud.storage_prefix, 'global_storage_path')
c = ("[Core]",
"LogLevel =",
"DataDir = ",
"CacheDir = ",
"StateDir = ",
"Cloud = Aws",
"[AWS]",
"StoragePath = awssb/aws_storage_path",
"CredentialPath =",
"[GCP]",
"StoragePath = googlesb/google_storage_path"
c = ("[core]",
"loglevel =",
"cloud = Aws",
"[aws]",
"storagepath = awssb/aws_storage_path",
"credentialpath =",
"[gcp]",
"storagepath = googlesb/google_storage_path"
)
s = StringIO('\n'.join(c))
conf = self._conf(s)
cloud = DataCloud(None, conf)
self.assertEqual(cloud.typ, 'AWS')
self.assertIsInstance(cloud._cloud, DataCloudAWS)
self.assertEqual(cloud._cloud.storage_bucket, 'awssb')
self.assertEqual(cloud._cloud.storage_prefix, 'aws_storage_path')
@@ -90,15 +85,12 @@ class TestConfigTest(TestCase):
""" in absence of [AWS] -> CredentialPath, aws creds should be read from ~/.aws/credentials """
default_path = os.path.expanduser('~/.aws/credentials')
c = ("[Core]",
"LogLevel =",
"DataDir = ",
"CacheDir = ",
"StateDir = ",
"Cloud = aws",
"[AWS]",
"StoragePath = awssb/aws_storage_path",
"CredentialPath =",
c = ("[core]",
"loglevel =",
"cloud = aws",
"[aws]",
"storagepath = awssb/aws_storage_path",
"credentialpath =",
)
s = StringIO('\n'.join(c))
@@ -121,15 +113,12 @@ class TestConfigTest(TestCase):
def aws_credentials_specified_tes_t_DISABLED(self, isfile_function):
""" in presence of [AWS] -> CredentialPath, use those credentials """
c = ("[Core]",
"LogLevel =",
"DataDir = ",
"CacheDir = ",
"StateDir = ",
"Cloud = aws",
"[AWS]",
"StoragePath = awssb/aws_storage_path",
"CredentialPath = some_credential_path",
c = ("[core]",
"loglevel =",
"cloud = aws",
"[aws]",
"storagePath = awssb/aws_storage_path",
"credentialpath = some_credential_path",
)
s = StringIO('\n'.join(c))
@@ -160,7 +149,7 @@ class TestConfigCLI(TestDvc):
return True
def test(self):
section = 'SetSection'
section = 'setsection'
field = 'setfield'
section_field = '{}.{}'.format(section, field)
value = 'setvalue'
tests/test_data_cloud.py
+35 -22
View File
@@ -12,17 +12,28 @@ from dvc.cloud.instance_manager import CloudSettings
from tests.basic_env import TestDvc
class TestDataCloud(TestDvc):
def test(self):
for k,v in DataCloud.CLOUD_MAP.items():
config = {Config.SECTION_CORE: {'Cloud': k},
k: {'StoragePath': 'a/b',
'ProjectName': 'name'}}
cloud = DataCloud(self.dvc.cache, config)
self.assertIsInstance(cloud._cloud, v)
TEST_REMOTE = 'upstream'
TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE)
TEST_CONFIG = {Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE},
TEST_SECTION: {Config.SECTION_REMOTE_URL: ''}}
class TestDataCloud(TestDvc):
def _test_cloud(self, config, cl):
cloud = DataCloud(self.dvc.cache, config)
self.assertIsInstance(cloud._cloud, cl)
def test(self):
config = TEST_CONFIG
for scheme, cl in [('s3://', DataCloudAWS), ('gs://', DataCloudGCP), ('/', DataCloudLOCAL)]:
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = scheme + 'a/b'
self._test_cloud(config, cl)
def test_unsupported(self):
with self.assertRaises(ConfigError) as cx:
config = {Config.SECTION_CORE: {'Cloud': 'not_supported_type'}}
config = TEST_CONFIG
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = 'notsupportedscheme://a/b'
DataCloud(self.dvc.cache, config)
@@ -114,12 +125,13 @@ class TestDataCloudAWS(TestDataCloudBase):
if not self._should_test():
return
repo = self.TEST_REPO_BUCKET + '/' + str(uuid.uuid4())
repo = 's3://' + self.TEST_REPO_BUCKET + '/' + str(uuid.uuid4())
# Setup cloud
config = {'StoragePath': repo,
'Region': self.TEST_REPO_REGION}
cloud_settings = CloudSettings(self.dvc.cache, None, config)
config = TEST_CONFIG
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
config[TEST_SECTION][Config.SECTION_AWS_REGION] = self.TEST_REPO_REGION
cloud_settings = CloudSettings(self.dvc.cache, None, config[TEST_SECTION])
self.cloud = DataCloudAWS(cloud_settings)
def test(self):
@@ -129,7 +141,6 @@ class TestDataCloudAWS(TestDataCloudBase):
class TestDataCloudGCP(TestDataCloudBase):
TEST_REPO_GCP_BUCKET = 'dvc-test'
TEST_REPO_GCP_PROJECT='dvc-project'
def _should_test(self):
if os.getenv("DVC_TEST_GCP") == "true":
@@ -147,12 +158,12 @@ class TestDataCloudGCP(TestDataCloudBase):
if os.getenv("GOOGLE_APPLICATION_CREDENTIALS") and os.getenv("GCP_CREDS"):
shutil.copyfile(self.GCP_CREDS_FILE, os.getenv("GOOGLE_APPLICATION_CREDENTIALS"))
repo = self.TEST_REPO_GCP_BUCKET + '/' + str(uuid.uuid4())
repo = 'gs://' + self.TEST_REPO_GCP_BUCKET + '/' + str(uuid.uuid4())
# Setup cloud
config = {'StoragePath': repo,
'Region': self.TEST_REPO_GCP_PROJECT}
cloud_settings = CloudSettings(self.dvc.cache, None, config)
config = TEST_CONFIG
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
cloud_settings = CloudSettings(self.dvc.cache, None, config[TEST_SECTION])
self.cloud = DataCloudGCP(cloud_settings)
def test(self):
@@ -167,8 +178,10 @@ class TestDataCloudLOCAL(TestDataCloudBase):
def _setup_cloud(self):
self.dname = 'cloud'
os.mkdir(self.dname)
config = {'StoragePath': self.dname}
cloud_settings = CloudSettings(self.dvc.cache, None, config)
config = TEST_CONFIG
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = self.dname
cloud_settings = CloudSettings(self.dvc.cache, None, config[TEST_SECTION])
self.cloud = DataCloudLOCAL(cloud_settings)
def test(self):
@@ -186,8 +199,8 @@ class TestDataCloudLocalCli(TestDvc):
dname = 'cloud'
os.mkdir(dname)
self.main(['config', 'Core.cloud', 'LOCAL'])
self.main(['config', 'LOCAL.StoragePath', dname])
self.main(['config', 'core.cloud', 'local'])
self.main(['config', 'local.storagepath', dname])
stage = self.dvc.add(self.FOO)
cache = stage.outs[0].cache