mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 15:28:28 +01:00
Merge pull request #1467 from efiop/1284
Support relative path for cache dir and local remotes
This commit is contained in:
+4
-5
@@ -18,15 +18,14 @@ class Cache(object):
|
||||
name = Config.SECTION_REMOTE_FMT.format(local)
|
||||
sect = project.config._config[name]
|
||||
else:
|
||||
cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR)
|
||||
default_cache_dir = os.path.join(project.dvc_dir, self.CACHE_DIR)
|
||||
cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
|
||||
cache_type = config.get(Config.SECTION_CACHE_TYPE)
|
||||
protected = config.get(Config.SECTION_CACHE_PROTECTED)
|
||||
|
||||
if not os.path.isabs(cache_dir):
|
||||
cache_dir = os.path.join(project.dvc_dir, cache_dir)
|
||||
cache_dir = os.path.abspath(os.path.realpath(cache_dir))
|
||||
|
||||
sect = {
|
||||
Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD,
|
||||
project.dvc_dir),
|
||||
Config.SECTION_REMOTE_URL: cache_dir,
|
||||
Config.SECTION_CACHE_TYPE: cache_type,
|
||||
Config.SECTION_CACHE_PROTECTED: protected
|
||||
|
||||
+68
-102
@@ -21,6 +21,7 @@ from dvc.command.config import CmdConfig
|
||||
from dvc.command.checkout import CmdCheckout
|
||||
from dvc.command.remote import CmdRemoteAdd, CmdRemoteRemove
|
||||
from dvc.command.remote import CmdRemoteModify, CmdRemoteList, CmdRemoteDefault
|
||||
from dvc.command.cache import CmdCacheDir
|
||||
from dvc.command.metrics import CmdMetricsShow, CmdMetricsAdd
|
||||
from dvc.command.metrics import CmdMetricsRemove, CmdMetricsModify
|
||||
from dvc.command.install import CmdInstall
|
||||
@@ -643,11 +644,32 @@ def parse_args(argv=None):
|
||||
help='Collect garbage for all given projects.')
|
||||
gc_parser.set_defaults(func=CmdGC)
|
||||
|
||||
# Parent parser used in config/remote
|
||||
parent_config_parser = argparse.ArgumentParser(
|
||||
add_help=False,
|
||||
parents=[parent_parser])
|
||||
parent_config_parser.add_argument(
|
||||
'--global',
|
||||
dest='glob',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use global config.')
|
||||
parent_config_parser.add_argument(
|
||||
'--system',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use system config.')
|
||||
parent_config_parser.add_argument(
|
||||
'--local',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use local config.')
|
||||
|
||||
# Config
|
||||
CONFIG_HELP = 'Get or set config options.'
|
||||
config_parser = subparsers.add_parser(
|
||||
'config',
|
||||
parents=[parent_parser],
|
||||
parents=[parent_config_parser],
|
||||
description=CONFIG_HELP,
|
||||
help=CONFIG_HELP)
|
||||
config_parser.add_argument(
|
||||
@@ -664,22 +686,6 @@ def parse_args(argv=None):
|
||||
nargs='?',
|
||||
default=None,
|
||||
help='Option value.')
|
||||
config_parser.add_argument(
|
||||
'--global',
|
||||
dest='glob',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use global config.')
|
||||
config_parser.add_argument(
|
||||
'--system',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use system config.')
|
||||
config_parser.add_argument(
|
||||
'--local',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use local config.')
|
||||
config_parser.set_defaults(func=CmdConfig)
|
||||
|
||||
# Remote
|
||||
@@ -700,7 +706,7 @@ def parse_args(argv=None):
|
||||
REMOTE_ADD_HELP = 'Add remote.'
|
||||
remote_add_parser = remote_subparsers.add_parser(
|
||||
'add',
|
||||
parents=[parent_parser],
|
||||
parents=[parent_config_parser],
|
||||
description=REMOTE_ADD_HELP,
|
||||
help=REMOTE_ADD_HELP)
|
||||
remote_add_parser.add_argument(
|
||||
@@ -709,22 +715,6 @@ def parse_args(argv=None):
|
||||
remote_add_parser.add_argument(
|
||||
'url',
|
||||
help='URL.')
|
||||
remote_add_parser.add_argument(
|
||||
'--global',
|
||||
dest='glob',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use global config.')
|
||||
remote_add_parser.add_argument(
|
||||
'--system',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use system config.')
|
||||
remote_add_parser.add_argument(
|
||||
'--local',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use local config.')
|
||||
remote_add_parser.add_argument(
|
||||
'-d',
|
||||
'--default',
|
||||
@@ -736,7 +726,7 @@ def parse_args(argv=None):
|
||||
REMOTE_DEFAULT_HELP = 'Set/unset default remote.'
|
||||
remote_default_parser = remote_subparsers.add_parser(
|
||||
'default',
|
||||
parents=[parent_parser],
|
||||
parents=[parent_config_parser],
|
||||
description=REMOTE_DEFAULT_HELP,
|
||||
help=REMOTE_DEFAULT_HELP)
|
||||
remote_default_parser.add_argument(
|
||||
@@ -749,55 +739,23 @@ def parse_args(argv=None):
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Unset default remote.')
|
||||
remote_default_parser.add_argument(
|
||||
'--global',
|
||||
dest='glob',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use global config.')
|
||||
remote_default_parser.add_argument(
|
||||
'--system',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use system config.')
|
||||
remote_default_parser.add_argument(
|
||||
'--local',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use local config.')
|
||||
remote_default_parser.set_defaults(func=CmdRemoteDefault)
|
||||
|
||||
REMOTE_REMOVE_HELP = 'Remove remote.'
|
||||
remote_remove_parser = remote_subparsers.add_parser(
|
||||
'remove',
|
||||
parents=[parent_parser],
|
||||
parents=[parent_config_parser],
|
||||
description=REMOTE_REMOVE_HELP,
|
||||
help=REMOTE_REMOVE_HELP)
|
||||
remote_remove_parser.add_argument(
|
||||
'name',
|
||||
help='Name')
|
||||
remote_remove_parser.add_argument(
|
||||
'--global',
|
||||
dest='glob',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use global config.')
|
||||
remote_remove_parser.add_argument(
|
||||
'--system',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use system config.')
|
||||
remote_remove_parser.add_argument(
|
||||
'--local',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use local config.')
|
||||
remote_remove_parser.set_defaults(func=CmdRemoteRemove)
|
||||
|
||||
REMOTE_MODIFY_HELP = 'Modify remote.'
|
||||
remote_modify_parser = remote_subparsers.add_parser(
|
||||
'modify',
|
||||
parents=[parent_parser],
|
||||
parents=[parent_config_parser],
|
||||
description=REMOTE_MODIFY_HELP,
|
||||
help=REMOTE_MODIFY_HELP)
|
||||
remote_modify_parser.add_argument(
|
||||
@@ -816,48 +774,56 @@ def parse_args(argv=None):
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='Unset option.')
|
||||
remote_modify_parser.add_argument(
|
||||
'--global',
|
||||
dest='glob',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use global config.')
|
||||
remote_modify_parser.add_argument(
|
||||
'--system',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use system config.')
|
||||
remote_modify_parser.add_argument(
|
||||
'--local',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use local config.')
|
||||
remote_modify_parser.set_defaults(func=CmdRemoteModify)
|
||||
|
||||
REMOTE_LIST_HELP = 'List remotes.'
|
||||
remote_list_parser = remote_subparsers.add_parser(
|
||||
'list',
|
||||
parents=[parent_parser],
|
||||
parents=[parent_config_parser],
|
||||
description=REMOTE_LIST_HELP,
|
||||
help=REMOTE_LIST_HELP)
|
||||
remote_list_parser.add_argument(
|
||||
'--global',
|
||||
dest='glob',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use global config.')
|
||||
remote_list_parser.add_argument(
|
||||
'--system',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use system config.')
|
||||
remote_list_parser.add_argument(
|
||||
'--local',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Use local config.')
|
||||
remote_list_parser.set_defaults(func=CmdRemoteList)
|
||||
|
||||
# Cache
|
||||
CACHE_HELP = 'Manage cache settings.'
|
||||
cache_parser = subparsers.add_parser(
|
||||
'cache',
|
||||
parents=[parent_parser],
|
||||
description=CACHE_HELP,
|
||||
help=CACHE_HELP)
|
||||
|
||||
cache_subparsers = cache_parser.add_subparsers(
|
||||
dest='cmd',
|
||||
help='Use dvc cache CMD --help for '
|
||||
'command-specific help.')
|
||||
|
||||
_fix_subparsers(cache_subparsers)
|
||||
|
||||
parent_cache_config_parser = argparse.ArgumentParser(
|
||||
add_help=False,
|
||||
parents=[parent_config_parser])
|
||||
CACHE_DIR_HELP = 'Configure cache directory location.'
|
||||
cache_dir_parser = cache_subparsers.add_parser(
|
||||
'dir',
|
||||
parents=[parent_cache_config_parser],
|
||||
description=CACHE_DIR_HELP,
|
||||
help=CACHE_DIR_HELP)
|
||||
cache_dir_parser.add_argument(
|
||||
'-u',
|
||||
'--unset',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='Unset option.')
|
||||
cache_dir_parser.add_argument(
|
||||
'value',
|
||||
nargs='?',
|
||||
default=None,
|
||||
help='Path to cache directory. Relative paths are '
|
||||
'resolved relative to the current directory and '
|
||||
'saved to config relative to the config file '
|
||||
'location.')
|
||||
cache_dir_parser.set_defaults(func=CmdCacheDir)
|
||||
|
||||
# Metrics
|
||||
METRICS_HELP = 'Get metrics from all branches.'
|
||||
metrics_parser = subparsers.add_parser(
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
from dvc.command.remote import CmdRemoteAdd
|
||||
from dvc.command.config import CmdConfig
|
||||
|
||||
|
||||
class CmdCacheDir(CmdConfig):
|
||||
def run(self):
|
||||
self.args.name = 'cache.dir'
|
||||
self.args.value = CmdRemoteAdd.resolve_path(self.args.value,
|
||||
self.configobj.filename)
|
||||
|
||||
return super(CmdCacheDir, self).run()
|
||||
@@ -1,12 +1,37 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
import dvc.logger as logger
|
||||
from dvc.config import Config
|
||||
from dvc.remote import _get, RemoteLOCAL
|
||||
from dvc.command.config import CmdConfig
|
||||
|
||||
|
||||
class CmdRemoteAdd(CmdConfig):
|
||||
@staticmethod
|
||||
def resolve_path(path, config_file):
|
||||
"""Resolve path relative to config file location.
|
||||
|
||||
Args:
|
||||
path: Path to be resolved.
|
||||
config_file: Path to config file, which `path` is specified
|
||||
relative to.
|
||||
|
||||
Returns:
|
||||
Path relative to the `config_file` location. If `path` is an
|
||||
absolute path then it will be returned without change.
|
||||
|
||||
"""
|
||||
if os.path.isabs(path):
|
||||
return path
|
||||
return os.path.relpath(path, os.path.dirname(config_file))
|
||||
|
||||
def run(self):
|
||||
remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
|
||||
if remote == RemoteLOCAL:
|
||||
self.args.url = self.resolve_path(self.args.url,
|
||||
self.configobj.filename)
|
||||
|
||||
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
|
||||
ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
|
||||
if ret != 0:
|
||||
|
||||
+46
-13
@@ -17,11 +17,6 @@ class ConfigError(DvcException):
|
||||
ex)
|
||||
|
||||
|
||||
def supported_url(url):
|
||||
from dvc.remote import supported_url as supported
|
||||
return supported(url)
|
||||
|
||||
|
||||
def supported_cache_type(types):
|
||||
if isinstance(types, str):
|
||||
types = [t.strip() for t in types.split(',')]
|
||||
@@ -59,6 +54,10 @@ class Config(object):
|
||||
APPNAME = 'dvc'
|
||||
APPAUTHOR = 'iterative'
|
||||
|
||||
# NOTE: used internally in RemoteLOCAL to know config
|
||||
# location, that url should resolved relative to.
|
||||
PRIVATE_CWD = '_cwd'
|
||||
|
||||
CONFIG = 'config'
|
||||
CONFIG_LOCAL = 'config.local'
|
||||
|
||||
@@ -90,10 +89,12 @@ class Config(object):
|
||||
Optional(SECTION_CACHE_SSH): str,
|
||||
Optional(SECTION_CACHE_AZURE): str,
|
||||
|
||||
Optional(SECTION_CACHE_DIR, default='cache'): str,
|
||||
Optional(SECTION_CACHE_DIR): str,
|
||||
Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
|
||||
Optional(SECTION_CACHE_PROTECTED,
|
||||
default=False): And(str, is_bool, Use(to_bool)),
|
||||
|
||||
Optional(PRIVATE_CWD): str,
|
||||
}
|
||||
|
||||
# backward compatibility
|
||||
@@ -162,7 +163,7 @@ class Config(object):
|
||||
SECTION_REMOTE_PASSWORD = 'password'
|
||||
SECTION_REMOTE_ASK_PASSWORD = 'ask_password'
|
||||
SECTION_REMOTE_SCHEMA = {
|
||||
SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
|
||||
SECTION_REMOTE_URL: str,
|
||||
Optional(SECTION_AWS_REGION): str,
|
||||
Optional(SECTION_AWS_PROFILE): str,
|
||||
Optional(SECTION_AWS_CREDENTIALPATH): str,
|
||||
@@ -180,6 +181,8 @@ class Config(object):
|
||||
Optional(SECTION_REMOTE_PASSWORD): str,
|
||||
Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
|
||||
Optional(SECTION_AZURE_CONNECTION_STRING): str,
|
||||
|
||||
Optional(PRIVATE_CWD): str,
|
||||
}
|
||||
|
||||
SECTION_STATE = 'state'
|
||||
@@ -253,19 +256,48 @@ class Config(object):
|
||||
|
||||
self._config = None
|
||||
|
||||
def _load_config(self, path):
|
||||
config = configobj.ConfigObj(path)
|
||||
config = self._lower(config)
|
||||
self._resolve_paths(config, path)
|
||||
return config
|
||||
|
||||
@staticmethod
|
||||
def _resolve_path(path, config_file):
|
||||
assert os.path.isabs(config_file)
|
||||
config_dir = os.path.dirname(config_file)
|
||||
return os.path.abspath(os.path.join(config_dir, path))
|
||||
|
||||
def _resolve_cache_path(self, config, fname):
|
||||
cache = config.get(self.SECTION_CACHE)
|
||||
if cache is None:
|
||||
return
|
||||
|
||||
cache_dir = cache.get(self.SECTION_CACHE_DIR)
|
||||
if cache_dir is None:
|
||||
return
|
||||
|
||||
cache[self.PRIVATE_CWD] = os.path.dirname(fname)
|
||||
|
||||
def _resolve_paths(self, config, fname):
|
||||
self._resolve_cache_path(config, fname)
|
||||
for name, section in config.items():
|
||||
if self.SECTION_REMOTE_URL not in section.keys():
|
||||
continue
|
||||
|
||||
section[self.PRIVATE_CWD] = os.path.dirname(fname)
|
||||
|
||||
def load(self, validate=True):
|
||||
self._load()
|
||||
try:
|
||||
self._config = configobj.ConfigObj(self.system_config_file)
|
||||
user = configobj.ConfigObj(self.global_config_file)
|
||||
config = configobj.ConfigObj(self.config_file)
|
||||
local = configobj.ConfigObj(self.config_local_file)
|
||||
self._config = self._load_config(self.system_config_file)
|
||||
user = self._load_config(self.global_config_file)
|
||||
config = self._load_config(self.config_file)
|
||||
local = self._load_config(self.config_local_file)
|
||||
|
||||
# NOTE: schema doesn't support ConfigObj.Section validation, so we
|
||||
# need to convert our config to dict before passing it to
|
||||
self._config = self._lower(self._config)
|
||||
for c in [user, config, local]:
|
||||
c = self._lower(c)
|
||||
self._config = self._merge(self._config, c)
|
||||
|
||||
if validate:
|
||||
@@ -275,6 +307,7 @@ class Config(object):
|
||||
self._config = configobj.ConfigObj(self._config,
|
||||
write_empty_values=True)
|
||||
self._config.filename = self.config_file
|
||||
self._resolve_paths(self._config, self.config_file)
|
||||
except Exception as ex:
|
||||
raise ConfigError(ex)
|
||||
|
||||
|
||||
+2
-12
@@ -1,6 +1,7 @@
|
||||
import dvc.logger as logger
|
||||
from dvc.config import Config, ConfigError
|
||||
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from dvc.remote.gs import RemoteGS
|
||||
from dvc.remote.azure import RemoteAzure
|
||||
@@ -41,13 +42,6 @@ class DataCloud(object):
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def supported(config):
|
||||
for cloud in DataCloud.CLOUD_MAP.values():
|
||||
if cloud.supported(config):
|
||||
return cloud
|
||||
return None
|
||||
|
||||
def _init_remote(self, remote):
|
||||
section = Config.SECTION_REMOTE_FMT.format(remote)
|
||||
cloud_config = self._config.get(section, None)
|
||||
@@ -55,11 +49,7 @@ class DataCloud(object):
|
||||
msg = "can't find remote section '{}' in config"
|
||||
raise ConfigError(msg.format(section))
|
||||
|
||||
cloud_type = self.supported(cloud_config)
|
||||
if not cloud_type:
|
||||
raise ConfigError("unsupported cloud '{}'".format(cloud_config))
|
||||
|
||||
return self._init_cloud(cloud_config, cloud_type)
|
||||
return Remote(self.project, cloud_config)
|
||||
|
||||
def _init_compat(self):
|
||||
name = self._core.get(Config.SECTION_CORE_CLOUD, '').strip().lower()
|
||||
|
||||
+7
-11
@@ -6,28 +6,24 @@ from dvc.remote.s3 import RemoteS3
|
||||
from dvc.remote.ssh import RemoteSSH
|
||||
from dvc.remote.http import RemoteHTTP
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import UnsupportedRemoteError
|
||||
|
||||
|
||||
REMOTES = [
|
||||
RemoteAzure,
|
||||
RemoteGS,
|
||||
RemoteHDFS,
|
||||
RemoteHTTP,
|
||||
RemoteLOCAL,
|
||||
RemoteS3,
|
||||
RemoteSSH,
|
||||
# NOTE: RemoteLOCAL is the default
|
||||
]
|
||||
|
||||
|
||||
def supported_url(url):
|
||||
config = {Config.SECTION_REMOTE_URL: url}
|
||||
return any(remote.supported(config) for remote in REMOTES)
|
||||
def _get(config):
|
||||
for remote in REMOTES:
|
||||
if remote.supported(config):
|
||||
return remote
|
||||
return RemoteLOCAL
|
||||
|
||||
|
||||
def Remote(project, config):
|
||||
for remote in REMOTES:
|
||||
if remote.supported(config):
|
||||
return remote(project, config)
|
||||
raise UnsupportedRemoteError(str(config))
|
||||
return _get(config)(project, config)
|
||||
|
||||
+2
-3
@@ -38,9 +38,8 @@ class RemoteAzure(RemoteBase):
|
||||
|
||||
def __init__(self, project, config):
|
||||
super(RemoteAzure, self).__init__(project, config)
|
||||
self.project = project
|
||||
|
||||
self.url = config.get(Config.SECTION_REMOTE_URL)
|
||||
self.url = config.get(Config.SECTION_REMOTE_URL, 'azure://')
|
||||
match = re.match(self.REGEX, self.url) # backward compatibility
|
||||
|
||||
path = match.group('path')
|
||||
@@ -49,7 +48,7 @@ class RemoteAzure(RemoteBase):
|
||||
or match.group('container_name') # backward compatibility
|
||||
or os.getenv('AZURE_STORAGE_CONTAINER_NAME'))
|
||||
|
||||
self.prefix = urlparse(self.url).path.lstrip('/')
|
||||
self.prefix = urlparse(self.url).path.lstrip('/') if path else ''
|
||||
|
||||
self.connection_string = (
|
||||
config.get(Config.SECTION_AZURE_CONNECTION_STRING)
|
||||
|
||||
+27
-21
@@ -36,31 +36,24 @@ class RemoteBaseCmdError(DvcException):
|
||||
super(RemoteBaseCmdError, self).__init__(m.format(cmd, ret, err))
|
||||
|
||||
|
||||
class RemoteMissingDepsError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class RemoteBase(object):
|
||||
scheme = None
|
||||
REGEX = None
|
||||
REQUIRES = {}
|
||||
JOBS = 4 * cpu_count()
|
||||
|
||||
def __init__(self, project, config):
|
||||
pass
|
||||
|
||||
def __repr__(self):
|
||||
return "{class_name}: '{url}'".format(
|
||||
class_name=type(self).__name__,
|
||||
url=(self.url or 'No url')
|
||||
)
|
||||
|
||||
def compat_config(config):
|
||||
return config.copy()
|
||||
|
||||
@classmethod
|
||||
def supported(cls, config):
|
||||
url = config[Config.SECTION_REMOTE_URL]
|
||||
url_ok = cls.match(url) is not None
|
||||
deps_ok = all(cls.REQUIRES.values())
|
||||
if url_ok and not deps_ok:
|
||||
missing = [k for k, v in cls.REQUIRES.items() if v is None]
|
||||
logger.warning(
|
||||
self.project = project
|
||||
deps_ok = all(self.REQUIRES.values())
|
||||
if not deps_ok:
|
||||
missing = [k for k, v in self.REQUIRES.items() if v is None]
|
||||
url = config.get(Config.SECTION_REMOTE_URL,
|
||||
'{}://'.format(self.scheme))
|
||||
msg = (
|
||||
"URL '{}' is supported but requires these missing "
|
||||
"dependencies: {}. If you have installed dvc using pip, "
|
||||
"choose one of these options to proceed: \n"
|
||||
@@ -77,10 +70,23 @@ class RemoteBase(object):
|
||||
"If you have installed dvc from a binary package and you "
|
||||
"are still seeing this message, please report it to us "
|
||||
"using https://github.com/iterative/dvc/issues. Thank you!"
|
||||
.format(url, missing, " ".join(missing), cls.scheme)
|
||||
.format(url, missing, " ".join(missing), self.scheme)
|
||||
)
|
||||
raise RemoteMissingDepsError(msg)
|
||||
|
||||
return url_ok and deps_ok
|
||||
def __repr__(self):
|
||||
return "{class_name}: '{url}'".format(
|
||||
class_name=type(self).__name__,
|
||||
url=(self.url or 'No url')
|
||||
)
|
||||
|
||||
def compat_config(config):
|
||||
return config.copy()
|
||||
|
||||
@classmethod
|
||||
def supported(cls, config):
|
||||
url = config[Config.SECTION_REMOTE_URL]
|
||||
return cls.match(url) is not None
|
||||
|
||||
@classmethod
|
||||
def match(cls, url):
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ class RemoteGS(RemoteBase):
|
||||
PARAM_CHECKSUM = 'md5'
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
super(RemoteGS, self).__init__(project, config)
|
||||
storagepath = 'gs://'
|
||||
storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, '/')
|
||||
storagepath.lstrip('/')
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ class RemoteHDFS(RemoteBase):
|
||||
PARAM_CHECKSUM = 'checksum'
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
super(RemoteHDFS, self).__init__(project, config)
|
||||
self.url = config.get(Config.SECTION_REMOTE_URL, '/')
|
||||
self.prefix = self.url
|
||||
self.user = self.group('user')
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ class RemoteHTTP(RemoteBase):
|
||||
PARAM_ETAG = 'etag'
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
super(RemoteHTTP, self).__init__(project, config)
|
||||
self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
|
||||
self.url = self.cache_dir
|
||||
self.path_info = {'scheme': 'http'}
|
||||
|
||||
+7
-3
@@ -20,7 +20,7 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
class RemoteLOCAL(RemoteBase):
|
||||
scheme = ''
|
||||
REGEX = r'^(?P<path>(/+|.:\\+).*)$'
|
||||
REGEX = r'^(?P<path>.*)$'
|
||||
PARAM_CHECKSUM = 'md5'
|
||||
PARAM_PATH = 'path'
|
||||
PARAM_RELPATH = 'relpath'
|
||||
@@ -35,12 +35,16 @@ class RemoteLOCAL(RemoteBase):
|
||||
}
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
self.state = self.project.state
|
||||
super(RemoteLOCAL, self).__init__(project, config)
|
||||
self.state = self.project.state if self.project else None
|
||||
self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
|
||||
storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
|
||||
self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
|
||||
|
||||
if self.cache_dir is not None and not os.path.isabs(self.cache_dir):
|
||||
cwd = config[Config.PRIVATE_CWD]
|
||||
self.cache_dir = os.path.abspath(os.path.join(cwd, self.cache_dir))
|
||||
|
||||
types = config.get(Config.SECTION_CACHE_TYPE, None)
|
||||
if types:
|
||||
if isinstance(types, str):
|
||||
|
||||
+1
-1
@@ -38,7 +38,7 @@ class RemoteS3(RemoteBase):
|
||||
PARAM_CHECKSUM = 'etag'
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
super(RemoteS3, self).__init__(project, config)
|
||||
|
||||
storagepath = 's3://{}'.format(
|
||||
config.get(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
|
||||
|
||||
+1
-1
@@ -57,7 +57,7 @@ class RemoteSSH(RemoteBase):
|
||||
TIMEOUT = 1800
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
super(RemoteSSH, self).__init__(project, config)
|
||||
self.url = config.get(Config.SECTION_REMOTE_URL, '/')
|
||||
self.host = self.group('host')
|
||||
self.user = self.group('user')
|
||||
|
||||
+1
-1
@@ -10,5 +10,5 @@ os.putenv('DVC_HOME', REPO_ROOT)
|
||||
os.putenv('DVC_TEST', 'true')
|
||||
|
||||
cmd = 'nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive ' \
|
||||
'--cover-erase --cover-package=dvc --with-coverage'
|
||||
'--cover-erase --cover-package=dvc --with-coverage --all-modules'
|
||||
check_call(cmd, shell=True)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import shutil
|
||||
import configobj
|
||||
|
||||
from dvc.cache import Cache
|
||||
from dvc.main import main
|
||||
@@ -114,3 +115,33 @@ class TestCacheLinkType(TestDvc):
|
||||
|
||||
ret = main(['add', self.FOO])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
class TestCmdCacheDir(TestDvc):
|
||||
def test_abs_path(self):
|
||||
dname = os.path.join(os.path.dirname(self._root_dir), 'dir')
|
||||
ret = main(['cache', 'dir', dname])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
config = configobj.ConfigObj(self.dvc.config.config_file)
|
||||
self.assertEqual(config['cache']['dir'], dname)
|
||||
|
||||
def test_relative_path(self):
|
||||
tmpdir = self.mkdtemp()
|
||||
dname = os.path.relpath(tmpdir)
|
||||
ret = main(['cache', 'dir', dname])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
# NOTE: we are in the project's root and config is in .dvc/, so
|
||||
# dir path written to config should be just one level above.
|
||||
rel = os.path.join('..', dname)
|
||||
config = configobj.ConfigObj(self.dvc.config.config_file)
|
||||
self.assertEqual(config['cache']['dir'], rel)
|
||||
|
||||
ret = main(['add', self.FOO])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
subdirs = os.listdir(tmpdir)
|
||||
self.assertEqual(len(subdirs), 1)
|
||||
files = os.listdir(os.path.join(tmpdir, subdirs[0]))
|
||||
self.assertEqual(len(files), 1)
|
||||
|
||||
@@ -8,7 +8,7 @@ import getpass
|
||||
import platform
|
||||
|
||||
from dvc.main import main
|
||||
from dvc.config import Config, ConfigError
|
||||
from dvc.config import Config
|
||||
from dvc.data_cloud import (DataCloud, RemoteS3, RemoteGS, RemoteAzure,
|
||||
RemoteLOCAL, RemoteSSH, RemoteHDFS, RemoteHTTP)
|
||||
from dvc.remote.base import STATUS_OK, STATUS_NEW, STATUS_DELETED
|
||||
@@ -177,17 +177,6 @@ class TestDataCloud(TestDvc):
|
||||
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = remote_url
|
||||
self._test_cloud(config, cl)
|
||||
|
||||
def test_unsupported(self):
|
||||
remote_url = 'notsupportedscheme://a/b'
|
||||
config = TEST_CONFIG
|
||||
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = remote_url
|
||||
|
||||
# NOTE: making sure that error doesn't raise too early
|
||||
cloud = DataCloud(self.dvc, config=config)
|
||||
|
||||
with self.assertRaises(ConfigError):
|
||||
cloud._cloud
|
||||
|
||||
|
||||
class TestDataCloudBase(TestDvc):
|
||||
def _get_cloud_class(self):
|
||||
|
||||
+10
-5
@@ -3,8 +3,6 @@ import configobj
|
||||
|
||||
from dvc.main import main
|
||||
from dvc.command.remote import CmdRemoteAdd
|
||||
from dvc.exceptions import UnsupportedRemoteError
|
||||
from dvc.remote import Remote
|
||||
from dvc.config import Config
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
@@ -45,9 +43,16 @@ class TestRemote(TestDvc):
|
||||
ret = cmd.run()
|
||||
self.assertNotEqual(ret, 0)
|
||||
|
||||
def test_unsupported(self):
|
||||
with self.assertRaises(UnsupportedRemoteError):
|
||||
Remote(self.dvc, {Config.SECTION_REMOTE_URL: 'unsupported://url'})
|
||||
def test_relative_path(self):
|
||||
dname = os.path.join('..', 'path', 'to', 'dir')
|
||||
ret = main(['remote', 'add', 'mylocal', dname])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
# NOTE: we are in the project's root and config is in .dvc/, so
|
||||
# dir path written to config should be just one level above.
|
||||
rel = os.path.join('..', dname)
|
||||
config = configobj.ConfigObj(self.dvc.config.config_file)
|
||||
self.assertEqual(config['remote "mylocal"']['url'], rel)
|
||||
|
||||
|
||||
class TestRemoteRemoveDefault(TestDvc):
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import mock
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.remote.base import RemoteBase, RemoteMissingDepsError
|
||||
from dvc.remote import REMOTES, RemoteLOCAL
|
||||
|
||||
|
||||
class TestMissingDeps(TestCase):
|
||||
def test(self):
|
||||
remotes = REMOTES + [RemoteLOCAL, RemoteBase]
|
||||
for remote_class in remotes:
|
||||
REQUIRES = {'foo': None, 'bar': None, 'mock': mock}
|
||||
with mock.patch.object(remote_class, "REQUIRES", REQUIRES):
|
||||
with self.assertRaises(RemoteMissingDepsError):
|
||||
remote_class(None, {})
|
||||
Reference in New Issue
Block a user