Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

config: allow cross-referenced remotes

Browse Source 
Close #1614
This commit is contained in:
Mr. Outis
2019-04-08 22:10:03 -05:00
parent 441e4f4745
commit d355b1e66e
8 changed files with 163 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cache.py
+42 -43
View File
@@ -26,14 +26,14 @@ class Cache(object):
if local:
name = Config.SECTION_REMOTE_FMT.format(local)
sect = repo.config.config[name]
settings = repo.config.config[name]
else:
default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
cache_type = config.get(Config.SECTION_CACHE_TYPE)
protected = config.get(Config.SECTION_CACHE_PROTECTED)
sect = {
settings = {
Config.PRIVATE_CWD: config.get(
Config.PRIVATE_CWD, repo.dvc_dir
),
@@ -42,51 +42,50 @@ class Cache(object):
Config.SECTION_CACHE_PROTECTED: protected,
}
self._local = Remote(repo, sect)
self._s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
self._gs = self._get_remote(config, Config.SECTION_CACHE_GS)
self._ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
self._hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
self._azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
@property
def local(self):
"""Remote instance for local cache."""
return self._local
@property
def s3(self):
"""Remote instance for AWS S3 cache."""
return self._s3
@property
def gs(self):
"""Remote instance for Google Cloud Storage cache."""
return self._gs
@property
def ssh(self):
"""Remote instance for SSH cache."""
return self._ssh
@property
def hdfs(self):
"""Remote instance for HDFS cache."""
return self._hdfs
@property
def azure(self):
"""Remote instance for azure cache."""
return self._azure
self.local = Remote(repo, settings)
self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
def _get_remote(self, config, name):
"""
The config file is stored in a way that allows you to have a
cache for each remote.
This is needed when specifying external outputs
(as they require you to have an external cache location).
Imagine a config file like the following:
['remote "dvc-storage"']
url = ssh://localhost/tmp
ask_password = true
[cache]
ssh = dvc-storage
This method resolves the name under the cache section into the
correct Remote instance.
Args:
config (dict): The cache section on the config file
name (str): Name of the section we are interested in to retrieve
Returns:
remote (dvc.Remote): Remote instance that the section is referring.
None when there's no remote with that name.
Example:
>>> _get_remote(config={'ssh': 'dvc-storage'}, name='ssh')
"""
from dvc.remote import Remote
remote = config.get(name, None)
remote = config.get(name)
if not remote:
return None
name = Config.SECTION_REMOTE_FMT.format(remote)
sect = self.repo.config.config[name]
return Remote(self.repo, sect)
settings = self.repo.config.get_remote_settings(remote)
return Remote(self.repo, settings)
dvc/command/remote.py
+1 -1
View File
@@ -36,7 +36,7 @@ class CmdRemoteAdd(CmdConfig):
from dvc.remote import _get, RemoteLOCAL
remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
if remote == RemoteLOCAL:
if remote == RemoteLOCAL and not self.args.url.startswith("remote://"):
self.args.url = self.resolve_path(
self.args.url, self.configobj.filename
)
dvc/config.py
+51 -1
View File
@@ -2,7 +2,7 @@
from __future__ import unicode_literals
from dvc.utils.compat import str, open
from dvc.utils.compat import str, open, urlparse
import os
import errno
@@ -484,6 +484,56 @@ class Config(object): # pylint: disable=too-many-instance-attributes
msg = "failed to write config '{}'".format(conf.filename)
raise ConfigError(msg, exc)
def get_remote_settings(self, name):
import posixpath
"""
Args:
name (str): The name of the remote that we want to retrieve
Returns:
dict: The content beneath the given remote name.
Example:
>>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
>>> get_remote_settings("server")
{'url': 'ssh://localhost/'}
"""
settings = self.config[self.SECTION_REMOTE_FMT.format(name)]
parsed = urlparse(settings["url"])
# Support for cross referenced remotes.
# This will merge the settings, giving priority to the outer reference.
# For example, having:
#
# dvc remote add server ssh://localhost
# dvc remote modify server user root
# dvc remote modify server ask_password true
#
# dvc remote add images remote://server/tmp/pictures
# dvc remote modify images user alice
# dvc remote modify images ask_password false
# dvc remote modify images password asdf1234
#
# Results on a config dictionary like:
#
# {
# "url": "ssh://localhost/tmp/pictures",
# "user": "alice",
# "password": "asdf1234",
# "ask_password": False,
# }
#
if parsed.scheme == "remote":
reference = self.get_remote_settings(parsed.netloc)
url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
merged = reference.copy()
merged.update(settings)
merged["url"] = url
return merged
return settings
@staticmethod
def unset(config, section, opt=None):
"""Unsets specified option and/or section in the config.
dvc/dependency/__init__.py
+5 -6
View File
@@ -2,9 +2,6 @@ from __future__ import unicode_literals
import schema
from dvc.config import Config
from dvc.utils.compat import urlparse
import dvc.output as output
from dvc.output.base import OutputBase
from dvc.dependency.s3 import DependencyS3
@@ -46,11 +43,13 @@ del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
def _get(stage, p, info):
from dvc.utils.compat import urlparse
parsed = urlparse(p)
if parsed.scheme == "remote":
name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
sect = stage.repo.config.config[name]
remote = Remote(stage.repo, sect)
settings = stage.repo.config.get_remote_settings(parsed.netloc)
remote = Remote(stage.repo, settings)
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
for d in DEPS:
dvc/output/__init__.py
+3 -4
View File
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
import schema
from dvc.config import Config
from dvc.utils.compat import urlparse, str
from dvc.output.base import OutputBase
@@ -59,10 +58,10 @@ SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool
def _get(stage, p, info, cache, metric, persist=False, tags=None):
parsed = urlparse(p)
if parsed.scheme == "remote":
name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
sect = stage.repo.config.config[name]
remote = Remote(stage.repo, sect)
settings = stage.repo.config.get_remote_settings(parsed.netloc)
remote = Remote(stage.repo, settings)
return OUTS_MAP[remote.scheme](
stage,
p,
tests/test_cache.py
+9
View File
@@ -70,6 +70,15 @@ class TestExternalCacheDir(TestDvc):
self.assertFalse(os.path.exists(".dvc/cache"))
self.assertNotEqual(len(os.listdir(cache_dir)), 0)
def test_remote_references(self):
assert main(["remote", "add", "storage", "ssh://localhost"]) == 0
assert main(["remote", "add", "cache", "remote://storage/tmp"]) == 0
assert main(["config", "cache.ssh", "cache"]) == 0
self.dvc.__init__()
assert self.dvc.cache.ssh.url == "ssh://localhost/tmp"
class TestSharedCacheDir(TestDir):
def test(self):
tests/test_remote.py
+8
View File
@@ -66,6 +66,14 @@ class TestRemote(TestDvc):
main(["remote", "add", "-f", remote_name, remote_url]), 0
)
def test_referencing_other_remotes(self):
assert main(["remote", "add", "foo", "ssh://localhost/"]) == 0
assert main(["remote", "add", "bar", "remote://foo/dvc-storage"]) == 0
config = configobj.ConfigObj(self.dvc.config.config_file)
assert config['remote "bar"']["url"] == "remote://foo/dvc-storage"
class TestRemoteRemoveDefault(TestDvc):
def test(self):
tests/test_stage.py
+44
View File
@@ -1,5 +1,8 @@
import yaml
import tempfile
import os
from dvc.main import main
from dvc.output.local import OutputLOCAL
from dvc.remote.local import RemoteLOCAL
from dvc.stage import Stage, StageFileFormatError
@@ -121,3 +124,44 @@ class TestDefaultWorkingDirectory(TestDvc):
with self.dvc.state:
stage = Stage.load(self.dvc, stage.relpath)
self.assertFalse(stage.changed())
class TestExternalRemoteResolution(TestDvc):
def test_remote_output(self):
tmp_path = tempfile.mkdtemp()
storage = os.path.join(tmp_path, "storage")
file_path = os.path.join(storage, "file")
os.makedirs(storage)
assert main(["remote", "add", "tmp", tmp_path]) == 0
assert main(["remote", "add", "storage", "remote://tmp/storage"]) == 0
assert (
main(
[
"run",
"-O",
"remote://storage/file",
"echo file > {path}".format(path=file_path),
]
)
== 0
)
assert os.path.exists(file_path)
def test_remote_dependency(self):
tmp_path = tempfile.mkdtemp()
storage = os.path.join(tmp_path, "storage")
file_path = os.path.join(storage, "file")
os.makedirs(storage)
with open(file_path, "w") as fobj:
fobj.write("Isle of Dogs")
assert main(["remote", "add", "tmp", tmp_path]) == 0
assert main(["remote", "add", "storage", "remote://tmp/storage"]) == 0
assert main(["import", "remote://storage/file", "movie.txt"]) == 0
assert os.path.exists("movie.txt")