diff --git a/dvc/cache.py b/dvc/cache.py index 35ee9a084..e1b9d7b5d 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -26,14 +26,14 @@ class Cache(object): if local: name = Config.SECTION_REMOTE_FMT.format(local) - sect = repo.config.config[name] + settings = repo.config.config[name] else: default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR) cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir) cache_type = config.get(Config.SECTION_CACHE_TYPE) protected = config.get(Config.SECTION_CACHE_PROTECTED) - sect = { + settings = { Config.PRIVATE_CWD: config.get( Config.PRIVATE_CWD, repo.dvc_dir ), @@ -42,51 +42,50 @@ class Cache(object): Config.SECTION_CACHE_PROTECTED: protected, } - self._local = Remote(repo, sect) - - self._s3 = self._get_remote(config, Config.SECTION_CACHE_S3) - self._gs = self._get_remote(config, Config.SECTION_CACHE_GS) - self._ssh = self._get_remote(config, Config.SECTION_CACHE_SSH) - self._hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS) - self._azure = self._get_remote(config, Config.SECTION_CACHE_AZURE) - - @property - def local(self): - """Remote instance for local cache.""" - return self._local - - @property - def s3(self): - """Remote instance for AWS S3 cache.""" - return self._s3 - - @property - def gs(self): - """Remote instance for Google Cloud Storage cache.""" - return self._gs - - @property - def ssh(self): - """Remote instance for SSH cache.""" - return self._ssh - - @property - def hdfs(self): - """Remote instance for HDFS cache.""" - return self._hdfs - - @property - def azure(self): - """Remote instance for azure cache.""" - return self._azure + self.local = Remote(repo, settings) + self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3) + self.gs = self._get_remote(config, Config.SECTION_CACHE_GS) + self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH) + self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS) + self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE) def _get_remote(self, config, name): + """ + The config file is stored in a way that allows you to have a + cache for each remote. + + This is needed when specifying external outputs + (as they require you to have an external cache location). + + Imagine a config file like the following: + + ['remote "dvc-storage"'] + url = ssh://localhost/tmp + ask_password = true + + [cache] + ssh = dvc-storage + + This method resolves the name under the cache section into the + correct Remote instance. + + Args: + config (dict): The cache section on the config file + name (str): Name of the section we are interested in to retrieve + + Returns: + remote (dvc.Remote): Remote instance that the section is referring. + None when there's no remote with that name. + + Example: + >>> _get_remote(config={'ssh': 'dvc-storage'}, name='ssh') + """ from dvc.remote import Remote - remote = config.get(name, None) + remote = config.get(name) + if not remote: return None - name = Config.SECTION_REMOTE_FMT.format(remote) - sect = self.repo.config.config[name] - return Remote(self.repo, sect) + settings = self.repo.config.get_remote_settings(remote) + return Remote(self.repo, settings) diff --git a/dvc/command/remote.py b/dvc/command/remote.py index f69a7ba6e..803592ef4 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -36,7 +36,7 @@ class CmdRemoteAdd(CmdConfig): from dvc.remote import _get, RemoteLOCAL remote = _get({Config.SECTION_REMOTE_URL: self.args.url}) - if remote == RemoteLOCAL: + if remote == RemoteLOCAL and not self.args.url.startswith("remote://"): self.args.url = self.resolve_path( self.args.url, self.configobj.filename ) diff --git a/dvc/config.py b/dvc/config.py index 25472546c..360e7fc47 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals -from dvc.utils.compat import str, open +from dvc.utils.compat import str, open, urlparse import os import errno @@ -484,6 +484,56 @@ class Config(object): # pylint: disable=too-many-instance-attributes msg = "failed to write config '{}'".format(conf.filename) raise ConfigError(msg, exc) + def get_remote_settings(self, name): + import posixpath + + """ + Args: + name (str): The name of the remote that we want to retrieve + + Returns: + dict: The content beneath the given remote name. + + Example: + >>> config = {'remote "server"': {'url': 'ssh://localhost/'}} + >>> get_remote_settings("server") + {'url': 'ssh://localhost/'} + """ + settings = self.config[self.SECTION_REMOTE_FMT.format(name)] + parsed = urlparse(settings["url"]) + + # Support for cross referenced remotes. + # This will merge the settings, giving priority to the outer reference. + # For example, having: + # + # dvc remote add server ssh://localhost + # dvc remote modify server user root + # dvc remote modify server ask_password true + # + # dvc remote add images remote://server/tmp/pictures + # dvc remote modify images user alice + # dvc remote modify images ask_password false + # dvc remote modify images password asdf1234 + # + # Results on a config dictionary like: + # + # { + # "url": "ssh://localhost/tmp/pictures", + # "user": "alice", + # "password": "asdf1234", + # "ask_password": False, + # } + # + if parsed.scheme == "remote": + reference = self.get_remote_settings(parsed.netloc) + url = posixpath.join(reference["url"], parsed.path.lstrip("/")) + merged = reference.copy() + merged.update(settings) + merged["url"] = url + return merged + + return settings + @staticmethod def unset(config, section, opt=None): """Unsets specified option and/or section in the config. diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 0d372ce44..86a41b3c5 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -2,9 +2,6 @@ from __future__ import unicode_literals import schema -from dvc.config import Config -from dvc.utils.compat import urlparse - import dvc.output as output from dvc.output.base import OutputBase from dvc.dependency.s3 import DependencyS3 @@ -46,11 +43,13 @@ del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] def _get(stage, p, info): + from dvc.utils.compat import urlparse + parsed = urlparse(p) + if parsed.scheme == "remote": - name = Config.SECTION_REMOTE_FMT.format(parsed.netloc) - sect = stage.repo.config.config[name] - remote = Remote(stage.repo, sect) + settings = stage.repo.config.get_remote_settings(parsed.netloc) + remote = Remote(stage.repo, settings) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) for d in DEPS: diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index 2eb642205..7b7176a3a 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals import schema -from dvc.config import Config from dvc.utils.compat import urlparse, str from dvc.output.base import OutputBase @@ -59,10 +58,10 @@ SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool def _get(stage, p, info, cache, metric, persist=False, tags=None): parsed = urlparse(p) + if parsed.scheme == "remote": - name = Config.SECTION_REMOTE_FMT.format(parsed.netloc) - sect = stage.repo.config.config[name] - remote = Remote(stage.repo, sect) + settings = stage.repo.config.get_remote_settings(parsed.netloc) + remote = Remote(stage.repo, settings) return OUTS_MAP[remote.scheme]( stage, p, diff --git a/tests/test_cache.py b/tests/test_cache.py index 26a896aff..9fbba5fcd 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -70,6 +70,15 @@ class TestExternalCacheDir(TestDvc): self.assertFalse(os.path.exists(".dvc/cache")) self.assertNotEqual(len(os.listdir(cache_dir)), 0) + def test_remote_references(self): + assert main(["remote", "add", "storage", "ssh://localhost"]) == 0 + assert main(["remote", "add", "cache", "remote://storage/tmp"]) == 0 + assert main(["config", "cache.ssh", "cache"]) == 0 + + self.dvc.__init__() + + assert self.dvc.cache.ssh.url == "ssh://localhost/tmp" + class TestSharedCacheDir(TestDir): def test(self): diff --git a/tests/test_remote.py b/tests/test_remote.py index b298c092e..61f5ace7a 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -66,6 +66,14 @@ class TestRemote(TestDvc): main(["remote", "add", "-f", remote_name, remote_url]), 0 ) + def test_referencing_other_remotes(self): + assert main(["remote", "add", "foo", "ssh://localhost/"]) == 0 + assert main(["remote", "add", "bar", "remote://foo/dvc-storage"]) == 0 + + config = configobj.ConfigObj(self.dvc.config.config_file) + + assert config['remote "bar"']["url"] == "remote://foo/dvc-storage" + class TestRemoteRemoveDefault(TestDvc): def test(self): diff --git a/tests/test_stage.py b/tests/test_stage.py index 67266ada9..528f8eddc 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -1,5 +1,8 @@ import yaml +import tempfile +import os +from dvc.main import main from dvc.output.local import OutputLOCAL from dvc.remote.local import RemoteLOCAL from dvc.stage import Stage, StageFileFormatError @@ -121,3 +124,44 @@ class TestDefaultWorkingDirectory(TestDvc): with self.dvc.state: stage = Stage.load(self.dvc, stage.relpath) self.assertFalse(stage.changed()) + + +class TestExternalRemoteResolution(TestDvc): + def test_remote_output(self): + tmp_path = tempfile.mkdtemp() + storage = os.path.join(tmp_path, "storage") + file_path = os.path.join(storage, "file") + + os.makedirs(storage) + + assert main(["remote", "add", "tmp", tmp_path]) == 0 + assert main(["remote", "add", "storage", "remote://tmp/storage"]) == 0 + assert ( + main( + [ + "run", + "-O", + "remote://storage/file", + "echo file > {path}".format(path=file_path), + ] + ) + == 0 + ) + + assert os.path.exists(file_path) + + def test_remote_dependency(self): + tmp_path = tempfile.mkdtemp() + storage = os.path.join(tmp_path, "storage") + file_path = os.path.join(storage, "file") + + os.makedirs(storage) + + with open(file_path, "w") as fobj: + fobj.write("Isle of Dogs") + + assert main(["remote", "add", "tmp", tmp_path]) == 0 + assert main(["remote", "add", "storage", "remote://tmp/storage"]) == 0 + assert main(["import", "remote://storage/file", "movie.txt"]) == 0 + + assert os.path.exists("movie.txt")