diff --git a/dvc/dependency/base.py b/dvc/dependency/base.py index 7a519f9cf..a1224ac7a 100644 --- a/dvc/dependency/base.py +++ b/dvc/dependency/base.py @@ -20,26 +20,27 @@ class DependencyBase(object): PARAM_PATH = 'path' - def __init__(self, stage, path): + def __init__(self, stage, path, info=None): self.stage = stage self.project = stage.project - self.path = path + self.url = path + self.info = info def __repr__(self): - return "{class_name}: '{path}'".format( + return "{class_name}: '{url}'".format( class_name=type(self).__name__, - path=(self.path or 'No path') + url=(self.url or 'No url') ) def __str__(self): - return self.path + return self.url @classmethod def match(cls, url): return re.match(cls.REGEX, url) def group(self, name): - match = self.match(self.path) + match = self.match(self.url) if not match: return None return match.group(name) @@ -48,6 +49,14 @@ class DependencyBase(object): def supported(cls, url): return cls.match(url) is not None + @property + def scheme(self): + return self.path_info['scheme'] + + @property + def path(self): + return self.path_info['path'] + @property def sep(self): return '/' @@ -62,18 +71,16 @@ class DependencyBase(object): def status(self): if self.changed(): # FIXME better msgs - if self.path_info['scheme'] == 'local': - p = self.rel_path - else: - p = self.path - return {p: 'changed'} + return {str(self): 'changed'} return {} def save(self): raise NotImplementedError def dumpd(self): - return {self.PARAM_PATH: self.path} + ret = self.info.copy() + ret[self.PARAM_PATH] = self.url + return ret def download(self, to_info): self.remote.download([self.path_info], [to_info]) diff --git a/dvc/dependency/gs.py b/dvc/dependency/gs.py index 6f31f3d9e..0bcc14d70 100644 --- a/dvc/dependency/gs.py +++ b/dvc/dependency/gs.py @@ -13,8 +13,7 @@ class DependencyGS(DependencyS3): REGEX = RemoteGS.REGEX def __init__(self, stage, path, info=None, remote=None): - super(DependencyGS, self).__init__(stage, path) - self.info = info + super(DependencyGS, self).__init__(stage, path, info=info) self.remote = remote if remote else RemoteGS(stage.project, {}) bucket = remote.bucket if remote else urlparse(path).netloc path = urlparse(path).path.lstrip('/') diff --git a/dvc/dependency/hdfs.py b/dvc/dependency/hdfs.py index 81506ba97..dc4a7c41a 100644 --- a/dvc/dependency/hdfs.py +++ b/dvc/dependency/hdfs.py @@ -13,8 +13,7 @@ class DependencyHDFS(DependencyBase): REGEX = RemoteHDFS.REGEX def __init__(self, stage, path, info=None, remote=None): - super(DependencyHDFS, self).__init__(stage, path) - self.info = info + super(DependencyHDFS, self).__init__(stage, path, info=info) self.remote = remote if remote else RemoteHDFS(stage.project, {}) if remote: path = posixpath.join(remote.url, urlparse(path).path.lstrip('/')) @@ -31,8 +30,3 @@ class DependencyHDFS(DependencyBase): def save(self): self.info = self.remote.save_info(self.path_info) - - def dumpd(self): - ret = self.info - ret[self.PARAM_PATH] = self.path - return ret diff --git a/dvc/dependency/http.py b/dvc/dependency/http.py index a2cccf454..df2951ae9 100644 --- a/dvc/dependency/http.py +++ b/dvc/dependency/http.py @@ -11,9 +11,8 @@ class DependencyHTTP(DependencyBase): REGEX = RemoteHTTP.REGEX def __init__(self, stage, path, info=None, remote=None): - super(DependencyHTTP, self).__init__(stage, path) + super(DependencyHTTP, self).__init__(stage, path, info=info) - self.info = info self.remote = remote or RemoteHTTP(stage.project, {}) if path.startswith('remote'): @@ -27,11 +26,6 @@ class DependencyHTTP(DependencyBase): def save(self): self.info = self.remote.save_info(self.path_info) - def dumpd(self): - ret = self.info - ret[self.PARAM_PATH] = self.path - return ret - def changed(self): if not self.exists: return True diff --git a/dvc/dependency/local.py b/dvc/dependency/local.py index f944e5e6a..4ba7e9bdf 100644 --- a/dvc/dependency/local.py +++ b/dvc/dependency/local.py @@ -19,33 +19,33 @@ class DependencyLOCAL(DependencyBase): IsNotFileOrDirError = DependencyIsNotFileOrDirError def __init__(self, stage, path, info=None, remote=None): - self.stage = stage - self.project = stage.project - self.info = info + super(DependencyLOCAL, self).__init__(stage, path, info) if remote is not None: self.remote = remote else: self.remote = RemoteLOCAL(stage.project, {}) if remote: - path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/')) + p = os.path.join(remote.prefix, + urlparse(self.url).path.lstrip('/')) + else: + p = path - if not os.path.isabs(path): - path = self.remote.to_ospath(path) - path = os.path.join(stage.cwd, path) - self.path = os.path.abspath(os.path.normpath(path)) + if not os.path.isabs(p): + p = self.remote.to_ospath(p) + p = os.path.join(stage.cwd, p) + p = os.path.abspath(os.path.normpath(p)) self.path_info = {'scheme': 'local', - 'path': self.path} + 'path': p} def __str__(self): return self.rel_path @property def is_local(self): - assert os.path.isabs(self.path) - assert os.path.isabs(self.project.root_dir) - return self.path.startswith(self.project.root_dir) + return (urlparse(self.url).scheme != 'remote' + and not os.path.isabs(self.url)) @property def sep(self): @@ -83,7 +83,7 @@ class DependencyLOCAL(DependencyBase): path = self.remote.unixpath(os.path.relpath(self.path, self.stage.cwd)) else: - path = self.path + path = self.url info = self.info.copy() info[self.PARAM_PATH] = path diff --git a/dvc/dependency/s3.py b/dvc/dependency/s3.py index d2f0c4eef..62b9330bc 100644 --- a/dvc/dependency/s3.py +++ b/dvc/dependency/s3.py @@ -13,8 +13,7 @@ class DependencyS3(DependencyBase): REGEX = RemoteS3.REGEX def __init__(self, stage, path, info=None, remote=None): - super(DependencyS3, self).__init__(stage, path) - self.info = info + super(DependencyS3, self).__init__(stage, path, info=info) self.remote = remote or RemoteS3(stage.project, {}) bucket = remote.bucket if remote else urlparse(path).netloc @@ -33,8 +32,3 @@ class DependencyS3(DependencyBase): def save(self): self.info = self.remote.save_info(self.path_info) - - def dumpd(self): - ret = self.info - ret[self.PARAM_PATH] = self.path - return ret diff --git a/dvc/dependency/ssh.py b/dvc/dependency/ssh.py index f4463213f..2f25a9707 100644 --- a/dvc/dependency/ssh.py +++ b/dvc/dependency/ssh.py @@ -13,8 +13,7 @@ class DependencySSH(DependencyBase): REGEX = RemoteSSH.REGEX def __init__(self, stage, path, info=None, remote=None): - super(DependencySSH, self).__init__(stage, path) - self.info = info + super(DependencySSH, self).__init__(stage, path, info=info) self.remote = remote if remote else RemoteSSH(stage.project, {}) host = remote.host if remote else self.group('host') @@ -24,7 +23,7 @@ class DependencySSH(DependencyBase): path = posixpath.join(remote.prefix, urlparse(path).path.lstrip('/')) else: - path = self.match(self.path).group('path') + path = self.match(self.url).group('path') self.path_info = {'scheme': 'ssh', 'host': host, @@ -40,8 +39,3 @@ class DependencySSH(DependencyBase): def save(self): self.info = self.remote.save_info(self.path_info) - - def dumpd(self): - ret = self.info - ret[self.PARAM_PATH] = self.path - return ret diff --git a/dvc/output/local.py b/dvc/output/local.py index eda2aa6b0..bd9ff504d 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -127,7 +127,7 @@ class OutputLOCAL(DependencyLOCAL): self.project.scm.ignore_remove(self.path) self.remote.move(self.path_info, out.path_info) - self.path = out.path + self.url = out.url self.path_info = out.path_info self.save() diff --git a/dvc/project.py b/dvc/project.py index 5eebf9d39..a6f605770 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -653,7 +653,7 @@ class Project(object): info['branch'] = branch ret = [info] - if out.path_info['scheme'] != 'local': + if out.scheme != 'local': return ret md5 = info[out.remote.PARAM_CHECKSUM] @@ -1098,7 +1098,7 @@ class Project(object): msg = "unable to find file '{}' in the pipeline".format(path) raise DvcException(msg) - if out.path_info['scheme'] != 'local': + if out.scheme != 'local': msg = "output '{}' scheme '{}' is not supported for metrics" raise DvcException(msg.format(out.path, out.path_info['scheme'])) diff --git a/dvc/stage.py b/dvc/stage.py index 59da076c1..73e0abfee 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -224,7 +224,7 @@ class Stage(object): def unprotect_outs(self): for out in self.outs: - if out.path_info['scheme'] != 'local' or not out.exists: + if out.scheme != 'local' or not out.exists: continue self.project.unprotect(out.path) @@ -292,7 +292,7 @@ class Stage(object): return (cls.STAGE_FILE, cwd if cwd else os.getcwd()) out = outs[0] - if out.path_info['scheme'] == 'local': + if out.scheme == 'local': path = os.path else: path = posixpath @@ -583,7 +583,7 @@ class Stage(object): def check_missing_outputs(self): outs = [out for out in self.outs if not out.exists] - paths = [out.path if out.path_info['scheme'] != 'local' else + paths = [out.path if out.scheme != 'local' else out.rel_path for out in outs] if paths: raise MissingDataSource(paths) diff --git a/tests/test_add.py b/tests/test_add.py index 444264ce7..e7621b116 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -1,4 +1,5 @@ import os +import yaml import time import shutil import filecmp @@ -8,6 +9,7 @@ from dvc.utils import file_md5 from dvc.stage import Stage from dvc.exceptions import DvcException from dvc.output.base import OutputAlreadyTrackedError +from dvc.project import Project from tests.basic_env import TestDvc @@ -149,6 +151,36 @@ class TestAddExternalLocalFile(TestDvc): self.assertTrue(filecmp.cmp(fname, 'foo', shallow=False)) +class TestAddLocalRemoteFile(TestDvc): + def test(self): + """ + Making sure that 'remote' syntax is handled properly for local outs. + """ + cwd = os.getcwd() + remote = 'myremote' + + ret = main(['remote', 'add', remote, cwd]) + self.assertEqual(ret, 0) + + self.dvc = Project() + + foo = 'remote://{}/{}'.format(remote, self.FOO) + ret = main(['add', foo]) + self.assertEqual(ret, 0) + + with open('foo.dvc', 'r') as fobj: + d = yaml.safe_load(fobj) + self.assertEqual(d['outs'][0]['path'], foo) + + bar = os.path.join(cwd, self.BAR) + ret = main(['add', bar]) + self.assertEqual(ret, 0) + + with open('bar.dvc', 'r') as fobj: + d = yaml.safe_load(fobj) + self.assertEqual(d['outs'][0]['path'], bar) + + class TestCmdAdd(TestDvc): def test(self): ret = main(['add',