From b074c49318ced4c1d7bbd7e0f197154305d8e58e Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 15 Jun 2018 02:31:35 +0300 Subject: [PATCH] dvc: support external local deps Fixes #764 Signed-off-by: Ruslan Kuprieiev --- dvc/dependency/base.py | 5 --- dvc/dependency/local.py | 29 +++++++++++++--- dvc/output/local.py | 12 +++++-- dvc/remote/local.py | 4 +++ dvc/stage.py | 12 +++---- tests/test_repro.py | 76 +++++++++++++++++++++++++++++++++++++---- 6 files changed, 111 insertions(+), 27 deletions(-) diff --git a/dvc/dependency/base.py b/dvc/dependency/base.py index 246bc75b0..525e40209 100644 --- a/dvc/dependency/base.py +++ b/dvc/dependency/base.py @@ -11,11 +11,6 @@ class DependencyError(DvcException): super(DependencyError, self).__init__('Dependency \'{}\' error: {}'.format(path, msg)) -class DependencyOutsideOfRepoError(DependencyError): - def __init__(self, path): - super(DependencyOutsideOfRepoError, self).__init__(path, 'outside of repository') - - class DependencyDoesNotExistError(DependencyError): def __init__(self, path): super(DependencyDoesNotExistError, self).__init__(path, 'does not exist') diff --git a/dvc/dependency/local.py b/dvc/dependency/local.py index 4fdb3e8fa..081ae81e6 100644 --- a/dvc/dependency/local.py +++ b/dvc/dependency/local.py @@ -3,6 +3,11 @@ import schema import posixpath import ntpath +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse + from dvc.dependency.base import * from dvc.logger import Logger from dvc.utils import remove @@ -20,12 +25,17 @@ class DependencyLOCAL(DependencyBase): def __init__(self, stage, path, info=None, remote=None): self.stage = stage self.project = stage.project - if not os.path.isabs(path): - path = self.unixpath(path) - path = os.path.join(stage.cwd, path) - self.path = os.path.normpath(path) self.info = info self.remote = remote if remote != None else RemoteLOCAL(stage.project, {}) + + if remote: + path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/')) + + if not os.path.isabs(path): + path = self.ospath(path) + path = os.path.join(stage.cwd, path) + self.path = os.path.normpath(path) + self.path_info = {'scheme': 'local', 'path': self.path} @@ -54,12 +64,23 @@ class DependencyLOCAL(DependencyBase): self.info = self.remote.save_info(self.path_info) + def ospath(self, path): + if os.name == 'nt': + return self.ntpath(path) + return self.unixpath(path) + @staticmethod def unixpath(path): assert not ntpath.isabs(path) assert not posixpath.isabs(path) return path.replace('\\', '/') + @staticmethod + def ntpath(path): + assert not ntpath.isabs(path) + assert not posixpath.isabs(path) + return path.replace('/', '\\') + def dumpd(self): if self.path.startswith(self.stage.project.root_dir): path = self.unixpath(os.path.relpath(self.path, self.stage.cwd)) diff --git a/dvc/output/local.py b/dvc/output/local.py index e36ec20e0..53e62e31f 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -66,10 +66,16 @@ class OutputLOCAL(DependencyLOCAL): self.project.logger.debug(msg.format(self.rel_path)) return - if self.project.scm.is_tracked(self.path): - raise OutputAlreadyTrackedError(self.rel_path) + if self.path.startswith(self.project.root_dir): + if self.project.scm.is_tracked(self.path): + raise OutputAlreadyTrackedError(self.rel_path) + + if self.use_cache: + self.project.scm.ignore(self.path) self.info = self.project.cache.local.save(self.path_info) - def remove(self): + def remove(self, ignore_remove=False): self.remote.remove(self.path_info) + if ignore_remove and self.use_cache: + self.project.scm.ignore_remove(self.path) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index a30d625b1..38901c215 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -35,6 +35,10 @@ class RemoteLOCAL(RemoteBase): self.state = State(self.cache_dir) + @property + def prefix(self): + return self.cache_dir + def all(self): clist = [] for entry in os.listdir(self.cache_dir): diff --git a/dvc/stage.py b/dvc/stage.py index d1f658526..231677923 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -115,14 +115,12 @@ class Stage(object): return ret - def remove_outs(self): + def remove_outs(self, ignore_remove=False): for out in self.outs: - out.remove() - if out.use_cache: - self.project.scm.ignore_remove(out.path) + out.remove(ignore_remove=ignore_remove) def remove(self): - self.remove_outs() + self.remove_outs(ignore_remove=True) os.unlink(self.path) def reproduce(self, force=False): @@ -131,7 +129,7 @@ class Stage(object): if self.cmd: # Removing outputs only if we actually have command to reproduce - self.remove_outs() + self.remove_outs(ignore_remove=False) self.run() @@ -223,8 +221,6 @@ class Stage(object): for out in self.outs: out.save() - if out.use_cache and out.path_info['scheme'] == 'local': - self.project.scm.ignore(out.path) def run(self): if not self.is_data_source: diff --git a/tests/test_repro.py b/tests/test_repro.py index cfc823dca..ff55fc1c0 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -3,6 +3,7 @@ import yaml import stat import shutil import filecmp +import tempfile from subprocess import Popen, PIPE import boto3 @@ -222,31 +223,47 @@ class TestReproExternalBase(TestDvc): def should_test(self): return False + @property + def cache_scheme(self): + return self.scheme + + @property + def scheme(self): + return None + + @property + def scheme_sep(self): + return '://' + + @property + def sep(self): + return '/' + def test(self): if not self.should_test(): return - cache = self.scheme + '://' + self.bucket + '/' + str(uuid.uuid4()) + cache = self.scheme + self.scheme_sep + self.bucket + self.sep + str(uuid.uuid4()) - ret = main(['config', 'cache.' + self.scheme, 'myrepo']) + ret = main(['config', 'cache.' + self.cache_scheme, 'myrepo']) self.assertEqual(ret, 0) ret = main(['remote', 'add', 'myrepo', cache]) self.assertEqual(ret, 0) remote_name = 'myremote' remote_key = str(uuid.uuid4()) - remote = self.scheme + '://' + self.bucket + '/' + remote_key + remote = self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key ret = main(['remote', 'add', remote_name, remote]) self.assertEqual(ret, 0) self.dvc = Project('.') - foo_key = remote_key + '/' + self.FOO - bar_key = remote_key + '/' + self.BAR + foo_key = remote_key + self.sep + self.FOO + bar_key = remote_key + self.sep + self.BAR - foo_path = self.scheme + '://' + self.bucket + '/' + foo_key - bar_path = self.scheme + '://' + self.bucket + '/' + bar_key + foo_path = self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key + bar_path = self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key # Using both plain and remote notation out_foo_path = 'remote://' + remote_name + '/' + self.FOO @@ -351,6 +368,51 @@ class TestReproExternalHDFS(TestReproExternalBase): self.assertEqual(p.returncode, 0) +class TestReproExternalLOCAL(TestReproExternalBase): + def setUp(self): + super(TestReproExternalLOCAL, self).setUp() + self.tmpdir = tempfile.mkdtemp() + + def should_test(self): + return True + + @property + def cache_scheme(self): + return 'local' + + @property + def scheme(self): + return '' + + @property + def scheme_sep(self): + return '' + + @property + def sep(self): + return os.sep + + @property + def bucket(self): + return self.tmpdir + + @property + def cmd(self): + if os.name == 'nt': + return 'copy' + return 'cp' + + def write(self, bucket, key, body): + path = os.path.join(bucket, key) + dname = os.path.dirname(path) + + if not os.path.exists(dname): + os.makedirs(dname) + + with open(path, 'w+') as fd: + fd.write(body) + + class TestReproShell(TestDvc): def test(self): if os.name == 'nt':