Merge pull request #777 from efiop/master

dvc: support external local deps

dvc/dependency/base.py

@@ -11,11 +11,6 @@ class DependencyError(DvcException):
         super(DependencyError, self).__init__('Dependency \'{}\' error: {}'.format(path, msg))
 
 
-class DependencyOutsideOfRepoError(DependencyError):
-    def __init__(self, path):
-        super(DependencyOutsideOfRepoError, self).__init__(path, 'outside of repository')
-
-
 class DependencyDoesNotExistError(DependencyError):
     def __init__(self, path):
         super(DependencyDoesNotExistError, self).__init__(path, 'does not exist')

dvc/dependency/local.py

@@ -3,6 +3,11 @@ import schema
 import posixpath
 import ntpath
 
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
 from dvc.dependency.base import *
 from dvc.logger import Logger
 from dvc.utils import remove
@@ -20,12 +25,17 @@ class DependencyLOCAL(DependencyBase):
     def __init__(self, stage, path, info=None, remote=None):
         self.stage = stage
         self.project = stage.project
-        if not os.path.isabs(path):
-            path = self.unixpath(path)
-            path = os.path.join(stage.cwd, path)
-        self.path = os.path.normpath(path)
         self.info = info
         self.remote = remote if remote != None else RemoteLOCAL(stage.project, {})
+
+        if remote:
+            path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/'))
+
+        if not os.path.isabs(path):
+            path = self.ospath(path)
+            path = os.path.join(stage.cwd, path)
+        self.path = os.path.normpath(path)
+
         self.path_info = {'scheme': 'local',
                           'path': self.path}
 
@@ -54,12 +64,23 @@ class DependencyLOCAL(DependencyBase):
 
         self.info = self.remote.save_info(self.path_info)
 
+    def ospath(self, path):
+        if os.name == 'nt':
+            return self.ntpath(path)
+        return self.unixpath(path)
+
     @staticmethod
     def unixpath(path):
         assert not ntpath.isabs(path)
         assert not posixpath.isabs(path)
         return path.replace('\\', '/')
 
+    @staticmethod
+    def ntpath(path):
+        assert not ntpath.isabs(path)
+        assert not posixpath.isabs(path)
+        return path.replace('/', '\\')
+
     def dumpd(self):
         if self.path.startswith(self.stage.project.root_dir):
             path = self.unixpath(os.path.relpath(self.path, self.stage.cwd))

dvc/output/local.py

@@ -66,10 +66,16 @@ class OutputLOCAL(DependencyLOCAL):
             self.project.logger.debug(msg.format(self.rel_path))
             return
 
-        if self.project.scm.is_tracked(self.path):
-            raise OutputAlreadyTrackedError(self.rel_path)
+        if self.path.startswith(self.project.root_dir):
+            if self.project.scm.is_tracked(self.path):
+                raise OutputAlreadyTrackedError(self.rel_path)
+
+            if self.use_cache:
+                self.project.scm.ignore(self.path)
 
         self.info = self.project.cache.local.save(self.path_info)
 
-    def remove(self):
+    def remove(self, ignore_remove=False):
         self.remote.remove(self.path_info)
+        if ignore_remove and self.use_cache:
+            self.project.scm.ignore_remove(self.path)

dvc/remote/local.py

@@ -35,6 +35,10 @@ class RemoteLOCAL(RemoteBase):
 
         self.state = State(self.cache_dir)
 
+    @property
+    def prefix(self):
+        return self.cache_dir
+
     def all(self):
         clist = []
         for entry in os.listdir(self.cache_dir):

dvc/stage.py

@@ -115,14 +115,12 @@ class Stage(object):
 
         return ret
 
-    def remove_outs(self):
+    def remove_outs(self, ignore_remove=False):
         for out in self.outs:
-            out.remove()
-            if out.use_cache:
-                self.project.scm.ignore_remove(out.path)
+            out.remove(ignore_remove=ignore_remove)
 
     def remove(self):
-        self.remove_outs()
+        self.remove_outs(ignore_remove=True)
         os.unlink(self.path)
 
     def reproduce(self, force=False):
@@ -131,7 +129,7 @@ class Stage(object):
 
         if self.cmd:
             # Removing outputs only if we actually have command to reproduce
-            self.remove_outs()
+            self.remove_outs(ignore_remove=False)
 
         self.run()
 
@@ -223,8 +221,6 @@ class Stage(object):
 
         for out in self.outs:
             out.save()
-            if out.use_cache and out.path_info['scheme'] == 'local':
-                self.project.scm.ignore(out.path)
 
     def run(self):
         if not self.is_data_source:

tests/test_repro.py

@@ -3,6 +3,7 @@ import yaml
 import stat
 import shutil
 import filecmp
+import tempfile
 from subprocess import Popen, PIPE
 
 import boto3
@@ -222,31 +223,47 @@ class TestReproExternalBase(TestDvc):
     def should_test(self):
         return False
 
+    @property
+    def cache_scheme(self):
+        return self.scheme
+
+    @property
+    def scheme(self):
+        return None
+
+    @property
+    def scheme_sep(self):
+        return '://'
+
+    @property
+    def sep(self):
+        return '/'
+
     def test(self):
         if not self.should_test():
             return
 
-        cache = self.scheme + '://' + self.bucket + '/' + str(uuid.uuid4())
+        cache = self.scheme + self.scheme_sep + self.bucket + self.sep + str(uuid.uuid4())
 
-        ret = main(['config', 'cache.' + self.scheme, 'myrepo'])
+        ret = main(['config', 'cache.' + self.cache_scheme, 'myrepo'])
         self.assertEqual(ret, 0)
         ret = main(['remote', 'add', 'myrepo', cache])
         self.assertEqual(ret, 0)
 
         remote_name = 'myremote'
         remote_key = str(uuid.uuid4())
-        remote = self.scheme + '://' + self.bucket + '/' + remote_key
+        remote = self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key
 
         ret = main(['remote', 'add', remote_name, remote])
         self.assertEqual(ret, 0)
 
         self.dvc = Project('.')
 
-        foo_key = remote_key + '/' + self.FOO
-        bar_key = remote_key + '/' + self.BAR
+        foo_key = remote_key + self.sep + self.FOO
+        bar_key = remote_key + self.sep + self.BAR
 
-        foo_path = self.scheme + '://' + self.bucket + '/' + foo_key
-        bar_path = self.scheme + '://' + self.bucket + '/' + bar_key
+        foo_path = self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key
+        bar_path = self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key
 
         # Using both plain and remote notation
         out_foo_path = 'remote://' + remote_name + '/' + self.FOO
@@ -351,6 +368,51 @@ class TestReproExternalHDFS(TestReproExternalBase):
         self.assertEqual(p.returncode, 0)
 
  
+class TestReproExternalLOCAL(TestReproExternalBase):
+    def setUp(self):
+        super(TestReproExternalLOCAL, self).setUp()
+        self.tmpdir = tempfile.mkdtemp()
+
+    def should_test(self):
+        return True
+
+    @property
+    def cache_scheme(self):
+        return 'local'
+
+    @property
+    def scheme(self):
+        return ''
+
+    @property
+    def scheme_sep(self):
+        return ''
+
+    @property
+    def sep(self):
+        return os.sep
+
+    @property
+    def bucket(self):
+        return self.tmpdir
+
+    @property
+    def cmd(self):
+        if os.name == 'nt':
+            return 'copy'
+        return 'cp'
+
+    def write(self, bucket, key, body):
+        path = os.path.join(bucket, key)
+        dname = os.path.dirname(path)
+
+        if not os.path.exists(dname):
+            os.makedirs(dname)
+
+        with open(path, 'w+') as fd:
+            fd.write(body)
+
+
 class TestReproShell(TestDvc):
     def test(self):
         if os.name == 'nt':