import: allow downloading regular files/dirs tracked by git

Fixes #2862

dvc/dependency/repo.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import copy
+import os
 from contextlib import contextmanager
 
 from funcy import merge
@@ -8,6 +9,9 @@ from funcy import merge
 from .local import DependencyLOCAL
 from dvc.external_repo import external_repo
 from dvc.utils.compat import str
+from dvc.exceptions import OutputNotFoundError
+from dvc.exceptions import PathMissingError
+from dvc.utils.fs import fs_copy
 
 
 class DependencyREPO(DependencyLOCAL):
@@ -72,10 +76,42 @@ class DependencyREPO(DependencyLOCAL):
 
         return out
 
+    @staticmethod
+    def _is_git_file(repo, path):
+        if not os.path.isabs(path):
+            try:
+                output = repo.find_out_by_relpath(path)
+                if not output.use_cache:
+                    return True
+            except OutputNotFoundError:
+                return True
+        return False
+
+    def _copy_if_git_file(self, to_path):
+        src_path = self.def_path
+        with self._make_repo(
+            cache_dir=self.repo.cache.local.cache_dir
+        ) as repo:
+            if not self._is_git_file(repo, src_path):
+                return False
+
+            src_full_path = os.path.join(repo.root_dir, src_path)
+            dst_full_path = os.path.abspath(to_path)
+            fs_copy(src_full_path, dst_full_path)
+        return True
+
     def download(self, to):
-        out = self.fetch()
-        to.info = copy.copy(out.info)
-        to.checkout()
+        try:
+            if self._copy_if_git_file(to.fspath):
+                return
+
+            out = self.fetch()
+            to.info = copy.copy(out.info)
+            to.checkout()
+        except (FileNotFoundError):
+            raise PathMissingError(
+                self.def_path, self.def_repo[self.PARAM_URL]
+            )
 
     def update(self):
         with self._make_repo(rev_lock=None) as repo:

dvc/exceptions.py

@@ -344,3 +344,12 @@ class NoOutputInExternalRepoError(DvcException):
 class HTTPError(DvcException):
     def __init__(self, code, reason):
         super(HTTPError, self).__init__("'{} {}'".format(code, reason))
+
+
+class PathMissingError(DvcException):
+    def __init__(self, path, repo):
+        msg = (
+            "The path '{}' does not exist in the target repository '{}'"
+            " neighther as an output nor a git-handled file."
+        )
+        super(PathMissingError, self).__init__(msg.format(path, repo))

dvc/repo/get.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import shutil
 from dvc.utils.compat import FileNotFoundError
 
 import shortuuid
@@ -10,12 +9,14 @@ from dvc.exceptions import (
     NotDvcRepoError,
     OutputNotFoundError,
     UrlNotDvcRepoError,
+    PathMissingError,
 )
 from dvc.external_repo import external_repo
 from dvc.path_info import PathInfo
 from dvc.stage import Stage
 from dvc.utils import resolve_output
 from dvc.utils.fs import remove
+from dvc.utils.fs import fs_copy
 
 logger = logging.getLogger(__name__)
 
@@ -28,15 +29,6 @@ class GetDVCFileError(DvcException):
         )
 
 
-class PathMissingError(DvcException):
-    def __init__(self, path, repo):
-        msg = (
-            "The path '{}' does not exist in the target repository '{}'"
-            " neighther as an output nor a git-handled file."
-        )
-        super(PathMissingError, self).__init__(msg.format(path, repo))
-
-
 @staticmethod
 def get(url, path, out=None, rev=None):
     out = resolve_output(path, out)
@@ -76,7 +68,7 @@ def get(url, path, out=None, rev=None):
                 if os.path.isabs(path):
                     raise FileNotFoundError
 
-                _copy(os.path.join(repo.root_dir, path), out)
+                fs_copy(os.path.join(repo.root_dir, path), out)
 
     except (OutputNotFoundError, FileNotFoundError):
         raise PathMissingError(path, url)
@@ -94,10 +86,3 @@ def _get_cached(repo, output, out):
         # This might happen when pull haven't really pulled all the files
         if failed:
             raise FileNotFoundError
-
-
-def _copy(src, dst):
-    if os.path.isdir(src):
-        shutil.copytree(src, dst)
-    else:
-        shutil.copy2(src, dst)

dvc/utils/fs.py

@@ -22,6 +22,13 @@ from dvc.utils.compat import str
 logger = logging.getLogger(__name__)
 
 
+def fs_copy(src, dst):
+    if os.path.isdir(src):
+        shutil.copytree(src, dst)
+    else:
+        shutil.copy2(src, dst)
+
+
 def get_inode(path):
     inode = System.inode(path)
     logger.debug("Path {} inode {}".format(path, inode))

tests/func/test_import.py

@@ -8,7 +8,9 @@ import pytest
 from mock import patch
 
 from dvc.config import Config
-from dvc.exceptions import DownloadError, NoOutputInExternalRepoError
+from dvc.exceptions import DownloadError
+from dvc.exceptions import PathMissingError
+from dvc.exceptions import NoOutputInExternalRepoError
 from dvc.stage import Stage
 from dvc.system import System
 from dvc.utils import makedirs
@@ -27,6 +29,40 @@ def test_import(git, dvc_repo, erepo):
     assert git.git.check_ignore(dst)
 
 
+def test_import_git_file(git, dvc_repo, erepo):
+    src = "some_file"
+    dst = "some_file_imported"
+
+    src_path = os.path.join(erepo.root_dir, src)
+    erepo.create(src_path, "hello")
+    erepo.dvc.scm.add([src_path])
+    erepo.dvc.scm.commit("add a regular file")
+
+    dvc_repo.imp(erepo.root_dir, src, dst)
+
+    assert os.path.exists(dst)
+    assert os.path.isfile(dst)
+    assert filecmp.cmp(src_path, dst, shallow=False)
+    assert git.git.check_ignore(dst)
+
+
+def test_import_git_dir(git, dvc_repo, erepo):
+    src = "some_directory"
+    dst = "some_directory_imported"
+
+    src_file_path = os.path.join(erepo.root_dir, src, "file.txt")
+    erepo.create(src_file_path, "hello")
+    erepo.dvc.scm.add([src_file_path])
+    erepo.dvc.scm.commit("add a regular dir")
+
+    dvc_repo.imp(erepo.root_dir, src, dst)
+
+    assert os.path.exists(dst)
+    assert os.path.isdir(dst)
+    trees_equal(os.path.join(erepo.root_dir, src), dst)
+    assert git.git.check_ignore(dst)
+
+
 def test_import_dir(git, dvc_repo, erepo):
     src = erepo.DATA_DIR
     dst = erepo.DATA_DIR + "_imported"
@@ -39,6 +75,28 @@ def test_import_dir(git, dvc_repo, erepo):
     assert git.git.check_ignore(dst)
 
 
+def test_import_non_cached(git, dvc_repo, erepo):
+    src = "non_cached_output"
+    dst = src + "_imported"
+
+    erepo.dvc.run(
+        cmd="echo hello > {}".format(src),
+        outs_no_cache=[src],
+        cwd=erepo.root_dir,
+    )
+
+    src_path = os.path.join(erepo.root_dir, src)
+    erepo.dvc.scm.add([src_path])
+    erepo.dvc.scm.commit("add a non-cached output")
+
+    dvc_repo.imp(erepo.root_dir, src, dst)
+
+    assert os.path.exists(dst)
+    assert os.path.isfile(dst)
+    assert filecmp.cmp(src_path, dst, shallow=False)
+    assert git.git.check_ignore(dst)
+
+
 def test_import_rev(git, dvc_repo, erepo):
     src = "version"
     dst = src
@@ -157,5 +215,8 @@ def test_pull_non_workspace(git, dvc_repo, erepo):
 
 
 def test_import_non_existing(dvc_repo, erepo):
-    with pytest.raises(NoOutputInExternalRepoError):
+    with pytest.raises(PathMissingError):
         dvc_repo.imp(erepo.root_dir, "invalid_output")
+    # https://github.com/iterative/dvc/pull/2837#discussion_r352123053
+    with pytest.raises(NoOutputInExternalRepoError):
+        dvc_repo.imp(erepo.root_dir, "/root/", "root")