get: copy regular files

Allows `dvc get` to copy regular files or directories.

fixes: #2515

dvc/repo/get.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import shutil
 
 import shortuuid
 
@@ -29,32 +30,41 @@ def get(url, path, out=None, rev=None):
     # reflink and/or hardlink. Not using tempfile.TemporaryDirectory
     # because it will create a symlink to tmpfs, which defeats the purpose
     # and won't work with reflink/hardlink.
-    dpath = os.path.dirname(os.path.abspath(out))
+    abspath = os.path.abspath(out)
+    dpath = os.path.dirname(abspath)
     tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid()))
     try:
         with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo:
-            # Note: we need to replace state, because in case of getting DVC
-            # dependency on CIFS or NFS filesystems, sqlite-based state
-            # will be unable to obtain lock
-            repo.state = StateNoop()
+            tmp_repo_path = os.path.join(repo.root_dir, path)
+            if os.path.exists(tmp_repo_path):
+                if os.path.isdir(tmp_repo_path):
+                    shutil.copytree(tmp_repo_path, abspath)
+                else:
+                    shutil.copy2(tmp_repo_path, abspath)
+            else:
+                # Note: we need to replace state, because in case of getting
+                # DVC dependency on CIFS or NFS filesystems, sqlite-based state
+                # will be unable to obtain lock
+                repo.state = StateNoop()
 
-            # Try any links possible to avoid data duplication.
-            #
-            # Not using symlink, because we need to remove cache after we are
-            # done, and to make that work we would have to copy data over
-            # anyway before removing the cache, so we might just copy it
-            # right away.
-            #
-            # Also, we can't use theoretical "move" link type here, because
-            # the same cache file might be used a few times in a directory.
-            repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
+                # Try any links possible to avoid data duplication.
+                #
+                # Not using symlink, because we need to remove cache after we
+                # are done, and to make that work we would have to copy data
+                # over anyway before removing the cache, so we might just copy
+                # it right away.
+                #
+                # Also, we can't use theoretical "move" link type here, because
+                # the same cache file might be used a few times in a directory.
+                repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
 
-            o = repo.find_out_by_relpath(path)
-            with repo.state:
-                repo.cloud.pull(o.get_used_cache())
-            o.path_info = PathInfo(os.path.abspath(out))
-            with o.repo.state:
-                o.checkout()
+                o = repo.find_out_by_relpath(path)
+
+                with repo.state:
+                    repo.cloud.pull(o.get_used_cache())
+                o.path_info = PathInfo(os.path.abspath(out))
+                with o.repo.state:
+                    o.checkout()
 
     except NotDvcRepoError:
         raise UrlNotDvcRepoError(url)

tests/basic_env.py

@@ -41,7 +41,8 @@ class TestDirFixture(object):
     # in tests, we replace foo with bar, so we need to make sure that when we
     # modify a file in our tests, its content length changes.
     BAR_CONTENTS = BAR + "r"
-    CODE = "code.py"
+    CODE_DIR = "code"
+    CODE = "code/code.py"
     CODE_CONTENTS = (
         "import sys\nimport shutil\n"
         "shutil.copyfile(sys.argv[1], sys.argv[2])"
@@ -90,6 +91,7 @@ class TestDirFixture(object):
         self._pushd(self._root_dir)
         self.create(self.FOO, self.FOO_CONTENTS)
         self.create(self.BAR, self.BAR_CONTENTS)
+        os.mkdir(self.CODE_DIR)
         self.create(self.CODE, self.CODE_CONTENTS)
         os.mkdir(self.DATA_DIR)
         os.mkdir(self.DATA_SUB_DIR)

tests/func/test_get.py

@@ -38,7 +38,29 @@ def test_get_repo_dir(erepo):
     trees_equal(src, dst)
 
 
-def test_cache_type_is_properly_overridden(erepo):
+def test_get_regular_file(repo_dir, erepo):
+    src = erepo.CODE
+    dst = erepo.CODE
+
+    Repo.get(erepo.root_dir, src, dst)
+
+    assert os.path.exists(dst)
+    assert os.path.isfile(dst)
+    assert filecmp.cmp(repo_dir.CODE, dst, shallow=False)
+
+
+def test_get_regular_dir(repo_dir, erepo):
+    src = erepo.CODE_DIR
+    dst = erepo.CODE_DIR
+
+    Repo.get(erepo.root_dir, src, dst)
+
+    assert os.path.exists(dst)
+    assert os.path.isdir(dst)
+    trees_equal(src, dst)
+
+
+def test_cache_type_is_properly_overridden(repo_dir, erepo):
     erepo.dvc.config.set(
         Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink"
     )

tests/func/test_repro.py

@@ -738,9 +738,9 @@ class TestCmdReproChdirCwdBackwardCompatible(TestDvc):
         os.mkdir(dname)
         foo = os.path.join(dname, self.FOO)
         bar = os.path.join(dname, self.BAR)
-        code = os.path.join(dname, self.CODE)
+        code_dir = os.path.join(dname, self.CODE_DIR)
         shutil.copyfile(self.FOO, foo)
-        shutil.copyfile(self.CODE, code)
+        shutil.copytree(self.CODE_DIR, code_dir)
 
         ret = main(
             [
@@ -776,9 +776,9 @@ class TestCmdReproChdir(TestDvc):
         os.mkdir(dname)
         foo = os.path.join(dname, self.FOO)
         bar = os.path.join(dname, self.BAR)
-        code = os.path.join(dname, self.CODE)
+        code_dir = os.path.join(dname, self.CODE_DIR)
         shutil.copyfile(self.FOO, foo)
-        shutil.copyfile(self.CODE, code)
+        shutil.copytree(self.CODE_DIR, code_dir)
 
         ret = main(
             [