erepo: refactor download/fetch logic (#5234)

* erepo: don't use _get_tree_for

* erepo: refactor download/fetch logic

* dvc: unify handling existing output
This commit is contained in:
Ruslan Kuprieiev
2021-01-10 10:48:36 +02:00
committed by GitHub
parent 6b05cc5e24
commit c83a4ea406
16 changed files with 167 additions and 210 deletions
+28 -14
View File
@@ -92,10 +92,9 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
dest = tmp_dir / "file"
with external_repo(os.fspath(erepo_dir)) as repo:
_, _, save_infos = repo.fetch_external(
[os.path.join("subdir", "file")]
repo.repo_tree.download(
PathInfo(repo.root_dir) / "subdir" / "file", PathInfo(dest),
)
repo.cache.local.checkout(PathInfo(dest), save_infos[0])
assert dest.is_file()
assert dest.read_text() == "contents"
@@ -188,10 +187,19 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo:
repo.get_external("dir", "out")
repo.repo_tree.download(
PathInfo(repo.root_dir) / "dir",
PathInfo(tmp_dir / "out"),
follow_subrepos=False,
)
expected_files = {"foo": "foo", "bar": "bar", ".gitignore": "/foo\n"}
assert (tmp_dir / "out").read_text() == expected_files
# clear cache to test saving to cache
cache_dir = tmp_dir / repo.cache.local.cache_dir
remove(cache_dir)
makedirs(cache_dir)
expected_hash = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir")
assert (
repo.repo_tree.get_hash(
@@ -200,16 +208,18 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
== expected_hash
)
# clear cache to test `fetch_external` again
cache_dir = tmp_dir / repo.cache.local.cache_dir
remove(cache_dir)
makedirs(cache_dir)
assert repo.fetch_external(["dir"]) == (
len(expected_files),
0,
[expected_hash],
repo.cache.local.save(
PathInfo(repo.root_dir) / "dir",
repo.repo_tree,
expected_hash,
link=False,
)
assert set(cache_dir.glob("*/*")) == {
cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir",
cache_dir / "37" / "b51d194a7513e45b56f6524f2d51f2",
cache_dir / "94" / "7d2b84e5aa88170e80dff467a5bfb6",
cache_dir / "ac" / "bd18db4cc2f85cedef654fccc4a4d8",
}
def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
@@ -223,6 +233,10 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo:
repo.get_external("dir", "out")
repo.repo_tree.download(
PathInfo(repo.root_dir) / "dir",
PathInfo(tmp_dir / "out"),
follow_subrepos=False,
)
# subrepo files should not be here
assert (tmp_dir / "out").read_text() == scm_files
+18 -48
View File
@@ -1,12 +1,9 @@
import logging
import os
import mock
import pytest
from dvc.cache import Cache
from dvc.exceptions import ConfirmRemoveError, PathMissingError
from dvc.external_repo import IsADVCRepoError
from dvc.main import main
from dvc.repo import Repo
from dvc.repo.get import GetDVCFileError
@@ -34,46 +31,13 @@ def test_get_repo_file_replace_without_confirmation(tmp_dir, erepo_dir):
Repo.get(os.fspath(erepo_dir), "file", "file_imported")
# getting another file with a name that already exists in Repo.
with pytest.raises(ConfirmRemoveError):
with pytest.raises(FileExistsError):
Repo.get(os.fspath(erepo_dir), "file2", "file_imported")
assert os.path.isfile("file_imported")
assert (tmp_dir / "file_imported").read_text() == "contents"
def test_get_repo_file_replace_yes(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "contents", commit="create file")
erepo_dir.dvc_gen(
"file2", "something different", commit="create file2"
)
Repo.get(os.fspath(erepo_dir), "file", "file_imported")
# getting another file with a name that already exists in Repo.
with mock.patch("dvc.prompt.confirm", return_value=True):
Repo.get(os.fspath(erepo_dir), "file2", "file_imported")
assert os.path.isfile("file_imported")
assert (tmp_dir / "file_imported").read_text() == "something different"
def test_get_repo_file_replace_no(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "contents", commit="create file")
erepo_dir.dvc_gen(
"file2", "something different", commit="create file2"
)
Repo.get(os.fspath(erepo_dir), "file", "file_imported")
# getting another file with a name that already exists in Repo.
with mock.patch("dvc.prompt.confirm", return_value=False):
with pytest.raises(ConfirmRemoveError):
Repo.get(os.fspath(erepo_dir), "file2", "file_imported")
assert os.path.isfile("file_imported")
assert (tmp_dir / "file_imported").read_text() == "contents"
def test_get_repo_dir(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen({"dir": {"file": "contents"}}, commit="create dir")
@@ -187,17 +151,17 @@ def test_non_cached_output(tmp_dir, erepo_dir):
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
def test_absolute_file_outside_repo(tmp_dir, erepo_dir):
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
Repo.get(os.fspath(erepo_dir), "/root/")
def test_absolute_file_outside_git_repo(tmp_dir, git_dir):
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
Repo.get(os.fspath(git_dir), "/root/")
def test_unknown_path(tmp_dir, erepo_dir):
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
Repo.get(os.fspath(erepo_dir), "a_non_existing_file")
@@ -345,17 +309,23 @@ def test_granular_get_from_subrepos(tmp_dir, erepo_dir):
assert (tmp_dir / "out").read_text() == "bar"
def test_try_to_get_complete_repo(tmp_dir, dvc, erepo_dir):
def test_get_complete_repo(tmp_dir, dvc, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen({"foo": "foo"}, commit="add foo")
subrepo = erepo_dir / "subrepo"
make_subrepo(subrepo, erepo_dir.scm)
with subrepo.chdir():
subrepo.dvc_gen({"dir": {"bar": "bar"}}, commit="files in subrepo")
expected_message = "Cannot fetch a complete DVC repository"
with pytest.raises(IsADVCRepoError) as exc_info:
Repo.get(os.fspath(erepo_dir), "subrepo", out="out")
assert f"{expected_message} 'subrepo'" == str(exc_info.value)
Repo.get(os.fspath(erepo_dir), "subrepo", out="out_sub")
assert (tmp_dir / "out_sub").read_text() == {
".gitignore": "/dir\n",
"dir": {"bar": "bar"},
}
with pytest.raises(IsADVCRepoError) as exc_info:
Repo.get(os.fspath(erepo_dir), ".", out="out")
assert expected_message == str(exc_info.value)
Repo.get(os.fspath(erepo_dir), ".", out="out")
assert (tmp_dir / "out").read_text() == {
".gitignore": "/foo\n",
"foo": "foo",
}
+7 -3
View File
@@ -1,3 +1,5 @@
import os
import pytest
from dvc.dependency.base import DependencyDoesNotExistError
@@ -24,10 +26,12 @@ def test_get_dir(tmp_dir):
@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
def test_get_url_to_dir(tmp_dir, dname):
tmp_dir.gen({"foo": "foo contents", "dir": {"subdir": {}}})
def test_get_url_to_dir(
tmp_dir, tmp_path_factory, dname,
):
tmp_dir.gen({"src": {"foo": "foo contents"}, "dir": {"subdir": {}}})
Repo.get_url("foo", dname)
Repo.get_url(os.path.join("src", "foo"), dname)
assert (tmp_dir / dname).is_dir()
assert (tmp_dir / dname / "foo").read_text() == "foo contents"
+18 -12
View File
@@ -9,8 +9,7 @@ import dvc.data_cloud as cloud
from dvc.cache import Cache
from dvc.config import NoRemoteError
from dvc.dvcfile import Dvcfile
from dvc.exceptions import CollectCacheError, DownloadError, PathMissingError
from dvc.external_repo import IsADVCRepoError
from dvc.exceptions import CollectCacheError, DownloadError
from dvc.stage.exceptions import StagePathNotFoundError
from dvc.system import System
from dvc.utils.fs import makedirs, remove
@@ -339,6 +338,7 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
scm.tag("ref-to-branch")
# Overwrite via import
(tmp_dir / "foo_imported").unlink()
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", rev="master")
remove(stage.outs[0].cache_path)
@@ -347,11 +347,11 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
def test_import_non_existing(erepo_dir, tmp_dir, dvc):
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
@@ -497,20 +497,26 @@ def test_pull_imported_stage_from_subrepos(
assert (tmp_dir / "out").read_text() == files[key]
def test_try_import_complete_repo(tmp_dir, dvc, erepo_dir):
def test_import_complete_repo(tmp_dir, dvc, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen({"foo": "foo"}, commit="add foo")
subrepo = erepo_dir / "subrepo"
make_subrepo(subrepo, erepo_dir.scm)
with subrepo.chdir():
subrepo.dvc_gen({"dir": {"bar": "bar"}}, commit="files in subrepo")
expected_message = "Cannot fetch a complete DVC repository"
with pytest.raises(IsADVCRepoError) as exc_info:
dvc.imp(os.fspath(erepo_dir), "subrepo", out="out")
assert f"{expected_message} 'subrepo'" == str(exc_info.value)
dvc.imp(os.fspath(erepo_dir), "subrepo", out="out_sub")
assert (tmp_dir / "out_sub").read_text() == {
".gitignore": "/dir\n",
"dir": {"bar": "bar"},
}
with pytest.raises(IsADVCRepoError) as exc_info:
dvc.imp(os.fspath(erepo_dir), os.curdir, out="out")
assert expected_message == str(exc_info.value)
dvc.imp(os.fspath(erepo_dir), os.curdir, out="out")
assert (tmp_dir / "out").read_text() == {
".gitignore": "/foo\n",
"foo": "foo",
}
def test_import_with_no_exec(tmp_dir, dvc, erepo_dir):
+1 -1
View File
@@ -69,7 +69,7 @@ class TestImportFilename(TestDvc):
os.mkdir("sub")
path = os.path.join("sub", "bar.dvc")
ret = main(["import-url", "--file", path, self.external_source])
ret = main(["import-url", "--file", path, self.external_source, "out"])
self.assertEqual(0, ret)
self.assertTrue(os.path.exists(path))
+1
View File
@@ -893,6 +893,7 @@ class TestReproAlreadyCached(TestRepro):
self.assertNotEqual(run_out.hash_info, repro_out.hash_info)
def test_force_import(self):
remove(self.BAR)
ret = main(["import-url", self.FOO, self.BAR])
self.assertEqual(ret, 0)
+9 -1
View File
@@ -4,6 +4,7 @@ from unittest.mock import call
import pytest
from dvc.external_repo import external_repo
from dvc.path_info import PathInfo
from tests.unit.tree.test_repo import make_subrepo
@@ -96,6 +97,13 @@ def test_fetch_external_repo_jobs(tmp_dir, scm, mocker, dvc, local_remote):
with external_repo(str(tmp_dir)) as repo:
spy = mocker.spy(repo.cloud, "pull")
repo.fetch_external(["dir1"], jobs=3)
dvc.cache.local.save(
PathInfo(repo.root_dir) / "dir1",
repo.repo_tree,
None,
follow_subrepos=False,
jobs=3,
)
run_jobs = tuple(spy.call_args_list[0])[1].get("jobs")
assert run_jobs == 3
+3 -3
View File
@@ -377,7 +377,9 @@ def test_subrepos(tmp_dir, scm, dvc):
True,
[
PathInfo("dir") / "repo" / "foo.dvc",
PathInfo("dir") / "repo" / ".dvcignore",
PathInfo("dir") / "repo" / "dir1.dvc",
PathInfo("dir") / "repo2" / ".dvcignore",
PathInfo("dir") / "repo2" / "lorem.dvc",
PathInfo("dir") / "repo2" / "dir2.dvc",
],
@@ -409,12 +411,10 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
PathInfo("dir") / "repo",
PathInfo("dir") / "repo.txt",
PathInfo("dir") / "repo2",
PathInfo("dir") / "repo" / ".dvcignore",
PathInfo("dir") / "repo" / ".gitignore",
PathInfo("dir") / "repo" / "foo",
PathInfo("dir") / "repo" / "dir1",
PathInfo("dir") / "repo" / "dir1" / "bar",
PathInfo("dir") / "repo2" / ".dvcignore",
PathInfo("dir") / "repo2" / ".gitignore",
PathInfo("dir") / "repo2" / "lorem",
PathInfo("dir") / "repo2" / "dir2",
@@ -583,7 +583,7 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos):
for repo_dir in subrepos:
make_subrepo(repo_dir, scm)
extras = {".dvcignore", ".gitignore"} # these files are always there
extras = {".gitignore"} # these files are always there
expected = {}
for repo_dir in subrepos + [tmp_dir]:
base = os.path.basename(repo_dir)