mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
erepo: refactor download/fetch logic (#5234)
* erepo: don't use _get_tree_for * erepo: refactor download/fetch logic * dvc: unify handling existing output
This commit is contained in:
@@ -92,10 +92,9 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
|
||||
|
||||
dest = tmp_dir / "file"
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
_, _, save_infos = repo.fetch_external(
|
||||
[os.path.join("subdir", "file")]
|
||||
repo.repo_tree.download(
|
||||
PathInfo(repo.root_dir) / "subdir" / "file", PathInfo(dest),
|
||||
)
|
||||
repo.cache.local.checkout(PathInfo(dest), save_infos[0])
|
||||
|
||||
assert dest.is_file()
|
||||
assert dest.read_text() == "contents"
|
||||
@@ -188,10 +187,19 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
|
||||
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
|
||||
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.get_external("dir", "out")
|
||||
repo.repo_tree.download(
|
||||
PathInfo(repo.root_dir) / "dir",
|
||||
PathInfo(tmp_dir / "out"),
|
||||
follow_subrepos=False,
|
||||
)
|
||||
expected_files = {"foo": "foo", "bar": "bar", ".gitignore": "/foo\n"}
|
||||
assert (tmp_dir / "out").read_text() == expected_files
|
||||
|
||||
# clear cache to test saving to cache
|
||||
cache_dir = tmp_dir / repo.cache.local.cache_dir
|
||||
remove(cache_dir)
|
||||
makedirs(cache_dir)
|
||||
|
||||
expected_hash = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir")
|
||||
assert (
|
||||
repo.repo_tree.get_hash(
|
||||
@@ -200,16 +208,18 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
|
||||
== expected_hash
|
||||
)
|
||||
|
||||
# clear cache to test `fetch_external` again
|
||||
cache_dir = tmp_dir / repo.cache.local.cache_dir
|
||||
remove(cache_dir)
|
||||
makedirs(cache_dir)
|
||||
|
||||
assert repo.fetch_external(["dir"]) == (
|
||||
len(expected_files),
|
||||
0,
|
||||
[expected_hash],
|
||||
repo.cache.local.save(
|
||||
PathInfo(repo.root_dir) / "dir",
|
||||
repo.repo_tree,
|
||||
expected_hash,
|
||||
link=False,
|
||||
)
|
||||
assert set(cache_dir.glob("*/*")) == {
|
||||
cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir",
|
||||
cache_dir / "37" / "b51d194a7513e45b56f6524f2d51f2",
|
||||
cache_dir / "94" / "7d2b84e5aa88170e80dff467a5bfb6",
|
||||
cache_dir / "ac" / "bd18db4cc2f85cedef654fccc4a4d8",
|
||||
}
|
||||
|
||||
|
||||
def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
|
||||
@@ -223,6 +233,10 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
|
||||
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
|
||||
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.get_external("dir", "out")
|
||||
repo.repo_tree.download(
|
||||
PathInfo(repo.root_dir) / "dir",
|
||||
PathInfo(tmp_dir / "out"),
|
||||
follow_subrepos=False,
|
||||
)
|
||||
# subrepo files should not be here
|
||||
assert (tmp_dir / "out").read_text() == scm_files
|
||||
|
||||
+18
-48
@@ -1,12 +1,9 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
import mock
|
||||
import pytest
|
||||
|
||||
from dvc.cache import Cache
|
||||
from dvc.exceptions import ConfirmRemoveError, PathMissingError
|
||||
from dvc.external_repo import IsADVCRepoError
|
||||
from dvc.main import main
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.get import GetDVCFileError
|
||||
@@ -34,46 +31,13 @@ def test_get_repo_file_replace_without_confirmation(tmp_dir, erepo_dir):
|
||||
|
||||
Repo.get(os.fspath(erepo_dir), "file", "file_imported")
|
||||
# getting another file with a name that already exists in Repo.
|
||||
with pytest.raises(ConfirmRemoveError):
|
||||
with pytest.raises(FileExistsError):
|
||||
Repo.get(os.fspath(erepo_dir), "file2", "file_imported")
|
||||
|
||||
assert os.path.isfile("file_imported")
|
||||
assert (tmp_dir / "file_imported").read_text() == "contents"
|
||||
|
||||
|
||||
def test_get_repo_file_replace_yes(tmp_dir, erepo_dir):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("file", "contents", commit="create file")
|
||||
erepo_dir.dvc_gen(
|
||||
"file2", "something different", commit="create file2"
|
||||
)
|
||||
|
||||
Repo.get(os.fspath(erepo_dir), "file", "file_imported")
|
||||
# getting another file with a name that already exists in Repo.
|
||||
with mock.patch("dvc.prompt.confirm", return_value=True):
|
||||
Repo.get(os.fspath(erepo_dir), "file2", "file_imported")
|
||||
|
||||
assert os.path.isfile("file_imported")
|
||||
assert (tmp_dir / "file_imported").read_text() == "something different"
|
||||
|
||||
|
||||
def test_get_repo_file_replace_no(tmp_dir, erepo_dir):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("file", "contents", commit="create file")
|
||||
erepo_dir.dvc_gen(
|
||||
"file2", "something different", commit="create file2"
|
||||
)
|
||||
|
||||
Repo.get(os.fspath(erepo_dir), "file", "file_imported")
|
||||
# getting another file with a name that already exists in Repo.
|
||||
with mock.patch("dvc.prompt.confirm", return_value=False):
|
||||
with pytest.raises(ConfirmRemoveError):
|
||||
Repo.get(os.fspath(erepo_dir), "file2", "file_imported")
|
||||
|
||||
assert os.path.isfile("file_imported")
|
||||
assert (tmp_dir / "file_imported").read_text() == "contents"
|
||||
|
||||
|
||||
def test_get_repo_dir(tmp_dir, erepo_dir):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen({"dir": {"file": "contents"}}, commit="create dir")
|
||||
@@ -187,17 +151,17 @@ def test_non_cached_output(tmp_dir, erepo_dir):
|
||||
|
||||
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
|
||||
def test_absolute_file_outside_repo(tmp_dir, erepo_dir):
|
||||
with pytest.raises(PathMissingError):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
Repo.get(os.fspath(erepo_dir), "/root/")
|
||||
|
||||
|
||||
def test_absolute_file_outside_git_repo(tmp_dir, git_dir):
|
||||
with pytest.raises(PathMissingError):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
Repo.get(os.fspath(git_dir), "/root/")
|
||||
|
||||
|
||||
def test_unknown_path(tmp_dir, erepo_dir):
|
||||
with pytest.raises(PathMissingError):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
Repo.get(os.fspath(erepo_dir), "a_non_existing_file")
|
||||
|
||||
|
||||
@@ -345,17 +309,23 @@ def test_granular_get_from_subrepos(tmp_dir, erepo_dir):
|
||||
assert (tmp_dir / "out").read_text() == "bar"
|
||||
|
||||
|
||||
def test_try_to_get_complete_repo(tmp_dir, dvc, erepo_dir):
|
||||
def test_get_complete_repo(tmp_dir, dvc, erepo_dir):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen({"foo": "foo"}, commit="add foo")
|
||||
|
||||
subrepo = erepo_dir / "subrepo"
|
||||
make_subrepo(subrepo, erepo_dir.scm)
|
||||
with subrepo.chdir():
|
||||
subrepo.dvc_gen({"dir": {"bar": "bar"}}, commit="files in subrepo")
|
||||
|
||||
expected_message = "Cannot fetch a complete DVC repository"
|
||||
with pytest.raises(IsADVCRepoError) as exc_info:
|
||||
Repo.get(os.fspath(erepo_dir), "subrepo", out="out")
|
||||
assert f"{expected_message} 'subrepo'" == str(exc_info.value)
|
||||
Repo.get(os.fspath(erepo_dir), "subrepo", out="out_sub")
|
||||
assert (tmp_dir / "out_sub").read_text() == {
|
||||
".gitignore": "/dir\n",
|
||||
"dir": {"bar": "bar"},
|
||||
}
|
||||
|
||||
with pytest.raises(IsADVCRepoError) as exc_info:
|
||||
Repo.get(os.fspath(erepo_dir), ".", out="out")
|
||||
assert expected_message == str(exc_info.value)
|
||||
Repo.get(os.fspath(erepo_dir), ".", out="out")
|
||||
assert (tmp_dir / "out").read_text() == {
|
||||
".gitignore": "/foo\n",
|
||||
"foo": "foo",
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.dependency.base import DependencyDoesNotExistError
|
||||
@@ -24,10 +26,12 @@ def test_get_dir(tmp_dir):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
|
||||
def test_get_url_to_dir(tmp_dir, dname):
|
||||
tmp_dir.gen({"foo": "foo contents", "dir": {"subdir": {}}})
|
||||
def test_get_url_to_dir(
|
||||
tmp_dir, tmp_path_factory, dname,
|
||||
):
|
||||
tmp_dir.gen({"src": {"foo": "foo contents"}, "dir": {"subdir": {}}})
|
||||
|
||||
Repo.get_url("foo", dname)
|
||||
Repo.get_url(os.path.join("src", "foo"), dname)
|
||||
|
||||
assert (tmp_dir / dname).is_dir()
|
||||
assert (tmp_dir / dname / "foo").read_text() == "foo contents"
|
||||
|
||||
+18
-12
@@ -9,8 +9,7 @@ import dvc.data_cloud as cloud
|
||||
from dvc.cache import Cache
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.exceptions import CollectCacheError, DownloadError, PathMissingError
|
||||
from dvc.external_repo import IsADVCRepoError
|
||||
from dvc.exceptions import CollectCacheError, DownloadError
|
||||
from dvc.stage.exceptions import StagePathNotFoundError
|
||||
from dvc.system import System
|
||||
from dvc.utils.fs import makedirs, remove
|
||||
@@ -339,6 +338,7 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
|
||||
scm.tag("ref-to-branch")
|
||||
|
||||
# Overwrite via import
|
||||
(tmp_dir / "foo_imported").unlink()
|
||||
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", rev="master")
|
||||
|
||||
remove(stage.outs[0].cache_path)
|
||||
@@ -347,11 +347,11 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
|
||||
|
||||
|
||||
def test_import_non_existing(erepo_dir, tmp_dir, dvc):
|
||||
with pytest.raises(PathMissingError):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
|
||||
|
||||
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
|
||||
with pytest.raises(PathMissingError):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
|
||||
|
||||
|
||||
@@ -497,20 +497,26 @@ def test_pull_imported_stage_from_subrepos(
|
||||
assert (tmp_dir / "out").read_text() == files[key]
|
||||
|
||||
|
||||
def test_try_import_complete_repo(tmp_dir, dvc, erepo_dir):
|
||||
def test_import_complete_repo(tmp_dir, dvc, erepo_dir):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen({"foo": "foo"}, commit="add foo")
|
||||
|
||||
subrepo = erepo_dir / "subrepo"
|
||||
make_subrepo(subrepo, erepo_dir.scm)
|
||||
with subrepo.chdir():
|
||||
subrepo.dvc_gen({"dir": {"bar": "bar"}}, commit="files in subrepo")
|
||||
|
||||
expected_message = "Cannot fetch a complete DVC repository"
|
||||
with pytest.raises(IsADVCRepoError) as exc_info:
|
||||
dvc.imp(os.fspath(erepo_dir), "subrepo", out="out")
|
||||
assert f"{expected_message} 'subrepo'" == str(exc_info.value)
|
||||
dvc.imp(os.fspath(erepo_dir), "subrepo", out="out_sub")
|
||||
assert (tmp_dir / "out_sub").read_text() == {
|
||||
".gitignore": "/dir\n",
|
||||
"dir": {"bar": "bar"},
|
||||
}
|
||||
|
||||
with pytest.raises(IsADVCRepoError) as exc_info:
|
||||
dvc.imp(os.fspath(erepo_dir), os.curdir, out="out")
|
||||
assert expected_message == str(exc_info.value)
|
||||
dvc.imp(os.fspath(erepo_dir), os.curdir, out="out")
|
||||
assert (tmp_dir / "out").read_text() == {
|
||||
".gitignore": "/foo\n",
|
||||
"foo": "foo",
|
||||
}
|
||||
|
||||
|
||||
def test_import_with_no_exec(tmp_dir, dvc, erepo_dir):
|
||||
|
||||
@@ -69,7 +69,7 @@ class TestImportFilename(TestDvc):
|
||||
os.mkdir("sub")
|
||||
|
||||
path = os.path.join("sub", "bar.dvc")
|
||||
ret = main(["import-url", "--file", path, self.external_source])
|
||||
ret = main(["import-url", "--file", path, self.external_source, "out"])
|
||||
self.assertEqual(0, ret)
|
||||
self.assertTrue(os.path.exists(path))
|
||||
|
||||
|
||||
@@ -893,6 +893,7 @@ class TestReproAlreadyCached(TestRepro):
|
||||
self.assertNotEqual(run_out.hash_info, repro_out.hash_info)
|
||||
|
||||
def test_force_import(self):
|
||||
remove(self.BAR)
|
||||
ret = main(["import-url", self.FOO, self.BAR])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ from unittest.mock import call
|
||||
import pytest
|
||||
|
||||
from dvc.external_repo import external_repo
|
||||
from dvc.path_info import PathInfo
|
||||
from tests.unit.tree.test_repo import make_subrepo
|
||||
|
||||
|
||||
@@ -96,6 +97,13 @@ def test_fetch_external_repo_jobs(tmp_dir, scm, mocker, dvc, local_remote):
|
||||
|
||||
with external_repo(str(tmp_dir)) as repo:
|
||||
spy = mocker.spy(repo.cloud, "pull")
|
||||
repo.fetch_external(["dir1"], jobs=3)
|
||||
|
||||
dvc.cache.local.save(
|
||||
PathInfo(repo.root_dir) / "dir1",
|
||||
repo.repo_tree,
|
||||
None,
|
||||
follow_subrepos=False,
|
||||
jobs=3,
|
||||
)
|
||||
run_jobs = tuple(spy.call_args_list[0])[1].get("jobs")
|
||||
assert run_jobs == 3
|
||||
|
||||
@@ -377,7 +377,9 @@ def test_subrepos(tmp_dir, scm, dvc):
|
||||
True,
|
||||
[
|
||||
PathInfo("dir") / "repo" / "foo.dvc",
|
||||
PathInfo("dir") / "repo" / ".dvcignore",
|
||||
PathInfo("dir") / "repo" / "dir1.dvc",
|
||||
PathInfo("dir") / "repo2" / ".dvcignore",
|
||||
PathInfo("dir") / "repo2" / "lorem.dvc",
|
||||
PathInfo("dir") / "repo2" / "dir2.dvc",
|
||||
],
|
||||
@@ -409,12 +411,10 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
|
||||
PathInfo("dir") / "repo",
|
||||
PathInfo("dir") / "repo.txt",
|
||||
PathInfo("dir") / "repo2",
|
||||
PathInfo("dir") / "repo" / ".dvcignore",
|
||||
PathInfo("dir") / "repo" / ".gitignore",
|
||||
PathInfo("dir") / "repo" / "foo",
|
||||
PathInfo("dir") / "repo" / "dir1",
|
||||
PathInfo("dir") / "repo" / "dir1" / "bar",
|
||||
PathInfo("dir") / "repo2" / ".dvcignore",
|
||||
PathInfo("dir") / "repo2" / ".gitignore",
|
||||
PathInfo("dir") / "repo2" / "lorem",
|
||||
PathInfo("dir") / "repo2" / "dir2",
|
||||
@@ -583,7 +583,7 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos):
|
||||
for repo_dir in subrepos:
|
||||
make_subrepo(repo_dir, scm)
|
||||
|
||||
extras = {".dvcignore", ".gitignore"} # these files are always there
|
||||
extras = {".gitignore"} # these files are always there
|
||||
expected = {}
|
||||
for repo_dir in subrepos + [tmp_dir]:
|
||||
base = os.path.basename(repo_dir)
|
||||
|
||||
Reference in New Issue
Block a user