diff --git a/dvc/cache/base.py b/dvc/cache/base.py index dfde65a4c..294fea640 100644 --- a/dvc/cache/base.py +++ b/dvc/cache/base.py @@ -1,6 +1,8 @@ +import errno import itertools import json import logging +import os from concurrent.futures import ThreadPoolExecutor from copy import copy from typing import Optional @@ -347,7 +349,15 @@ class CloudCache: self.tree.scheme, ) + if not hash_info: + hash_info = tree.get_hash(path_info, **kwargs) + if not hash_info: + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), path_info + ) + self._save(path_info, tree, hash_info, save_link, **kwargs) + return hash_info def _save(self, path_info, tree, hash_info, save_link=True, **kwargs): to_info = self.tree.hash_to_path_info(hash_info.value) diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 185aa8dfc..87b011ebb 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -77,10 +77,16 @@ class RepoDependency(LocalDependency): with self._make_repo(cache_dir=cache.cache_dir) as repo: if self.def_repo.get(self.PARAM_REV_LOCK) is None: self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev() + path_info = PathInfo(repo.root_dir) / self.def_path + hash_info = cache.save( + path_info, + repo.repo_tree, + None, + jobs=jobs, + follow_subrepos=False, + ) - _, _, cache_infos = repo.fetch_external([self.def_path], jobs=jobs) - - cache.checkout(to.path_info, cache_infos[0]) + cache.checkout(to.path_info, hash_info) def update(self, rev=None): if rev: diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 9a7ee45d4..00d5abac1 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -3,12 +3,11 @@ import os import tempfile import threading from contextlib import contextmanager -from typing import Dict, Iterable +from typing import Dict -from funcy import cached_property, reraise, retry, wrap_with +from funcy import cached_property, retry, wrap_with from dvc.exceptions import ( - DvcException, FileMissingError, NoOutputInExternalRepoError, NoRemoteInExternalRepoError, @@ -16,17 +15,12 @@ from dvc.exceptions import ( OutputNotFoundError, PathMissingError, ) -from dvc.path_info import PathInfo from dvc.repo import Repo from dvc.utils import relpath logger = logging.getLogger(__name__) -class IsADVCRepoError(DvcException): - """Raised when it is not expected to be a dvc repo.""" - - @contextmanager def external_repo(url, rev=None, for_write=False, **kwargs): from dvc.config import NoRemoteError @@ -162,85 +156,6 @@ class ExternalRepo(Repo): return self.scm.get_rev() return self.tree.rev - def _fetch_to_cache(self, path_info, repo, callback, **kwargs): - # don't support subrepo traversal as it might fail due to difference - # in remotes - tree = self._get_tree_for(repo) - cache = repo.cache.local - - hash_info = tree.get_hash( - path_info, download_callback=callback, **kwargs - ) - cache.save( - path_info, - tree, - hash_info, - save_link=False, - download_callback=callback, - ) - return hash_info - - def fetch_external(self, paths: Iterable, **kwargs): - """Fetch specified external repo paths into cache. - - Returns 3-tuple in the form - (downloaded, failed, list(cache_infos)) - where cache_infos can be used as checkout targets for the - fetched paths. - """ - download_results = [] - failed = 0 - root = PathInfo(self.root_dir) - - paths = [root / path for path in paths] - - def download_update(result): - download_results.append(result) - - hash_infos = [] - for path in paths: - with reraise(FileNotFoundError, PathMissingError(path, self.url)): - metadata = self.repo_tree.metadata(path) - - self._check_repo(path, metadata.repo) - repo = metadata.repo - hash_info = self._fetch_to_cache( - path, repo, download_update, **kwargs - ) - hash_infos.append(hash_info) - - return sum(download_results), failed, hash_infos - - def _check_repo(self, path_info, repo): - if not repo: - return - - repo_path = PathInfo(repo.root_dir) - if path_info == repo_path and isinstance(repo, Repo): - message = "Cannot fetch a complete DVC repository" - if repo.root_dir != self.root_dir: - rel = relpath(repo.root_dir, self.root_dir) - message += f" '{rel}'" - raise IsADVCRepoError(message) - - def get_external(self, path, dest): - """Convenience wrapper for fetch_external and checkout.""" - path_info = PathInfo(self.root_dir) / path - with reraise(FileNotFoundError, PathMissingError(path, self.url)): - metadata = self.repo_tree.metadata(path_info) - - self._check_repo(path_info, metadata.repo) - if metadata.output_exists: - repo = metadata.repo - cache = repo.cache.local - # fetch DVC and git files to tmpdir cache, then checkout - save_info = self._fetch_to_cache(path_info, repo, None) - cache.checkout(PathInfo(dest), save_info) - else: - # git-only folder, just copy files directly to dest - tree = self._get_tree_for(metadata.repo) # ignore subrepos - tree.copytree(path_info, dest) - def _get_tree_for(self, repo, **kwargs): """ Provides a combined tree of a single repo with dvc + git/local tree. diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 475091e28..12af77885 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -80,21 +80,36 @@ def fetch( def _fetch_external(self, repo_url, repo_rev, files, jobs): from dvc.external_repo import external_repo + from dvc.path_info import PathInfo from dvc.scm.base import CloneError - failed, downloaded = 0, 0 + failed = 0 + + results = [] + + def cb(result): + results.append(result) + cache = self.cache.local try: with external_repo( repo_url, repo_rev, cache_dir=cache.cache_dir ) as repo: - d, f, _ = repo.fetch_external(files, jobs=jobs) - downloaded += d - failed += f + root = PathInfo(repo.root_dir) + for path in files: + path_info = root / path + self.cache.local.save( + path_info, + repo.repo_tree, + None, + jobs=jobs, + download_callback=cb, + follow_subrepos=False, + ) except CloneError: failed += 1 logger.exception( "failed to fetch data for '{}'".format(", ".join(files)) ) - return downloaded, failed + return sum(results), failed diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 68474dffb..d2c5efb92 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -2,6 +2,7 @@ import logging import os from dvc.exceptions import DvcException +from dvc.path_info import PathInfo from dvc.utils import resolve_output from dvc.utils.fs import remove @@ -49,6 +50,8 @@ def get(url, path, out=None, rev=None): with external_repo( url=url, rev=rev, cache_dir=tmp_dir, cache_types=cache_types ) as repo: - repo.get_external(path, out) + from_info = PathInfo(repo.root_dir) / path + to_info = PathInfo(out) + repo.repo_tree.download(from_info, to_info, follow_subrepos=False) finally: remove(tmp_dir) diff --git a/dvc/tree/base.py b/dvc/tree/base.py index 68053c83e..ff66a2ba8 100644 --- a/dvc/tree/base.py +++ b/dvc/tree/base.py @@ -400,6 +400,7 @@ class BaseTree: file_mode=None, dir_mode=None, jobs=None, + **kwargs, ): if not hasattr(self, "_download"): raise RemoteActionNotImplemented("download", self.scheme) @@ -423,9 +424,10 @@ class BaseTree: file_mode, dir_mode, jobs, + **kwargs, ) return self._download_file( - from_info, to_info, name, no_progress_bar, file_mode, dir_mode + from_info, to_info, name, no_progress_bar, file_mode, dir_mode, ) def _download_dir( @@ -437,8 +439,9 @@ class BaseTree: file_mode, dir_mode, jobs, + **kwargs, ): - from_infos = list(self.walk_files(from_info)) + from_infos = list(self.walk_files(from_info, **kwargs)) to_infos = ( to_info / info.relative_to(from_info) for info in from_infos ) diff --git a/dvc/tree/repo.py b/dvc/tree/repo.py index 21e319714..829815ff5 100644 --- a/dvc/tree/repo.py +++ b/dvc/tree/repo.py @@ -259,6 +259,7 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method def _walk(self, repo_walk, dvc_walk=None, dvcfiles=False): from dvc.dvcfile import is_valid_filename + from dvc.ignore import DvcIgnore assert repo_walk try: @@ -277,12 +278,16 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method shared = list(dvc_set & repo_set) dirs = shared + dvc_only + repo_only + def _func(fname): + if dvcfiles: + return True + + return not ( + is_valid_filename(fname) or fname == DvcIgnore.DVCIGNORE_FILE + ) + # merge file lists - files = { - fname - for fname in dvc_fnames + repo_fnames - if dvcfiles or not is_valid_filename(fname) - } + files = set(filter(_func, dvc_fnames + repo_fnames)) yield repo_root, dirs, list(files) @@ -403,29 +408,19 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method pass return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self)[0]) - def copytree(self, top, dest): - from dvc.utils.fs import copy_fobj_to_file, makedirs + def _download( + self, from_info, to_file, name=None, no_progress_bar=False, **kwargs + ): + import shutil - top = PathInfo(top) - dest = PathInfo(dest) + from dvc.progress import Tqdm - if not self.exists(top): - raise FileNotFoundError - - if self.isfile(top): - makedirs(dest.parent, exist_ok=True) - with self.open(top, mode="rb") as fobj: - copy_fobj_to_file(fobj, dest) - return - - for root, _, files in self.walk(top): - root = PathInfo(root) - dest_dir = dest / root.relative_to(top) - makedirs(dest_dir, exist_ok=True) - for fname in files: - src = root / fname - with self.open(src, mode="rb") as fobj: - copy_fobj_to_file(fobj, dest_dir / fname) + with open(to_file, "wb+") as to_fobj: + with Tqdm.wrapattr( + to_fobj, "write", desc=name, disable=no_progress_bar, + ) as wrapped: + with self.open(from_info, "rb", **kwargs) as from_fobj: + shutil.copyfileobj(from_fobj, wrapped) @property def hash_jobs(self): # pylint: disable=invalid-overridden-method diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index f17d3c7a6..f888b76b1 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -351,14 +351,21 @@ def env2bool(var, undefined=False): def resolve_output(inp, out): + import errno from urllib.parse import urlparse name = os.path.basename(os.path.normpath(urlparse(inp).path)) if not out: - return name - if os.path.isdir(out): - return os.path.join(out, name) - return out + ret = name + elif os.path.isdir(out): + ret = os.path.join(out, name) + else: + ret = out + + if os.path.exists(ret): + raise FileExistsError(errno.EEXIST, os.strerror(errno.EEXIST), ret) + + return ret def resolve_paths(repo, out): diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index dff3f3c03..7d9ba0804 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -92,10 +92,9 @@ def test_pull_subdir_file(tmp_dir, erepo_dir): dest = tmp_dir / "file" with external_repo(os.fspath(erepo_dir)) as repo: - _, _, save_infos = repo.fetch_external( - [os.path.join("subdir", "file")] + repo.repo_tree.download( + PathInfo(repo.root_dir) / "subdir" / "file", PathInfo(dest), ) - repo.cache.local.checkout(PathInfo(dest), save_infos[0]) assert dest.is_file() assert dest.read_text() == "contents" @@ -188,10 +187,19 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo") with external_repo(os.fspath(erepo_dir)) as repo: - repo.get_external("dir", "out") + repo.repo_tree.download( + PathInfo(repo.root_dir) / "dir", + PathInfo(tmp_dir / "out"), + follow_subrepos=False, + ) expected_files = {"foo": "foo", "bar": "bar", ".gitignore": "/foo\n"} assert (tmp_dir / "out").read_text() == expected_files + # clear cache to test saving to cache + cache_dir = tmp_dir / repo.cache.local.cache_dir + remove(cache_dir) + makedirs(cache_dir) + expected_hash = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir") assert ( repo.repo_tree.get_hash( @@ -200,16 +208,18 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): == expected_hash ) - # clear cache to test `fetch_external` again - cache_dir = tmp_dir / repo.cache.local.cache_dir - remove(cache_dir) - makedirs(cache_dir) - - assert repo.fetch_external(["dir"]) == ( - len(expected_files), - 0, - [expected_hash], + repo.cache.local.save( + PathInfo(repo.root_dir) / "dir", + repo.repo_tree, + expected_hash, + link=False, ) + assert set(cache_dir.glob("*/*")) == { + cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir", + cache_dir / "37" / "b51d194a7513e45b56f6524f2d51f2", + cache_dir / "94" / "7d2b84e5aa88170e80dff467a5bfb6", + cache_dir / "ac" / "bd18db4cc2f85cedef654fccc4a4d8", + } def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir): @@ -223,6 +233,10 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir): subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo") with external_repo(os.fspath(erepo_dir)) as repo: - repo.get_external("dir", "out") + repo.repo_tree.download( + PathInfo(repo.root_dir) / "dir", + PathInfo(tmp_dir / "out"), + follow_subrepos=False, + ) # subrepo files should not be here assert (tmp_dir / "out").read_text() == scm_files diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 91bdaa006..2890e38fb 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -1,12 +1,9 @@ import logging import os -import mock import pytest from dvc.cache import Cache -from dvc.exceptions import ConfirmRemoveError, PathMissingError -from dvc.external_repo import IsADVCRepoError from dvc.main import main from dvc.repo import Repo from dvc.repo.get import GetDVCFileError @@ -34,46 +31,13 @@ def test_get_repo_file_replace_without_confirmation(tmp_dir, erepo_dir): Repo.get(os.fspath(erepo_dir), "file", "file_imported") # getting another file with a name that already exists in Repo. - with pytest.raises(ConfirmRemoveError): + with pytest.raises(FileExistsError): Repo.get(os.fspath(erepo_dir), "file2", "file_imported") assert os.path.isfile("file_imported") assert (tmp_dir / "file_imported").read_text() == "contents" -def test_get_repo_file_replace_yes(tmp_dir, erepo_dir): - with erepo_dir.chdir(): - erepo_dir.dvc_gen("file", "contents", commit="create file") - erepo_dir.dvc_gen( - "file2", "something different", commit="create file2" - ) - - Repo.get(os.fspath(erepo_dir), "file", "file_imported") - # getting another file with a name that already exists in Repo. - with mock.patch("dvc.prompt.confirm", return_value=True): - Repo.get(os.fspath(erepo_dir), "file2", "file_imported") - - assert os.path.isfile("file_imported") - assert (tmp_dir / "file_imported").read_text() == "something different" - - -def test_get_repo_file_replace_no(tmp_dir, erepo_dir): - with erepo_dir.chdir(): - erepo_dir.dvc_gen("file", "contents", commit="create file") - erepo_dir.dvc_gen( - "file2", "something different", commit="create file2" - ) - - Repo.get(os.fspath(erepo_dir), "file", "file_imported") - # getting another file with a name that already exists in Repo. - with mock.patch("dvc.prompt.confirm", return_value=False): - with pytest.raises(ConfirmRemoveError): - Repo.get(os.fspath(erepo_dir), "file2", "file_imported") - - assert os.path.isfile("file_imported") - assert (tmp_dir / "file_imported").read_text() == "contents" - - def test_get_repo_dir(tmp_dir, erepo_dir): with erepo_dir.chdir(): erepo_dir.dvc_gen({"dir": {"file": "contents"}}, commit="create dir") @@ -187,17 +151,17 @@ def test_non_cached_output(tmp_dir, erepo_dir): # https://github.com/iterative/dvc/pull/2837#discussion_r352123053 def test_absolute_file_outside_repo(tmp_dir, erepo_dir): - with pytest.raises(PathMissingError): + with pytest.raises(FileNotFoundError): Repo.get(os.fspath(erepo_dir), "/root/") def test_absolute_file_outside_git_repo(tmp_dir, git_dir): - with pytest.raises(PathMissingError): + with pytest.raises(FileNotFoundError): Repo.get(os.fspath(git_dir), "/root/") def test_unknown_path(tmp_dir, erepo_dir): - with pytest.raises(PathMissingError): + with pytest.raises(FileNotFoundError): Repo.get(os.fspath(erepo_dir), "a_non_existing_file") @@ -345,17 +309,23 @@ def test_granular_get_from_subrepos(tmp_dir, erepo_dir): assert (tmp_dir / "out").read_text() == "bar" -def test_try_to_get_complete_repo(tmp_dir, dvc, erepo_dir): +def test_get_complete_repo(tmp_dir, dvc, erepo_dir): + with erepo_dir.chdir(): + erepo_dir.dvc_gen({"foo": "foo"}, commit="add foo") + subrepo = erepo_dir / "subrepo" make_subrepo(subrepo, erepo_dir.scm) with subrepo.chdir(): subrepo.dvc_gen({"dir": {"bar": "bar"}}, commit="files in subrepo") - expected_message = "Cannot fetch a complete DVC repository" - with pytest.raises(IsADVCRepoError) as exc_info: - Repo.get(os.fspath(erepo_dir), "subrepo", out="out") - assert f"{expected_message} 'subrepo'" == str(exc_info.value) + Repo.get(os.fspath(erepo_dir), "subrepo", out="out_sub") + assert (tmp_dir / "out_sub").read_text() == { + ".gitignore": "/dir\n", + "dir": {"bar": "bar"}, + } - with pytest.raises(IsADVCRepoError) as exc_info: - Repo.get(os.fspath(erepo_dir), ".", out="out") - assert expected_message == str(exc_info.value) + Repo.get(os.fspath(erepo_dir), ".", out="out") + assert (tmp_dir / "out").read_text() == { + ".gitignore": "/foo\n", + "foo": "foo", + } diff --git a/tests/func/test_get_url.py b/tests/func/test_get_url.py index deb4c21b9..2951f2de9 100644 --- a/tests/func/test_get_url.py +++ b/tests/func/test_get_url.py @@ -1,3 +1,5 @@ +import os + import pytest from dvc.dependency.base import DependencyDoesNotExistError @@ -24,10 +26,12 @@ def test_get_dir(tmp_dir): @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"]) -def test_get_url_to_dir(tmp_dir, dname): - tmp_dir.gen({"foo": "foo contents", "dir": {"subdir": {}}}) +def test_get_url_to_dir( + tmp_dir, tmp_path_factory, dname, +): + tmp_dir.gen({"src": {"foo": "foo contents"}, "dir": {"subdir": {}}}) - Repo.get_url("foo", dname) + Repo.get_url(os.path.join("src", "foo"), dname) assert (tmp_dir / dname).is_dir() assert (tmp_dir / dname / "foo").read_text() == "foo contents" diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 9fe6d4fce..044c8ad28 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -9,8 +9,7 @@ import dvc.data_cloud as cloud from dvc.cache import Cache from dvc.config import NoRemoteError from dvc.dvcfile import Dvcfile -from dvc.exceptions import CollectCacheError, DownloadError, PathMissingError -from dvc.external_repo import IsADVCRepoError +from dvc.exceptions import CollectCacheError, DownloadError from dvc.stage.exceptions import StagePathNotFoundError from dvc.system import System from dvc.utils.fs import makedirs, remove @@ -339,6 +338,7 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir): scm.tag("ref-to-branch") # Overwrite via import + (tmp_dir / "foo_imported").unlink() dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", rev="master") remove(stage.outs[0].cache_path) @@ -347,11 +347,11 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir): def test_import_non_existing(erepo_dir, tmp_dir, dvc): - with pytest.raises(PathMissingError): + with pytest.raises(FileNotFoundError): tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output") # https://github.com/iterative/dvc/pull/2837#discussion_r352123053 - with pytest.raises(PathMissingError): + with pytest.raises(FileNotFoundError): tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root") @@ -497,20 +497,26 @@ def test_pull_imported_stage_from_subrepos( assert (tmp_dir / "out").read_text() == files[key] -def test_try_import_complete_repo(tmp_dir, dvc, erepo_dir): +def test_import_complete_repo(tmp_dir, dvc, erepo_dir): + with erepo_dir.chdir(): + erepo_dir.dvc_gen({"foo": "foo"}, commit="add foo") + subrepo = erepo_dir / "subrepo" make_subrepo(subrepo, erepo_dir.scm) with subrepo.chdir(): subrepo.dvc_gen({"dir": {"bar": "bar"}}, commit="files in subrepo") - expected_message = "Cannot fetch a complete DVC repository" - with pytest.raises(IsADVCRepoError) as exc_info: - dvc.imp(os.fspath(erepo_dir), "subrepo", out="out") - assert f"{expected_message} 'subrepo'" == str(exc_info.value) + dvc.imp(os.fspath(erepo_dir), "subrepo", out="out_sub") + assert (tmp_dir / "out_sub").read_text() == { + ".gitignore": "/dir\n", + "dir": {"bar": "bar"}, + } - with pytest.raises(IsADVCRepoError) as exc_info: - dvc.imp(os.fspath(erepo_dir), os.curdir, out="out") - assert expected_message == str(exc_info.value) + dvc.imp(os.fspath(erepo_dir), os.curdir, out="out") + assert (tmp_dir / "out").read_text() == { + ".gitignore": "/foo\n", + "foo": "foo", + } def test_import_with_no_exec(tmp_dir, dvc, erepo_dir): diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index b58b5406c..a314de035 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -69,7 +69,7 @@ class TestImportFilename(TestDvc): os.mkdir("sub") path = os.path.join("sub", "bar.dvc") - ret = main(["import-url", "--file", path, self.external_source]) + ret = main(["import-url", "--file", path, self.external_source, "out"]) self.assertEqual(0, ret) self.assertTrue(os.path.exists(path)) diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index 0e3048948..5f1a7f280 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -893,6 +893,7 @@ class TestReproAlreadyCached(TestRepro): self.assertNotEqual(run_out.hash_info, repro_out.hash_info) def test_force_import(self): + remove(self.BAR) ret = main(["import-url", self.FOO, self.BAR]) self.assertEqual(ret, 0) diff --git a/tests/unit/test_external_repo.py b/tests/unit/test_external_repo.py index 1b501830e..d823821d1 100644 --- a/tests/unit/test_external_repo.py +++ b/tests/unit/test_external_repo.py @@ -4,6 +4,7 @@ from unittest.mock import call import pytest from dvc.external_repo import external_repo +from dvc.path_info import PathInfo from tests.unit.tree.test_repo import make_subrepo @@ -96,6 +97,13 @@ def test_fetch_external_repo_jobs(tmp_dir, scm, mocker, dvc, local_remote): with external_repo(str(tmp_dir)) as repo: spy = mocker.spy(repo.cloud, "pull") - repo.fetch_external(["dir1"], jobs=3) + + dvc.cache.local.save( + PathInfo(repo.root_dir) / "dir1", + repo.repo_tree, + None, + follow_subrepos=False, + jobs=3, + ) run_jobs = tuple(spy.call_args_list[0])[1].get("jobs") assert run_jobs == 3 diff --git a/tests/unit/tree/test_repo.py b/tests/unit/tree/test_repo.py index 046418698..78c547dae 100644 --- a/tests/unit/tree/test_repo.py +++ b/tests/unit/tree/test_repo.py @@ -377,7 +377,9 @@ def test_subrepos(tmp_dir, scm, dvc): True, [ PathInfo("dir") / "repo" / "foo.dvc", + PathInfo("dir") / "repo" / ".dvcignore", PathInfo("dir") / "repo" / "dir1.dvc", + PathInfo("dir") / "repo2" / ".dvcignore", PathInfo("dir") / "repo2" / "lorem.dvc", PathInfo("dir") / "repo2" / "dir2.dvc", ], @@ -409,12 +411,10 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected): PathInfo("dir") / "repo", PathInfo("dir") / "repo.txt", PathInfo("dir") / "repo2", - PathInfo("dir") / "repo" / ".dvcignore", PathInfo("dir") / "repo" / ".gitignore", PathInfo("dir") / "repo" / "foo", PathInfo("dir") / "repo" / "dir1", PathInfo("dir") / "repo" / "dir1" / "bar", - PathInfo("dir") / "repo2" / ".dvcignore", PathInfo("dir") / "repo2" / ".gitignore", PathInfo("dir") / "repo2" / "lorem", PathInfo("dir") / "repo2" / "dir2", @@ -583,7 +583,7 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos): for repo_dir in subrepos: make_subrepo(repo_dir, scm) - extras = {".dvcignore", ".gitignore"} # these files are always there + extras = {".gitignore"} # these files are always there expected = {} for repo_dir in subrepos + [tmp_dir]: base = os.path.basename(repo_dir)