Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-05 15:28:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

erepo: refactor download/fetch logic (#5234)

Browse Source 
* erepo: don't use _get_tree_for

* erepo: refactor download/fetch logic

* dvc: unify handling existing output
This commit is contained in:
Ruslan Kuprieiev
2021-01-10 10:48:36 +02:00
committed by GitHub
parent 6b05cc5e24
commit c83a4ea406
16 changed files with 167 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cache/base.py
Vendored
+10
View File
@@ -1,6 +1,8 @@
import errno
import itertools
import json
import logging
import os
from concurrent.futures import ThreadPoolExecutor
from copy import copy
from typing import Optional
@@ -347,7 +349,15 @@ class CloudCache:
self.tree.scheme,
)
if not hash_info:
hash_info = tree.get_hash(path_info, **kwargs)
if not hash_info:
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path_info
)
self._save(path_info, tree, hash_info, save_link, **kwargs)
return hash_info
def _save(self, path_info, tree, hash_info, save_link=True, **kwargs):
to_info = self.tree.hash_to_path_info(hash_info.value)
dvc/dependency/repo.py
+9 -3
View File
@@ -77,10 +77,16 @@ class RepoDependency(LocalDependency):
with self._make_repo(cache_dir=cache.cache_dir) as repo:
if self.def_repo.get(self.PARAM_REV_LOCK) is None:
self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
path_info = PathInfo(repo.root_dir) / self.def_path
hash_info = cache.save(
path_info,
repo.repo_tree,
None,
jobs=jobs,
follow_subrepos=False,
)
_, _, cache_infos = repo.fetch_external([self.def_path], jobs=jobs)
cache.checkout(to.path_info, cache_infos[0])
cache.checkout(to.path_info, hash_info)
def update(self, rev=None):
if rev:
dvc/external_repo.py
+2 -87
View File
@@ -3,12 +3,11 @@ import os
import tempfile
import threading
from contextlib import contextmanager
from typing import Dict, Iterable
from typing import Dict
from funcy import cached_property, reraise, retry, wrap_with
from funcy import cached_property, retry, wrap_with
from dvc.exceptions import (
DvcException,
FileMissingError,
NoOutputInExternalRepoError,
NoRemoteInExternalRepoError,
@@ -16,17 +15,12 @@ from dvc.exceptions import (
OutputNotFoundError,
PathMissingError,
)
from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.utils import relpath
logger = logging.getLogger(__name__)
class IsADVCRepoError(DvcException):
"""Raised when it is not expected to be a dvc repo."""
@contextmanager
def external_repo(url, rev=None, for_write=False, **kwargs):
from dvc.config import NoRemoteError
@@ -162,85 +156,6 @@ class ExternalRepo(Repo):
return self.scm.get_rev()
return self.tree.rev
def _fetch_to_cache(self, path_info, repo, callback, **kwargs):
# don't support subrepo traversal as it might fail due to difference
# in remotes
tree = self._get_tree_for(repo)
cache = repo.cache.local
hash_info = tree.get_hash(
path_info, download_callback=callback, **kwargs
)
cache.save(
path_info,
tree,
hash_info,
save_link=False,
download_callback=callback,
)
return hash_info
def fetch_external(self, paths: Iterable, **kwargs):
"""Fetch specified external repo paths into cache.
Returns 3-tuple in the form
(downloaded, failed, list(cache_infos))
where cache_infos can be used as checkout targets for the
fetched paths.
"""
download_results = []
failed = 0
root = PathInfo(self.root_dir)
paths = [root / path for path in paths]
def download_update(result):
download_results.append(result)
hash_infos = []
for path in paths:
with reraise(FileNotFoundError, PathMissingError(path, self.url)):
metadata = self.repo_tree.metadata(path)
self._check_repo(path, metadata.repo)
repo = metadata.repo
hash_info = self._fetch_to_cache(
path, repo, download_update, **kwargs
)
hash_infos.append(hash_info)
return sum(download_results), failed, hash_infos
def _check_repo(self, path_info, repo):
if not repo:
return
repo_path = PathInfo(repo.root_dir)
if path_info == repo_path and isinstance(repo, Repo):
message = "Cannot fetch a complete DVC repository"
if repo.root_dir != self.root_dir:
rel = relpath(repo.root_dir, self.root_dir)
message += f" '{rel}'"
raise IsADVCRepoError(message)
def get_external(self, path, dest):
"""Convenience wrapper for fetch_external and checkout."""
path_info = PathInfo(self.root_dir) / path
with reraise(FileNotFoundError, PathMissingError(path, self.url)):
metadata = self.repo_tree.metadata(path_info)
self._check_repo(path_info, metadata.repo)
if metadata.output_exists:
repo = metadata.repo
cache = repo.cache.local
# fetch DVC and git files to tmpdir cache, then checkout
save_info = self._fetch_to_cache(path_info, repo, None)
cache.checkout(PathInfo(dest), save_info)
else:
# git-only folder, just copy files directly to dest
tree = self._get_tree_for(metadata.repo) # ignore subrepos
tree.copytree(path_info, dest)
def _get_tree_for(self, repo, **kwargs):
"""
Provides a combined tree of a single repo with dvc + git/local tree.
dvc/repo/fetch.py
+20 -5
View File
@@ -80,21 +80,36 @@ def fetch(
def _fetch_external(self, repo_url, repo_rev, files, jobs):
from dvc.external_repo import external_repo
from dvc.path_info import PathInfo
from dvc.scm.base import CloneError
failed, downloaded = 0, 0
failed = 0
results = []
def cb(result):
results.append(result)
cache = self.cache.local
try:
with external_repo(
repo_url, repo_rev, cache_dir=cache.cache_dir
) as repo:
d, f, _ = repo.fetch_external(files, jobs=jobs)
downloaded += d
failed += f
root = PathInfo(repo.root_dir)
for path in files:
path_info = root / path
self.cache.local.save(
path_info,
repo.repo_tree,
None,
jobs=jobs,
download_callback=cb,
follow_subrepos=False,
)
except CloneError:
failed += 1
logger.exception(
"failed to fetch data for '{}'".format(", ".join(files))
)
return downloaded, failed
return sum(results), failed
dvc/repo/get.py
+4 -1
View File
@@ -2,6 +2,7 @@ import logging
import os
from dvc.exceptions import DvcException
from dvc.path_info import PathInfo
from dvc.utils import resolve_output
from dvc.utils.fs import remove
@@ -49,6 +50,8 @@ def get(url, path, out=None, rev=None):
with external_repo(
url=url, rev=rev, cache_dir=tmp_dir, cache_types=cache_types
) as repo:
repo.get_external(path, out)
from_info = PathInfo(repo.root_dir) / path
to_info = PathInfo(out)
repo.repo_tree.download(from_info, to_info, follow_subrepos=False)
finally:
remove(tmp_dir)
dvc/tree/base.py
+5 -2
View File
@@ -400,6 +400,7 @@ class BaseTree:
file_mode=None,
dir_mode=None,
jobs=None,
**kwargs,
):
if not hasattr(self, "_download"):
raise RemoteActionNotImplemented("download", self.scheme)
@@ -423,9 +424,10 @@ class BaseTree:
file_mode,
dir_mode,
jobs,
**kwargs,
)
return self._download_file(
from_info, to_info, name, no_progress_bar, file_mode, dir_mode
from_info, to_info, name, no_progress_bar, file_mode, dir_mode,
)
def _download_dir(
@@ -437,8 +439,9 @@ class BaseTree:
file_mode,
dir_mode,
jobs,
**kwargs,
):
from_infos = list(self.walk_files(from_info))
from_infos = list(self.walk_files(from_info, **kwargs))
to_infos = (
to_info / info.relative_to(from_info) for info in from_infos
)
dvc/tree/repo.py
+21 -26
View File
@@ -259,6 +259,7 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
def _walk(self, repo_walk, dvc_walk=None, dvcfiles=False):
from dvc.dvcfile import is_valid_filename
from dvc.ignore import DvcIgnore
assert repo_walk
try:
@@ -277,12 +278,16 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
shared = list(dvc_set & repo_set)
dirs = shared + dvc_only + repo_only
def _func(fname):
if dvcfiles:
return True
return not (
is_valid_filename(fname) or fname == DvcIgnore.DVCIGNORE_FILE
)
# merge file lists
files = {
fname
for fname in dvc_fnames + repo_fnames
if dvcfiles or not is_valid_filename(fname)
}
files = set(filter(_func, dvc_fnames + repo_fnames))
yield repo_root, dirs, list(files)
@@ -403,29 +408,19 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
pass
return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self)[0])
def copytree(self, top, dest):
from dvc.utils.fs import copy_fobj_to_file, makedirs
def _download(
self, from_info, to_file, name=None, no_progress_bar=False, **kwargs
):
import shutil
top = PathInfo(top)
dest = PathInfo(dest)
from dvc.progress import Tqdm
if not self.exists(top):
raise FileNotFoundError
if self.isfile(top):
makedirs(dest.parent, exist_ok=True)
with self.open(top, mode="rb") as fobj:
copy_fobj_to_file(fobj, dest)
return
for root, _, files in self.walk(top):
root = PathInfo(root)
dest_dir = dest / root.relative_to(top)
makedirs(dest_dir, exist_ok=True)
for fname in files:
src = root / fname
with self.open(src, mode="rb") as fobj:
copy_fobj_to_file(fobj, dest_dir / fname)
with open(to_file, "wb+") as to_fobj:
with Tqdm.wrapattr(
to_fobj, "write", desc=name, disable=no_progress_bar,
) as wrapped:
with self.open(from_info, "rb", **kwargs) as from_fobj:
shutil.copyfileobj(from_fobj, wrapped)
@property
def hash_jobs(self): # pylint: disable=invalid-overridden-method
dvc/utils/__init__.py
+11 -4
View File
@@ -351,14 +351,21 @@ def env2bool(var, undefined=False):
def resolve_output(inp, out):
import errno
from urllib.parse import urlparse
name = os.path.basename(os.path.normpath(urlparse(inp).path))
if not out:
return name
if os.path.isdir(out):
return os.path.join(out, name)
return out
ret = name
elif os.path.isdir(out):
ret = os.path.join(out, name)
else:
ret = out
if os.path.exists(ret):
raise FileExistsError(errno.EEXIST, os.strerror(errno.EEXIST), ret)
return ret
def resolve_paths(repo, out):
tests/func/test_external_repo.py
+28 -14
View File
@@ -92,10 +92,9 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
dest = tmp_dir / "file"
with external_repo(os.fspath(erepo_dir)) as repo:
_, _, save_infos = repo.fetch_external(
[os.path.join("subdir", "file")]
repo.repo_tree.download(
PathInfo(repo.root_dir) / "subdir" / "file", PathInfo(dest),
)
repo.cache.local.checkout(PathInfo(dest), save_infos[0])
assert dest.is_file()
assert dest.read_text() == "contents"
@@ -188,10 +187,19 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo:
repo.get_external("dir", "out")
repo.repo_tree.download(
PathInfo(repo.root_dir) / "dir",
PathInfo(tmp_dir / "out"),
follow_subrepos=False,
)
expected_files = {"foo": "foo", "bar": "bar", ".gitignore": "/foo\n"}
assert (tmp_dir / "out").read_text() == expected_files
# clear cache to test saving to cache
cache_dir = tmp_dir / repo.cache.local.cache_dir
remove(cache_dir)
makedirs(cache_dir)
expected_hash = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir")
assert (
repo.repo_tree.get_hash(
@@ -200,16 +208,18 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
== expected_hash
)
# clear cache to test `fetch_external` again
cache_dir = tmp_dir / repo.cache.local.cache_dir
remove(cache_dir)
makedirs(cache_dir)
assert repo.fetch_external(["dir"]) == (
len(expected_files),
0,
[expected_hash],
repo.cache.local.save(
PathInfo(repo.root_dir) / "dir",
repo.repo_tree,
expected_hash,
link=False,
)
assert set(cache_dir.glob("*/*")) == {
cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir",
cache_dir / "37" / "b51d194a7513e45b56f6524f2d51f2",
cache_dir / "94" / "7d2b84e5aa88170e80dff467a5bfb6",
cache_dir / "ac" / "bd18db4cc2f85cedef654fccc4a4d8",
}
def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
@@ -223,6 +233,10 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo:
repo.get_external("dir", "out")
repo.repo_tree.download(
PathInfo(repo.root_dir) / "dir",
PathInfo(tmp_dir / "out"),
follow_subrepos=False,
)
# subrepo files should not be here
assert (tmp_dir / "out").read_text() == scm_files
tests/func/test_get.py
+18 -48
View File
@@ -1,12 +1,9 @@
import logging
import os
import mock
import pytest
from dvc.cache import Cache
from dvc.exceptions import ConfirmRemoveError, PathMissingError
from dvc.external_repo import IsADVCRepoError
from dvc.main import main
from dvc.repo import Repo
from dvc.repo.get import GetDVCFileError
@@ -34,46 +31,13 @@ def test_get_repo_file_replace_without_confirmation(tmp_dir, erepo_dir):
Repo.get(os.fspath(erepo_dir), "file", "file_imported")
# getting another file with a name that already exists in Repo.
with pytest.raises(ConfirmRemoveError):
with pytest.raises(FileExistsError):
Repo.get(os.fspath(erepo_dir), "file2", "file_imported")
assert os.path.isfile("file_imported")
assert (tmp_dir / "file_imported").read_text() == "contents"
def test_get_repo_file_replace_yes(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "contents", commit="create file")
erepo_dir.dvc_gen(
"file2", "something different", commit="create file2"
)
Repo.get(os.fspath(erepo_dir), "file", "file_imported")
# getting another file with a name that already exists in Repo.
with mock.patch("dvc.prompt.confirm", return_value=True):
Repo.get(os.fspath(erepo_dir), "file2", "file_imported")
assert os.path.isfile("file_imported")
assert (tmp_dir / "file_imported").read_text() == "something different"
def test_get_repo_file_replace_no(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "contents", commit="create file")
erepo_dir.dvc_gen(
"file2", "something different", commit="create file2"
)
Repo.get(os.fspath(erepo_dir), "file", "file_imported")
# getting another file with a name that already exists in Repo.
with mock.patch("dvc.prompt.confirm", return_value=False):
with pytest.raises(ConfirmRemoveError):
Repo.get(os.fspath(erepo_dir), "file2", "file_imported")
assert os.path.isfile("file_imported")
assert (tmp_dir / "file_imported").read_text() == "contents"
def test_get_repo_dir(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen({"dir": {"file": "contents"}}, commit="create dir")
@@ -187,17 +151,17 @@ def test_non_cached_output(tmp_dir, erepo_dir):
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
def test_absolute_file_outside_repo(tmp_dir, erepo_dir):
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
Repo.get(os.fspath(erepo_dir), "/root/")
def test_absolute_file_outside_git_repo(tmp_dir, git_dir):
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
Repo.get(os.fspath(git_dir), "/root/")
def test_unknown_path(tmp_dir, erepo_dir):
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
Repo.get(os.fspath(erepo_dir), "a_non_existing_file")
@@ -345,17 +309,23 @@ def test_granular_get_from_subrepos(tmp_dir, erepo_dir):
assert (tmp_dir / "out").read_text() == "bar"
def test_try_to_get_complete_repo(tmp_dir, dvc, erepo_dir):
def test_get_complete_repo(tmp_dir, dvc, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen({"foo": "foo"}, commit="add foo")
subrepo = erepo_dir / "subrepo"
make_subrepo(subrepo, erepo_dir.scm)
with subrepo.chdir():
subrepo.dvc_gen({"dir": {"bar": "bar"}}, commit="files in subrepo")
expected_message = "Cannot fetch a complete DVC repository"
with pytest.raises(IsADVCRepoError) as exc_info:
Repo.get(os.fspath(erepo_dir), "subrepo", out="out")
assert f"{expected_message} 'subrepo'" == str(exc_info.value)
Repo.get(os.fspath(erepo_dir), "subrepo", out="out_sub")
assert (tmp_dir / "out_sub").read_text() == {
".gitignore": "/dir\n",
"dir": {"bar": "bar"},
}
with pytest.raises(IsADVCRepoError) as exc_info:
Repo.get(os.fspath(erepo_dir), ".", out="out")
assert expected_message == str(exc_info.value)
Repo.get(os.fspath(erepo_dir), ".", out="out")
assert (tmp_dir / "out").read_text() == {
".gitignore": "/foo\n",
"foo": "foo",
}
tests/func/test_get_url.py
+7 -3
View File
@@ -1,3 +1,5 @@
import os
import pytest
from dvc.dependency.base import DependencyDoesNotExistError
@@ -24,10 +26,12 @@ def test_get_dir(tmp_dir):
@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
def test_get_url_to_dir(tmp_dir, dname):
tmp_dir.gen({"foo": "foo contents", "dir": {"subdir": {}}})
def test_get_url_to_dir(
tmp_dir, tmp_path_factory, dname,
):
tmp_dir.gen({"src": {"foo": "foo contents"}, "dir": {"subdir": {}}})
Repo.get_url("foo", dname)
Repo.get_url(os.path.join("src", "foo"), dname)
assert (tmp_dir / dname).is_dir()
assert (tmp_dir / dname / "foo").read_text() == "foo contents"
tests/func/test_import.py
+18 -12
View File
@@ -9,8 +9,7 @@ import dvc.data_cloud as cloud
from dvc.cache import Cache
from dvc.config import NoRemoteError
from dvc.dvcfile import Dvcfile
from dvc.exceptions import CollectCacheError, DownloadError, PathMissingError
from dvc.external_repo import IsADVCRepoError
from dvc.exceptions import CollectCacheError, DownloadError
from dvc.stage.exceptions import StagePathNotFoundError
from dvc.system import System
from dvc.utils.fs import makedirs, remove
@@ -339,6 +338,7 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
scm.tag("ref-to-branch")
# Overwrite via import
(tmp_dir / "foo_imported").unlink()
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", rev="master")
remove(stage.outs[0].cache_path)
@@ -347,11 +347,11 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
def test_import_non_existing(erepo_dir, tmp_dir, dvc):
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
@@ -497,20 +497,26 @@ def test_pull_imported_stage_from_subrepos(
assert (tmp_dir / "out").read_text() == files[key]
def test_try_import_complete_repo(tmp_dir, dvc, erepo_dir):
def test_import_complete_repo(tmp_dir, dvc, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen({"foo": "foo"}, commit="add foo")
subrepo = erepo_dir / "subrepo"
make_subrepo(subrepo, erepo_dir.scm)
with subrepo.chdir():
subrepo.dvc_gen({"dir": {"bar": "bar"}}, commit="files in subrepo")
expected_message = "Cannot fetch a complete DVC repository"
with pytest.raises(IsADVCRepoError) as exc_info:
dvc.imp(os.fspath(erepo_dir), "subrepo", out="out")
assert f"{expected_message} 'subrepo'" == str(exc_info.value)
dvc.imp(os.fspath(erepo_dir), "subrepo", out="out_sub")
assert (tmp_dir / "out_sub").read_text() == {
".gitignore": "/dir\n",
"dir": {"bar": "bar"},
}
with pytest.raises(IsADVCRepoError) as exc_info:
dvc.imp(os.fspath(erepo_dir), os.curdir, out="out")
assert expected_message == str(exc_info.value)
dvc.imp(os.fspath(erepo_dir), os.curdir, out="out")
assert (tmp_dir / "out").read_text() == {
".gitignore": "/foo\n",
"foo": "foo",
}
def test_import_with_no_exec(tmp_dir, dvc, erepo_dir):
tests/func/test_import_url.py
+1 -1
View File
@@ -69,7 +69,7 @@ class TestImportFilename(TestDvc):
os.mkdir("sub")
path = os.path.join("sub", "bar.dvc")
ret = main(["import-url", "--file", path, self.external_source])
ret = main(["import-url", "--file", path, self.external_source, "out"])
self.assertEqual(0, ret)
self.assertTrue(os.path.exists(path))
tests/func/test_repro.py
+1
View File
@@ -893,6 +893,7 @@ class TestReproAlreadyCached(TestRepro):
self.assertNotEqual(run_out.hash_info, repro_out.hash_info)
def test_force_import(self):
remove(self.BAR)
ret = main(["import-url", self.FOO, self.BAR])
self.assertEqual(ret, 0)
tests/unit/test_external_repo.py
+9 -1
View File
@@ -4,6 +4,7 @@ from unittest.mock import call
import pytest
from dvc.external_repo import external_repo
from dvc.path_info import PathInfo
from tests.unit.tree.test_repo import make_subrepo
@@ -96,6 +97,13 @@ def test_fetch_external_repo_jobs(tmp_dir, scm, mocker, dvc, local_remote):
with external_repo(str(tmp_dir)) as repo:
spy = mocker.spy(repo.cloud, "pull")
repo.fetch_external(["dir1"], jobs=3)
dvc.cache.local.save(
PathInfo(repo.root_dir) / "dir1",
repo.repo_tree,
None,
follow_subrepos=False,
jobs=3,
)
run_jobs = tuple(spy.call_args_list[0])[1].get("jobs")
assert run_jobs == 3
tests/unit/tree/test_repo.py
+3 -3
View File
@@ -377,7 +377,9 @@ def test_subrepos(tmp_dir, scm, dvc):
True,
[
PathInfo("dir") / "repo" / "foo.dvc",
PathInfo("dir") / "repo" / ".dvcignore",
PathInfo("dir") / "repo" / "dir1.dvc",
PathInfo("dir") / "repo2" / ".dvcignore",
PathInfo("dir") / "repo2" / "lorem.dvc",
PathInfo("dir") / "repo2" / "dir2.dvc",
],
@@ -409,12 +411,10 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
PathInfo("dir") / "repo",
PathInfo("dir") / "repo.txt",
PathInfo("dir") / "repo2",
PathInfo("dir") / "repo" / ".dvcignore",
PathInfo("dir") / "repo" / ".gitignore",
PathInfo("dir") / "repo" / "foo",
PathInfo("dir") / "repo" / "dir1",
PathInfo("dir") / "repo" / "dir1" / "bar",
PathInfo("dir") / "repo2" / ".dvcignore",
PathInfo("dir") / "repo2" / ".gitignore",
PathInfo("dir") / "repo2" / "lorem",
PathInfo("dir") / "repo2" / "dir2",
@@ -583,7 +583,7 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos):
for repo_dir in subrepos:
make_subrepo(repo_dir, scm)
extras = {".dvcignore", ".gitignore"} # these files are always there
extras = {".gitignore"} # these files are always there
expected = {}
for repo_dir in subrepos + [tmp_dir]:
base = os.path.basename(repo_dir)