mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
dvcfs: ls: don't call info()
Cuts down `dvc ls . data/mnist/dataset --recursive` for me from ~37 to ~25 seconds. Note that this also changes behaviour for ls with a broken symlink, as we were not able to see it before because of a failing `info()` call, but now we can follow a normal UNIX ls behaviour more closely.
This commit is contained in:
+13
-18
@@ -276,43 +276,38 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
key = self._get_key_from_relative(path)
|
||||
repo, dvc_fs, subkey = self._get_subrepo_info(key)
|
||||
|
||||
names = set()
|
||||
dvc_infos = {}
|
||||
if dvc_fs:
|
||||
with suppress(FileNotFoundError):
|
||||
dvc_path = _get_dvc_path(dvc_fs, subkey)
|
||||
for entry in dvc_fs.ls(dvc_path, detail=False):
|
||||
names.add(dvc_fs.path.name(entry))
|
||||
for info in dvc_fs.ls(dvc_path, detail=True):
|
||||
dvc_infos[dvc_fs.path.name(info["name"])] = info
|
||||
|
||||
fs_infos = {}
|
||||
ignore_subrepos = kwargs.get("ignore_subrepos", True)
|
||||
if not dvc_only:
|
||||
fs = self.repo.fs
|
||||
fs_path = self._from_key(key)
|
||||
try:
|
||||
for entry in repo.dvcignore.ls(
|
||||
fs, fs_path, detail=False, ignore_subrepos=ignore_subrepos
|
||||
for info in repo.dvcignore.ls(
|
||||
fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos
|
||||
):
|
||||
names.add(fs.path.name(entry))
|
||||
fs_infos[fs.path.name(info["name"])] = info
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
pass
|
||||
|
||||
dvcfiles = kwargs.get("dvcfiles", False)
|
||||
if not dvcfiles:
|
||||
names = (name for name in names if not _is_dvc_file(name))
|
||||
|
||||
infos = []
|
||||
paths = []
|
||||
names = set(dvc_infos.keys()) | set(fs_infos.keys())
|
||||
for name in names:
|
||||
entry_path = self.path.join(path, name)
|
||||
entry_key = key + (name,)
|
||||
try:
|
||||
info = self._info(
|
||||
entry_key,
|
||||
entry_path,
|
||||
ignore_subrepos=ignore_subrepos,
|
||||
check_ignored=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
if not dvcfiles and _is_dvc_file(name):
|
||||
continue
|
||||
|
||||
entry_path = self.path.join(path, name)
|
||||
info = _merge_info(repo, fs_infos.get(name), dvc_infos.get(name))
|
||||
info["name"] = entry_path
|
||||
infos.append(info)
|
||||
paths.append(entry_path)
|
||||
|
||||
|
||||
@@ -587,4 +587,10 @@ def test_broken_symlink(tmp_dir, dvc):
|
||||
"isexec": False,
|
||||
"path": ".dvcignore",
|
||||
},
|
||||
{
|
||||
"isout": False,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"path": "link",
|
||||
},
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user