mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
ls: implement --tree to show as a tree and --level to limit depth for recursion (#10598)
This commit is contained in:
+161
-36
@@ -1,3 +1,5 @@
|
||||
from typing import Callable
|
||||
|
||||
from dvc.cli import completion, formatter
|
||||
from dvc.cli.command import CmdBaseNoRepo
|
||||
from dvc.cli.utils import DictAction, append_doc_link
|
||||
@@ -9,7 +11,18 @@ from dvc.ui import ui
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
def _format_entry(entry, fmt, with_size=True, with_md5=False):
|
||||
def _get_formatter(with_color: bool = False) -> Callable[[dict], str]:
|
||||
def fmt(entry: dict) -> str:
|
||||
return entry["path"]
|
||||
|
||||
if with_color:
|
||||
ls_colors = LsColors()
|
||||
return ls_colors.format
|
||||
|
||||
return fmt
|
||||
|
||||
|
||||
def _format_entry(entry, name, with_size=True, with_hash=False):
|
||||
from dvc.utils.humanize import naturalsize
|
||||
|
||||
ret = []
|
||||
@@ -20,28 +33,28 @@ def _format_entry(entry, fmt, with_size=True, with_md5=False):
|
||||
else:
|
||||
size = naturalsize(size)
|
||||
ret.append(size)
|
||||
if with_md5:
|
||||
if with_hash:
|
||||
md5 = entry.get("md5", "")
|
||||
ret.append(md5)
|
||||
ret.append(fmt(entry))
|
||||
ret.append(name)
|
||||
return ret
|
||||
|
||||
|
||||
def show_entries(entries, with_color=False, with_size=False, with_md5=False):
|
||||
if with_color:
|
||||
ls_colors = LsColors()
|
||||
fmt = ls_colors.format
|
||||
else:
|
||||
|
||||
def fmt(entry):
|
||||
return entry["path"]
|
||||
|
||||
if with_size or with_md5:
|
||||
def show_entries(entries, with_color=False, with_size=False, with_hash=False):
|
||||
fmt = _get_formatter(with_color)
|
||||
if with_size or with_hash:
|
||||
colalign = ("right",) if with_size else None
|
||||
ui.table(
|
||||
[
|
||||
_format_entry(entry, fmt, with_size=with_size, with_md5=with_md5)
|
||||
_format_entry(
|
||||
entry,
|
||||
fmt(entry),
|
||||
with_size=with_size,
|
||||
with_hash=with_hash,
|
||||
)
|
||||
for entry in entries
|
||||
]
|
||||
],
|
||||
colalign=colalign,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -49,31 +62,130 @@ def show_entries(entries, with_color=False, with_size=False, with_md5=False):
|
||||
ui.write("\n".join(fmt(entry) for entry in entries))
|
||||
|
||||
|
||||
class TreePart:
|
||||
Edge = "├── "
|
||||
Line = "│ "
|
||||
Corner = "└── "
|
||||
Blank = " "
|
||||
|
||||
|
||||
def _build_tree_structure(
|
||||
entries, with_color=False, with_size=False, with_hash=False, _depth=0, _prefix=""
|
||||
):
|
||||
rows = []
|
||||
fmt = _get_formatter(with_color)
|
||||
|
||||
num_entries = len(entries)
|
||||
for i, (name, entry) in enumerate(entries.items()):
|
||||
# show full path for root, otherwise only show the name
|
||||
if _depth > 0:
|
||||
entry["path"] = name
|
||||
|
||||
is_last = i >= num_entries - 1
|
||||
tree_part = ""
|
||||
if _depth > 0:
|
||||
tree_part = TreePart.Corner if is_last else TreePart.Edge
|
||||
|
||||
row = _format_entry(
|
||||
entry,
|
||||
_prefix + tree_part + fmt(entry),
|
||||
with_size=with_size,
|
||||
with_hash=with_hash,
|
||||
)
|
||||
rows.append(row)
|
||||
|
||||
if contents := entry.get("contents"):
|
||||
new_prefix = _prefix
|
||||
if _depth > 0:
|
||||
new_prefix += TreePart.Blank if is_last else TreePart.Line
|
||||
new_rows = _build_tree_structure(
|
||||
contents,
|
||||
with_color=with_color,
|
||||
with_size=with_size,
|
||||
with_hash=with_hash,
|
||||
_depth=_depth + 1,
|
||||
_prefix=new_prefix,
|
||||
)
|
||||
rows.extend(new_rows)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def show_tree(entries, with_color=False, with_size=False, with_hash=False):
|
||||
import tabulate
|
||||
|
||||
rows = _build_tree_structure(
|
||||
entries,
|
||||
with_color=with_color,
|
||||
with_size=with_size,
|
||||
with_hash=with_hash,
|
||||
)
|
||||
|
||||
colalign = ("right",) if with_size else None
|
||||
|
||||
_orig = tabulate.PRESERVE_WHITESPACE
|
||||
tabulate.PRESERVE_WHITESPACE = True
|
||||
try:
|
||||
ui.table(rows, colalign=colalign)
|
||||
finally:
|
||||
tabulate.PRESERVE_WHITESPACE = _orig
|
||||
|
||||
|
||||
class CmdList(CmdBaseNoRepo):
|
||||
def run(self):
|
||||
def _show_tree(self):
|
||||
from dvc.repo.ls import ls_tree
|
||||
|
||||
entries = ls_tree(
|
||||
self.args.url,
|
||||
self.args.path,
|
||||
rev=self.args.rev,
|
||||
dvc_only=self.args.dvc_only,
|
||||
config=self.args.config,
|
||||
remote=self.args.remote,
|
||||
remote_config=self.args.remote_config,
|
||||
maxdepth=self.args.level,
|
||||
)
|
||||
show_tree(
|
||||
entries,
|
||||
with_color=True,
|
||||
with_size=self.args.size,
|
||||
with_hash=self.args.show_hash,
|
||||
)
|
||||
return 0
|
||||
|
||||
def _show_list(self):
|
||||
from dvc.repo import Repo
|
||||
|
||||
try:
|
||||
entries = Repo.ls(
|
||||
self.args.url,
|
||||
self.args.path,
|
||||
rev=self.args.rev,
|
||||
recursive=self.args.recursive,
|
||||
dvc_only=self.args.dvc_only,
|
||||
config=self.args.config,
|
||||
remote=self.args.remote,
|
||||
remote_config=self.args.remote_config,
|
||||
entries = Repo.ls(
|
||||
self.args.url,
|
||||
self.args.path,
|
||||
rev=self.args.rev,
|
||||
recursive=self.args.recursive,
|
||||
dvc_only=self.args.dvc_only,
|
||||
config=self.args.config,
|
||||
remote=self.args.remote,
|
||||
remote_config=self.args.remote_config,
|
||||
maxdepth=self.args.level,
|
||||
)
|
||||
if self.args.json:
|
||||
ui.write_json(entries)
|
||||
elif entries:
|
||||
show_entries(
|
||||
entries,
|
||||
with_color=True,
|
||||
with_size=self.args.size,
|
||||
with_hash=self.args.show_hash,
|
||||
)
|
||||
if self.args.json:
|
||||
ui.write_json(entries)
|
||||
elif entries:
|
||||
show_entries(
|
||||
entries,
|
||||
with_color=True,
|
||||
with_size=self.args.size,
|
||||
with_md5=self.args.show_hash,
|
||||
)
|
||||
return 0
|
||||
return 0
|
||||
|
||||
def run(self):
|
||||
if self.args.tree and self.args.json:
|
||||
raise DvcException("Cannot use --tree and --json options together.")
|
||||
|
||||
try:
|
||||
if self.args.tree:
|
||||
return self._show_tree()
|
||||
return self._show_list()
|
||||
except FileNotFoundError:
|
||||
logger.exception("")
|
||||
return 1
|
||||
@@ -102,6 +214,19 @@ def add_parser(subparsers, parent_parser):
|
||||
action="store_true",
|
||||
help="Recursively list files.",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"-T",
|
||||
"--tree",
|
||||
action="store_true",
|
||||
help="Recurse into directories as a tree.",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"-L",
|
||||
"--level",
|
||||
metavar="depth",
|
||||
type=int,
|
||||
help="Limit the depth of recursion.",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"--dvc-only", action="store_true", help="Show only DVC outputs."
|
||||
)
|
||||
|
||||
@@ -32,7 +32,9 @@ class LsColors:
|
||||
if entry.get("isexec", False):
|
||||
return self._format(text, code="ex")
|
||||
|
||||
_, ext = os.path.splitext(text)
|
||||
stem, ext = os.path.splitext(text)
|
||||
if not ext and stem.startswith("."):
|
||||
ext = stem
|
||||
return self._format(text, ext=ext)
|
||||
|
||||
def _format(self, text, code=None, ext=None):
|
||||
|
||||
+103
-46
@@ -4,8 +4,44 @@ from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
|
||||
|
||||
def _open_repo(
|
||||
url: str,
|
||||
rev: Optional[str] = None,
|
||||
config: Union[None, dict[str, Any], str] = None,
|
||||
remote: Optional[str] = None,
|
||||
remote_config: Optional[dict] = None,
|
||||
):
|
||||
from dvc.config import Config
|
||||
|
||||
from . import Repo
|
||||
|
||||
if config and not isinstance(config, dict):
|
||||
config_dict = Config.load_file(config)
|
||||
else:
|
||||
config_dict = None
|
||||
|
||||
return Repo.open(
|
||||
url,
|
||||
rev=rev,
|
||||
subrepos=True,
|
||||
uninitialized=True,
|
||||
config=config_dict,
|
||||
remote=remote,
|
||||
remote_config=remote_config,
|
||||
)
|
||||
|
||||
|
||||
def _adapt_info(info: dict[str, Any]) -> dict[str, Any]:
|
||||
dvc_info = info.get("dvc_info", {})
|
||||
return {
|
||||
"isout": dvc_info.get("isout", False),
|
||||
"isdir": info["type"] == "directory",
|
||||
"isexec": info.get("isexec", False),
|
||||
"size": info.get("size"),
|
||||
"md5": dvc_info.get("md5") or dvc_info.get("md5-dos2unix"),
|
||||
}
|
||||
|
||||
|
||||
def ls(
|
||||
url: str,
|
||||
@@ -16,6 +52,7 @@ def ls(
|
||||
config: Union[None, dict[str, Any], str] = None,
|
||||
remote: Optional[str] = None,
|
||||
remote_config: Optional[dict] = None,
|
||||
maxdepth: Optional[int] = None,
|
||||
):
|
||||
"""Methods for getting files and outputs for the repo.
|
||||
|
||||
@@ -41,60 +78,58 @@ def ls(
|
||||
"isexec": bool,
|
||||
}
|
||||
"""
|
||||
from dvc.config import Config
|
||||
|
||||
from . import Repo
|
||||
|
||||
if config and not isinstance(config, dict):
|
||||
config_dict = Config.load_file(config)
|
||||
else:
|
||||
config_dict = None
|
||||
|
||||
with Repo.open(
|
||||
url,
|
||||
rev=rev,
|
||||
subrepos=True,
|
||||
uninitialized=True,
|
||||
config=config_dict,
|
||||
remote=remote,
|
||||
remote_config=remote_config,
|
||||
) as repo:
|
||||
with _open_repo(url, rev, config, remote, remote_config) as repo:
|
||||
path = path or ""
|
||||
fs: DVCFileSystem = repo.dvcfs
|
||||
fs_path = fs.from_os_path(path)
|
||||
return _ls(fs, fs_path, recursive, dvc_only, maxdepth)
|
||||
|
||||
ret = _ls(repo, path, recursive, dvc_only)
|
||||
|
||||
ret_list = []
|
||||
for path, info in ret.items():
|
||||
info["path"] = path
|
||||
ret_list.append(info)
|
||||
ret_list.sort(key=lambda f: f["path"])
|
||||
return ret_list
|
||||
def ls_tree(
|
||||
url: str,
|
||||
path: Optional[str] = None,
|
||||
rev: Optional[str] = None,
|
||||
dvc_only: bool = False,
|
||||
config: Union[None, dict[str, Any], str] = None,
|
||||
remote: Optional[str] = None,
|
||||
remote_config: Optional[dict] = None,
|
||||
maxdepth: Optional[int] = None,
|
||||
):
|
||||
with _open_repo(url, rev, config, remote, remote_config) as repo:
|
||||
path = path or ""
|
||||
fs: DVCFileSystem = repo.dvcfs
|
||||
fs_path = fs.from_os_path(path)
|
||||
return _ls_tree(fs, fs_path, dvc_only, maxdepth)
|
||||
|
||||
|
||||
def _ls(
|
||||
repo: "Repo",
|
||||
fs: "DVCFileSystem",
|
||||
path: str,
|
||||
recursive: Optional[bool] = None,
|
||||
dvc_only: bool = False,
|
||||
maxdepth: Optional[int] = None,
|
||||
):
|
||||
fs: DVCFileSystem = repo.dvcfs
|
||||
fs_path = fs.from_os_path(path)
|
||||
|
||||
fs_path = fs.info(fs_path)["name"]
|
||||
fs_path = fs.info(path)["name"]
|
||||
|
||||
infos = {}
|
||||
if fs.isfile(fs_path):
|
||||
infos[os.path.basename(path)] = fs.info(fs_path)
|
||||
|
||||
# ignore maxdepth only if recursive is not set
|
||||
maxdepth = maxdepth if recursive else None
|
||||
if maxdepth == 0 or fs.isfile(fs_path):
|
||||
infos[os.path.basename(path) or os.curdir] = fs.info(fs_path)
|
||||
else:
|
||||
for root, dirs, files in fs.walk(
|
||||
fs_path, dvcfiles=True, dvc_only=dvc_only, detail=True
|
||||
fs_path,
|
||||
dvcfiles=True,
|
||||
dvc_only=dvc_only,
|
||||
detail=True,
|
||||
maxdepth=maxdepth,
|
||||
):
|
||||
if not recursive:
|
||||
files.update(dirs)
|
||||
|
||||
parts = fs.relparts(root, fs_path)
|
||||
if parts == (".",):
|
||||
parts = ()
|
||||
if not recursive or (maxdepth and len(parts) >= maxdepth - 1):
|
||||
files.update(dirs)
|
||||
|
||||
for name, entry in files.items():
|
||||
infos[os.path.join(*parts, name)] = entry
|
||||
@@ -102,15 +137,37 @@ def _ls(
|
||||
if not recursive:
|
||||
break
|
||||
|
||||
ret = {}
|
||||
for name, info in infos.items():
|
||||
dvc_info = info.get("dvc_info", {})
|
||||
ret[name] = {
|
||||
"isout": dvc_info.get("isout", False),
|
||||
"isdir": info["type"] == "directory",
|
||||
"isexec": info.get("isexec", False),
|
||||
"size": info.get("size"),
|
||||
"md5": dvc_info.get("md5") or dvc_info.get("md5-dos2unix"),
|
||||
}
|
||||
ret_list = []
|
||||
for p, info in sorted(infos.items(), key=lambda x: x[0]):
|
||||
_info = _adapt_info(info)
|
||||
_info["path"] = p
|
||||
ret_list.append(_info)
|
||||
return ret_list
|
||||
|
||||
|
||||
def _ls_tree(
|
||||
fs, path, dvc_only: bool = False, maxdepth: Optional[int] = None, _info=None
|
||||
):
|
||||
ret = {}
|
||||
info = _info or fs.info(path)
|
||||
|
||||
path = info["name"].rstrip(fs.sep) or os.curdir
|
||||
name = path.rsplit("/", 1)[-1]
|
||||
ls_info = _adapt_info(info)
|
||||
ls_info["path"] = path
|
||||
|
||||
recurse = maxdepth is None or maxdepth > 0
|
||||
if recurse and info["type"] == "directory":
|
||||
infos = fs.ls(path, dvcfiles=True, dvc_only=dvc_only, detail=True)
|
||||
infos.sort(key=lambda f: f["name"])
|
||||
maxdepth = maxdepth - 1 if maxdepth is not None else None
|
||||
contents = {}
|
||||
for info in infos:
|
||||
d = _ls_tree(
|
||||
fs, info["name"], dvc_only=dvc_only, maxdepth=maxdepth, _info=info
|
||||
)
|
||||
contents.update(d)
|
||||
ls_info["contents"] = contents
|
||||
|
||||
ret[name] = ls_info
|
||||
return ret
|
||||
|
||||
+8
-1
@@ -300,6 +300,7 @@ class Console:
|
||||
header_styles: Optional[Union[dict[str, "Styles"], Sequence["Styles"]]] = None,
|
||||
row_styles: Optional[Sequence["Styles"]] = None,
|
||||
borders: Union[bool, str] = False,
|
||||
colalign: Optional[tuple[str, ...]] = None,
|
||||
) -> None:
|
||||
from dvc.ui import table as t
|
||||
|
||||
@@ -321,7 +322,13 @@ class Console:
|
||||
return
|
||||
|
||||
return t.plain_table(
|
||||
self, data, headers, markdown=markdown, pager=pager, force=force
|
||||
self,
|
||||
data,
|
||||
headers,
|
||||
markdown=markdown,
|
||||
pager=pager,
|
||||
force=force,
|
||||
colalign=colalign,
|
||||
)
|
||||
|
||||
def status(self, status: str, **kwargs: Any) -> "Status":
|
||||
|
||||
@@ -29,6 +29,7 @@ def plain_table(
|
||||
markdown: bool = False,
|
||||
pager: bool = False,
|
||||
force: bool = True,
|
||||
colalign: Optional[tuple[str, ...]] = None,
|
||||
) -> None:
|
||||
from funcy import nullcontext
|
||||
from tabulate import tabulate
|
||||
@@ -40,6 +41,7 @@ def plain_table(
|
||||
disable_numparse=True,
|
||||
# None will be shown as "" by default, overriding
|
||||
missingval="-",
|
||||
colalign=colalign,
|
||||
)
|
||||
if markdown:
|
||||
# NOTE: md table is incomplete without the trailing newline
|
||||
|
||||
@@ -2,10 +2,12 @@ import os
|
||||
import shutil
|
||||
import textwrap
|
||||
from operator import itemgetter
|
||||
from os.path import join
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.ls import ls_tree
|
||||
from dvc.scm import CloneError
|
||||
|
||||
FS_STRUCTURE = {
|
||||
@@ -735,3 +737,264 @@ def test_ls_broken_dir(tmp_dir, dvc, M):
|
||||
|
||||
with pytest.raises(DataIndexDirError):
|
||||
Repo.ls(os.fspath(tmp_dir), recursive=True)
|
||||
|
||||
|
||||
def test_ls_maxdepth(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
files = Repo.ls(os.fspath(tmp_dir), "structure.xml", maxdepth=0, recursive=True)
|
||||
match_files(files, ((("structure.xml",), True),))
|
||||
|
||||
files = Repo.ls(os.fspath(tmp_dir), maxdepth=0, recursive=True)
|
||||
match_files(files, (((os.curdir,), False),))
|
||||
|
||||
files = Repo.ls(os.fspath(tmp_dir), maxdepth=1, recursive=True)
|
||||
match_files(
|
||||
files,
|
||||
(
|
||||
((".dvcignore",), False),
|
||||
((".gitignore",), False),
|
||||
(("README.md",), False),
|
||||
(("structure.xml.dvc",), False),
|
||||
(("model",), False),
|
||||
(("data",), False),
|
||||
(("structure.xml",), True),
|
||||
),
|
||||
)
|
||||
files = Repo.ls(os.fspath(tmp_dir), maxdepth=2, recursive=True)
|
||||
match_files(
|
||||
files,
|
||||
(
|
||||
((".dvcignore",), False),
|
||||
((".gitignore",), False),
|
||||
(("README.md",), False),
|
||||
((join("data", "subcontent"),), False),
|
||||
((join("model", ".gitignore"),), False),
|
||||
((join("model", "people.csv"),), True),
|
||||
((join("model", "people.csv.dvc"),), False),
|
||||
((join("model", "script.py"),), False),
|
||||
((join("model", "train.py"),), False),
|
||||
(("structure.xml",), True),
|
||||
(("structure.xml.dvc",), False),
|
||||
),
|
||||
)
|
||||
|
||||
files = Repo.ls(os.fspath(tmp_dir), maxdepth=3, recursive=True)
|
||||
match_files(
|
||||
files,
|
||||
(
|
||||
((".dvcignore",), False),
|
||||
((".gitignore",), False),
|
||||
(("README.md",), False),
|
||||
((join("data", "subcontent", ".gitignore"),), False),
|
||||
((join("data", "subcontent", "data.xml"),), True),
|
||||
((join("data", "subcontent", "data.xml.dvc"),), False),
|
||||
((join("data", "subcontent", "statistics"),), False),
|
||||
((join("model", ".gitignore"),), False),
|
||||
((join("model", "people.csv"),), True),
|
||||
((join("model", "people.csv.dvc"),), False),
|
||||
((join("model", "script.py"),), False),
|
||||
((join("model", "train.py"),), False),
|
||||
((join("structure.xml"),), True),
|
||||
((join("structure.xml.dvc"),), False),
|
||||
),
|
||||
)
|
||||
|
||||
files = Repo.ls(os.fspath(tmp_dir), maxdepth=4, recursive=True)
|
||||
match_files(
|
||||
files,
|
||||
(
|
||||
((".dvcignore",), False),
|
||||
((".gitignore",), False),
|
||||
(("README.md",), False),
|
||||
((join("data", "subcontent", ".gitignore"),), False),
|
||||
((join("data", "subcontent", "data.xml"),), True),
|
||||
((join("data", "subcontent", "data.xml.dvc"),), False),
|
||||
((join("data", "subcontent", "statistics", ".gitignore"),), False),
|
||||
((join("data", "subcontent", "statistics", "data.csv"),), True),
|
||||
((join("data", "subcontent", "statistics", "data.csv.dvc"),), False),
|
||||
((join("model", ".gitignore"),), False),
|
||||
((join("model", "people.csv"),), True),
|
||||
((join("model", "people.csv.dvc"),), False),
|
||||
((join("model", "script.py"),), False),
|
||||
((join("model", "train.py"),), False),
|
||||
(("structure.xml",), True),
|
||||
(("structure.xml.dvc",), False),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _simplify_tree(files):
|
||||
ret = {}
|
||||
for path, info in files.items():
|
||||
if content := info.get("contents"):
|
||||
ret[path] = _simplify_tree(content)
|
||||
else:
|
||||
ret[path] = None
|
||||
return ret
|
||||
|
||||
|
||||
def test_ls_tree(M, tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), "structure.xml")
|
||||
assert _simplify_tree(files) == {"structure.xml": None}
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir))
|
||||
|
||||
expected = {
|
||||
".": {
|
||||
".dvcignore": None,
|
||||
".gitignore": None,
|
||||
"README.md": None,
|
||||
"data": {
|
||||
"subcontent": {
|
||||
".gitignore": None,
|
||||
"data.xml": None,
|
||||
"data.xml.dvc": None,
|
||||
"statistics": {
|
||||
".gitignore": None,
|
||||
"data.csv": None,
|
||||
"data.csv.dvc": None,
|
||||
},
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
".gitignore": None,
|
||||
"people.csv": None,
|
||||
"people.csv.dvc": None,
|
||||
"script.py": None,
|
||||
"train.py": None,
|
||||
},
|
||||
"structure.xml": None,
|
||||
"structure.xml.dvc": None,
|
||||
}
|
||||
}
|
||||
assert _simplify_tree(files) == expected
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), "model")
|
||||
assert _simplify_tree(files) == {
|
||||
"model": {
|
||||
".gitignore": None,
|
||||
"people.csv": None,
|
||||
"people.csv.dvc": None,
|
||||
"script.py": None,
|
||||
"train.py": None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_ls_tree_dvc_only(M, tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), dvc_only=True)
|
||||
|
||||
expected = {
|
||||
".": {
|
||||
"data": {
|
||||
"subcontent": {"data.xml": None, "statistics": {"data.csv": None}}
|
||||
},
|
||||
"model": {"people.csv": None},
|
||||
"structure.xml": None,
|
||||
}
|
||||
}
|
||||
assert _simplify_tree(files) == expected
|
||||
|
||||
|
||||
def test_ls_tree_maxdepth(M, tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), maxdepth=0)
|
||||
assert _simplify_tree(files) == {".": None}
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), maxdepth=1)
|
||||
assert _simplify_tree(files) == {
|
||||
".": {
|
||||
".dvcignore": None,
|
||||
".gitignore": None,
|
||||
"README.md": None,
|
||||
"data": None,
|
||||
"model": None,
|
||||
"structure.xml": None,
|
||||
"structure.xml.dvc": None,
|
||||
}
|
||||
}
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), maxdepth=2)
|
||||
assert _simplify_tree(files) == {
|
||||
".": {
|
||||
".dvcignore": None,
|
||||
".gitignore": None,
|
||||
"README.md": None,
|
||||
"data": {"subcontent": None},
|
||||
"model": {
|
||||
".gitignore": None,
|
||||
"people.csv": None,
|
||||
"people.csv.dvc": None,
|
||||
"script.py": None,
|
||||
"train.py": None,
|
||||
},
|
||||
"structure.xml": None,
|
||||
"structure.xml.dvc": None,
|
||||
}
|
||||
}
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), maxdepth=3)
|
||||
assert _simplify_tree(files) == {
|
||||
".": {
|
||||
".dvcignore": None,
|
||||
".gitignore": None,
|
||||
"README.md": None,
|
||||
"data": {
|
||||
"subcontent": {
|
||||
".gitignore": None,
|
||||
"data.xml": None,
|
||||
"data.xml.dvc": None,
|
||||
"statistics": None,
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
".gitignore": None,
|
||||
"people.csv": None,
|
||||
"people.csv.dvc": None,
|
||||
"script.py": None,
|
||||
"train.py": None,
|
||||
},
|
||||
"structure.xml": None,
|
||||
"structure.xml.dvc": None,
|
||||
}
|
||||
}
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), maxdepth=4)
|
||||
assert _simplify_tree(files) == {
|
||||
".": {
|
||||
".dvcignore": None,
|
||||
".gitignore": None,
|
||||
"README.md": None,
|
||||
"data": {
|
||||
"subcontent": {
|
||||
".gitignore": None,
|
||||
"data.xml": None,
|
||||
"data.xml.dvc": None,
|
||||
"statistics": {
|
||||
".gitignore": None,
|
||||
"data.csv": None,
|
||||
"data.csv.dvc": None,
|
||||
},
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
".gitignore": None,
|
||||
"people.csv": None,
|
||||
"people.csv.dvc": None,
|
||||
"script.py": None,
|
||||
"train.py": None,
|
||||
},
|
||||
"structure.xml": None,
|
||||
"structure.xml.dvc": None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import json
|
||||
|
||||
from dvc.cli import parse_args
|
||||
from dvc.commands.ls import CmdList
|
||||
from dvc.commands.ls import CmdList, show_tree
|
||||
|
||||
|
||||
def _test_cli(mocker, *args):
|
||||
@@ -27,6 +27,7 @@ def test_list(mocker):
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -42,6 +43,7 @@ def test_list_recursive(mocker):
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -57,6 +59,7 @@ def test_list_git_ssh_rev(mocker):
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -73,6 +76,7 @@ def test_list_targets(mocker):
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -88,6 +92,7 @@ def test_list_outputs_only(mocker):
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -114,6 +119,65 @@ def test_list_config(mocker):
|
||||
config="myconfig",
|
||||
remote="myremote",
|
||||
remote_config={"k1": "v1", "k2": "v2"},
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
def test_list_level(mocker):
|
||||
url = "local_dir"
|
||||
m = _test_cli(mocker, url, None, "--level", "1")
|
||||
m.assert_called_once_with(
|
||||
url,
|
||||
None,
|
||||
recursive=False,
|
||||
rev=None,
|
||||
dvc_only=False,
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=1,
|
||||
)
|
||||
|
||||
|
||||
def test_list_tree(mocker):
|
||||
url = "[email protected]:repo"
|
||||
cli_args = parse_args(
|
||||
[
|
||||
"list",
|
||||
url,
|
||||
"local_dir",
|
||||
"--rev",
|
||||
"123",
|
||||
"--tree",
|
||||
"--show-hash",
|
||||
"--size",
|
||||
"--level",
|
||||
"2",
|
||||
"--dvc-only",
|
||||
"--config",
|
||||
"myconfig",
|
||||
"--remote",
|
||||
"myremote",
|
||||
"--remote-config",
|
||||
"k1=v1",
|
||||
"k2=v2",
|
||||
]
|
||||
)
|
||||
assert cli_args.func == CmdList
|
||||
|
||||
cmd = cli_args.func(cli_args)
|
||||
m = mocker.patch("dvc.repo.ls.ls_tree")
|
||||
|
||||
assert cmd.run() == 0
|
||||
m.assert_called_once_with(
|
||||
url,
|
||||
"local_dir",
|
||||
rev="123",
|
||||
dvc_only=True,
|
||||
config="myconfig",
|
||||
remote="myremote",
|
||||
remote_config={"k1": "v1", "k2": "v2"},
|
||||
maxdepth=2,
|
||||
)
|
||||
|
||||
|
||||
@@ -165,6 +229,268 @@ def test_show_colors(mocker, capsys, monkeypatch):
|
||||
]
|
||||
|
||||
|
||||
def test_show_size(mocker, capsys):
|
||||
cli_args = parse_args(["list", "local_dir", "--size"])
|
||||
assert cli_args.func == CmdList
|
||||
cmd = cli_args.func(cli_args)
|
||||
|
||||
result = [
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".dvcignore",
|
||||
"size": 100,
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".gitignore",
|
||||
"size": 200,
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": "README.md",
|
||||
"size": 100_000,
|
||||
},
|
||||
]
|
||||
mocker.patch("dvc.repo.Repo.ls", return_value=result)
|
||||
|
||||
assert cmd.run() == 0
|
||||
out, _ = capsys.readouterr()
|
||||
entries = out.splitlines()
|
||||
|
||||
assert entries == [" 100 .dvcignore", " 200 .gitignore", "97.7k README.md"]
|
||||
|
||||
|
||||
def test_show_hash(mocker, capsys):
|
||||
cli_args = parse_args(["list", "local_dir", "--show-hash"])
|
||||
assert cli_args.func == CmdList
|
||||
cmd = cli_args.func(cli_args)
|
||||
|
||||
result = [
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".dvcignore",
|
||||
"md5": "123",
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".gitignore",
|
||||
"md5": "456",
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": "README.md",
|
||||
"md5": "789",
|
||||
},
|
||||
]
|
||||
mocker.patch("dvc.repo.Repo.ls", return_value=result)
|
||||
|
||||
assert cmd.run() == 0
|
||||
out, _ = capsys.readouterr()
|
||||
entries = out.splitlines()
|
||||
|
||||
assert entries == ["123 .dvcignore", "456 .gitignore", "789 README.md"]
|
||||
|
||||
|
||||
def test_show_size_and_hash(mocker, capsys):
|
||||
cli_args = parse_args(["list", "local_dir", "--size", "--show-hash"])
|
||||
assert cli_args.func == CmdList
|
||||
cmd = cli_args.func(cli_args)
|
||||
|
||||
result = [
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".dvcignore",
|
||||
"size": 100,
|
||||
"md5": "123",
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".gitignore",
|
||||
"size": 200,
|
||||
"md5": "456",
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": "README.md",
|
||||
"size": 100_000,
|
||||
"md5": "789",
|
||||
},
|
||||
]
|
||||
mocker.patch("dvc.repo.Repo.ls", return_value=result)
|
||||
|
||||
assert cmd.run() == 0
|
||||
out, _ = capsys.readouterr()
|
||||
entries = out.splitlines()
|
||||
|
||||
assert entries == [
|
||||
" 100 123 .dvcignore",
|
||||
" 200 456 .gitignore",
|
||||
"97.7k 789 README.md",
|
||||
]
|
||||
|
||||
|
||||
def test_show_tree(capsys):
|
||||
entries = {
|
||||
"data": {
|
||||
"isout": True,
|
||||
"isdir": True,
|
||||
"isexec": False,
|
||||
"size": 192,
|
||||
"path": "data",
|
||||
"md5": "3fb071066d5d5b282f56a0169340346d.dir",
|
||||
"contents": {
|
||||
"dir": {
|
||||
"isout": False,
|
||||
"isdir": True,
|
||||
"isexec": False,
|
||||
"size": 96,
|
||||
"path": "data/dir",
|
||||
"md5": None,
|
||||
"contents": {
|
||||
"subdir": {
|
||||
"isout": True,
|
||||
"isdir": True,
|
||||
"isexec": False,
|
||||
"size": 96,
|
||||
"md5": None,
|
||||
"path": "data/dir/subdir",
|
||||
"contents": {
|
||||
"foobar": {
|
||||
"isout": True,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"size": 4,
|
||||
"md5": "d3b07384d113edec49eaa6238ad5ff00",
|
||||
}
|
||||
},
|
||||
},
|
||||
"foo": {
|
||||
"isout": False,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"size": 4,
|
||||
"path": "data/dir/foo",
|
||||
"md5": None,
|
||||
},
|
||||
},
|
||||
},
|
||||
"bar": {
|
||||
"isout": True,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"size": 4,
|
||||
"path": "data/bar",
|
||||
"md5": "c157a79031e1c40f85931829bc5fc552",
|
||||
},
|
||||
"large-file": {
|
||||
"isout": False,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"size": 1073741824,
|
||||
"path": "data/large-file",
|
||||
"md5": None,
|
||||
},
|
||||
"dir2": {
|
||||
"isout": True,
|
||||
"isdir": True,
|
||||
"isexec": False,
|
||||
"size": 96,
|
||||
"md5": None,
|
||||
"path": "data/dir2",
|
||||
"contents": {
|
||||
"foo": {
|
||||
"isout": True,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"size": 4,
|
||||
"path": "data/dir2/foo",
|
||||
"md5": "d3b07384d113edec49eaa6238ad5ff00",
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
show_tree(entries, with_color=False)
|
||||
out, _ = capsys.readouterr()
|
||||
expected = """\
|
||||
data
|
||||
├── dir
|
||||
│ ├── subdir
|
||||
│ │ └── foobar
|
||||
│ └── foo
|
||||
├── bar
|
||||
├── large-file
|
||||
└── dir2
|
||||
└── foo
|
||||
"""
|
||||
assert out == expected
|
||||
|
||||
show_tree(entries, with_color=False, with_size=True)
|
||||
out, _ = capsys.readouterr()
|
||||
expected = """\
|
||||
192 data
|
||||
96 ├── dir
|
||||
96 │ ├── subdir
|
||||
4 │ │ └── foobar
|
||||
4 │ └── foo
|
||||
4 ├── bar
|
||||
1.00G ├── large-file
|
||||
96 └── dir2
|
||||
4 └── foo
|
||||
"""
|
||||
assert out == expected
|
||||
|
||||
show_tree(entries, with_color=False, with_hash=True)
|
||||
out, _ = capsys.readouterr()
|
||||
expected = """\
|
||||
3fb071066d5d5b282f56a0169340346d.dir data
|
||||
- ├── dir
|
||||
- │ ├── subdir
|
||||
d3b07384d113edec49eaa6238ad5ff00 │ │ └── foobar
|
||||
- │ └── foo
|
||||
c157a79031e1c40f85931829bc5fc552 ├── bar
|
||||
- ├── large-file
|
||||
- └── dir2
|
||||
d3b07384d113edec49eaa6238ad5ff00 └── foo
|
||||
"""
|
||||
assert out == expected
|
||||
|
||||
show_tree(entries, with_color=False, with_hash=True, with_size=True)
|
||||
out, _ = capsys.readouterr()
|
||||
expected = """\
|
||||
192 3fb071066d5d5b282f56a0169340346d.dir data
|
||||
96 - ├── dir
|
||||
96 - │ ├── subdir
|
||||
4 d3b07384d113edec49eaa6238ad5ff00 │ │ └── foobar
|
||||
4 - │ └── foo
|
||||
4 c157a79031e1c40f85931829bc5fc552 ├── bar
|
||||
1.00G - ├── large-file
|
||||
96 - └── dir2
|
||||
4 d3b07384d113edec49eaa6238ad5ff00 └── foo
|
||||
"""
|
||||
assert out == expected
|
||||
|
||||
|
||||
def test_list_alias():
|
||||
cli_args = parse_args(["ls", "local_dir"])
|
||||
assert cli_args.func == CmdList
|
||||
|
||||
Reference in New Issue
Block a user