From d2366670482cd209ff3a3b45dc58dc45465b0eb6 Mon Sep 17 00:00:00 2001 From: Pete Bachant Date: Mon, 21 Oct 2024 03:47:03 -0700 Subject: [PATCH] Add option to show MD5 checksums in `dvc ls` (#10573) --- dvc/commands/ls/__init__.py | 45 ++++++++++++---- dvc/repo/ls.py | 1 + tests/func/test_ls.py | 105 ++++++++++++++++++++++++++++++++---- 3 files changed, 129 insertions(+), 22 deletions(-) diff --git a/dvc/commands/ls/__init__.py b/dvc/commands/ls/__init__.py index 455e3d9a3..f0c78639d 100644 --- a/dvc/commands/ls/__init__.py +++ b/dvc/commands/ls/__init__.py @@ -9,18 +9,25 @@ from dvc.ui import ui logger = logger.getChild(__name__) -def _format_entry(entry, fmt): +def _format_entry(entry, fmt, with_size=True, with_md5=False): from dvc.utils.humanize import naturalsize - size = entry.get("size") - if size is None: - size = "" - else: - size = naturalsize(size) - return size, fmt(entry) + ret = [] + if with_size: + size = entry.get("size") + if size is None: + size = "" + else: + size = naturalsize(size) + ret.append(size) + if with_md5: + md5 = entry.get("md5", "") + ret.append(md5) + ret.append(fmt(entry)) + return ret -def show_entries(entries, with_color=False, with_size=False): +def show_entries(entries, with_color=False, with_size=False, with_md5=False): if with_color: ls_colors = LsColors() fmt = ls_colors.format @@ -29,8 +36,13 @@ def show_entries(entries, with_color=False, with_size=False): def fmt(entry): return entry["path"] - if with_size: - ui.table([_format_entry(entry, fmt) for entry in entries]) + if with_size or with_md5: + ui.table( + [ + _format_entry(entry, fmt, with_size=with_size, with_md5=with_md5) + for entry in entries + ] + ) return # NOTE: this is faster than ui.table for very large number of entries @@ -55,7 +67,12 @@ class CmdList(CmdBaseNoRepo): if self.args.json: ui.write_json(entries) elif entries: - show_entries(entries, with_color=True, with_size=self.args.size) + show_entries( + entries, + with_color=True, + with_size=self.args.size, + with_md5=self.args.show_hash, + ) return 0 except FileNotFoundError: logger.exception("") @@ -123,6 +140,12 @@ def add_parser(subparsers, parent_parser): ), ) list_parser.add_argument("--size", action="store_true", help="Show sizes.") + list_parser.add_argument( + "--show-hash", + help="Display hash value for each item.", + action="store_true", + default=False, + ) list_parser.add_argument( "path", nargs="?", diff --git a/dvc/repo/ls.py b/dvc/repo/ls.py index 55f1e524f..ff30b64de 100644 --- a/dvc/repo/ls.py +++ b/dvc/repo/ls.py @@ -110,6 +110,7 @@ def _ls( "isdir": info["type"] == "directory", "isexec": info.get("isexec", False), "size": info.get("size"), + "md5": dvc_info.get("md5") or dvc_info.get("md5-dos2unix"), } return ret diff --git a/tests/func/test_ls.py b/tests/func/test_ls.py index 1a2806c7f..182233f00 100644 --- a/tests/func/test_ls.py +++ b/tests/func/test_ls.py @@ -222,7 +222,12 @@ def test_ls_repo_with_missed_path_dvc_only(tmp_dir, dvc, scm): tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc") with pytest.raises(FileNotFoundError): - Repo.ls(os.fspath(tmp_dir), path="missed_path", recursive=True, dvc_only=True) + Repo.ls( + os.fspath(tmp_dir), + path="missed_path", + recursive=True, + dvc_only=True, + ) def test_ls_repo_with_removed_dvc_dir(tmp_dir, dvc, scm): @@ -467,20 +472,49 @@ def test_ls_granular(erepo_dir, M): entries = Repo.ls(os.fspath(erepo_dir), os.path.join("dir", "subdir")) assert entries == [ - {"isout": True, "isdir": False, "isexec": False, "path": "bar", "size": 3}, - {"isout": True, "isdir": False, "isexec": False, "path": "foo", "size": 3}, + { + "isout": True, + "isdir": False, + "isexec": False, + "path": "bar", + "size": 3, + "md5": "37b51d194a7513e45b56f6524f2d51f2", + }, + { + "isout": True, + "isdir": False, + "isexec": False, + "path": "foo", + "size": 3, + "md5": "acbd18db4cc2f85cedef654fccc4a4d8", + }, ] entries = Repo.ls(os.fspath(erepo_dir), "dir") assert entries == [ - {"isout": True, "isdir": False, "isexec": False, "path": "1", "size": 1}, - {"isout": True, "isdir": False, "isexec": False, "path": "2", "size": 1}, + { + "isout": True, + "isdir": False, + "isexec": False, + "path": "1", + "size": 1, + "md5": "c4ca4238a0b923820dcc509a6f75849b", + }, + { + "isout": True, + "isdir": False, + "isexec": False, + "path": "2", + "size": 1, + "md5": "c81e728d9d4c2f636f067f89cc14862c", + }, { "isout": True, "isdir": True, "isexec": False, "path": "subdir", "size": M.instance_of(int), + "md5": None, }, ] @@ -506,14 +540,42 @@ def test_ls_target(erepo_dir, use_scm): return Repo.ls(os.fspath(erepo_dir), path) assert _ls(os.path.join("dir", "1")) == [ - {"isout": isout, "isdir": False, "isexec": False, "path": "1", "size": 1} + { + "isout": isout, + "isdir": False, + "isexec": False, + "path": "1", + "size": 1, + "md5": "c4ca4238a0b923820dcc509a6f75849b" if not use_scm else None, + } ] assert _ls(os.path.join("dir", "subdir", "foo")) == [ - {"isout": isout, "isdir": False, "isexec": False, "path": "foo", "size": 3} + { + "isout": isout, + "isdir": False, + "isexec": False, + "path": "foo", + "size": 3, + "md5": "acbd18db4cc2f85cedef654fccc4a4d8" if not use_scm else None, + } ] assert _ls(os.path.join("dir", "subdir")) == [ - {"isdir": False, "isexec": 0, "isout": isout, "path": "bar", "size": 3}, - {"isdir": False, "isexec": 0, "isout": isout, "path": "foo", "size": 3}, + { + "isdir": False, + "isexec": 0, + "isout": isout, + "path": "bar", + "size": 3, + "md5": "37b51d194a7513e45b56f6524f2d51f2" if not use_scm else None, + }, + { + "isdir": False, + "isexec": 0, + "isout": isout, + "path": "foo", + "size": 3, + "md5": "acbd18db4cc2f85cedef654fccc4a4d8" if not use_scm else None, + }, ] @@ -576,6 +638,7 @@ def test_broken_symlink(tmp_dir, dvc, M): "isexec": False, "path": ".dvcignore", "size": M.instance_of(int), + "md5": None, }, { "isout": False, @@ -583,6 +646,7 @@ def test_broken_symlink(tmp_dir, dvc, M): "isexec": False, "path": "link", "size": 0, + "md5": None, }, ] @@ -614,14 +678,23 @@ def test_ls_broken_dir(tmp_dir, dvc, M): "isout": False, "path": ".dvcignore", "size": M.instance_of(int), + "md5": None, + }, + { + "isdir": True, + "isexec": False, + "isout": True, + "path": "broken", + "size": 3, + "md5": "630bd47b538d2a513c7d267d07e0bc44.dir", }, - {"isdir": True, "isexec": False, "isout": True, "path": "broken", "size": 3}, { "isdir": False, "isexec": False, "isout": False, "path": "broken.dvc", "size": M.instance_of(int), + "md5": None, }, { "isdir": True, @@ -629,6 +702,7 @@ def test_ls_broken_dir(tmp_dir, dvc, M): "isout": True, "path": "dir", "size": M.instance_of(int), + "md5": "91aaa9bb58b657d623ef143b195a67e4.dir", }, { "isdir": False, @@ -636,14 +710,23 @@ def test_ls_broken_dir(tmp_dir, dvc, M): "isout": False, "path": "dir.dvc", "size": M.instance_of(int), + "md5": None, + }, + { + "isdir": False, + "isexec": False, + "isout": True, + "path": "foo", + "size": 3, + "md5": "acbd18db4cc2f85cedef654fccc4a4d8", }, - {"isdir": False, "isexec": False, "isout": True, "path": "foo", "size": 3}, { "isdir": False, "isexec": False, "isout": False, "path": "foo.dvc", "size": M.instance_of(int), + "md5": None, }, ]