mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-03 07:18:26 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
746f028f6c | ||
|
|
3651939e11 | ||
|
|
d236667048 | ||
|
|
d22a68f7a8 | ||
|
|
683ae7a951 | ||
|
|
d5054ce861 | ||
|
|
e884f1b8f0 | ||
|
|
9b5772fab8 | ||
|
|
c1aaeffc2d | ||
|
|
ad70dd322b | ||
|
|
72b6971730 | ||
|
|
ee1864af8e | ||
|
|
52877485d1 | ||
|
|
a923f110e3 | ||
|
|
b8266ed15f | ||
|
|
2c871ba432 | ||
|
|
32d11f1e05 | ||
|
|
f56343d6ae | ||
|
|
7b8922b20f | ||
|
|
436a2ac793 |
@@ -18,12 +18,16 @@ jobs:
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- uses: hynek/setup-cached-uv@v2
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: astral-sh/setup-uv@v3
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
|
||||
- run: uv pip install '.[tests]' --system
|
||||
- name: run benchmarks on base branch
|
||||
run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
|
||||
|
||||
@@ -27,15 +27,15 @@ jobs:
|
||||
with:
|
||||
python-version: 3.12
|
||||
|
||||
- uses: hynek/setup-cached-uv@v2
|
||||
- run: uv pip install --upgrade setuptools_scm build twine --system
|
||||
- uses: astral-sh/setup-uv@v3
|
||||
- run: uv pip install --upgrade setuptools_scm twine --system
|
||||
|
||||
- name: Force version for Test PyPI upload
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags') }}
|
||||
run: echo version=$(python -m setuptools_scm | awk -F+ '{print $1}' | tail -1) >> $GITHUB_ENV
|
||||
|
||||
- run: echo 'PKG = "pip"'>dvc/_build.py
|
||||
- run: python -m build --installer=uv
|
||||
- run: uv build
|
||||
env:
|
||||
SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DVC: ${{ env.version }}
|
||||
- run: twine check --strict dist/*
|
||||
@@ -67,6 +67,7 @@ jobs:
|
||||
- name: Upload package to Test PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
attestations: true
|
||||
repository-url: https://test.pypi.org/legacy/
|
||||
skip-existing: true
|
||||
|
||||
@@ -91,3 +92,5 @@ jobs:
|
||||
|
||||
- name: Upload package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
attestations: true
|
||||
|
||||
@@ -38,11 +38,11 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
|
||||
- uses: hynek/setup-cached-uv@v2
|
||||
- uses: astral-sh/setup-uv@v3
|
||||
with:
|
||||
cache-dependency-path: |
|
||||
dvc/pyproject.toml
|
||||
${{ matrix.plugin }}/pyproject.toml
|
||||
enable-cache: true
|
||||
cache-dependency-glob: "**/pyproject.toml"
|
||||
cache-suffix: ${{ matrix.pyv }}
|
||||
|
||||
- name: Install plugin + DVC@PR
|
||||
run: |
|
||||
|
||||
@@ -31,9 +31,10 @@ jobs:
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- uses: hynek/setup-cached-uv@v2
|
||||
- uses: astral-sh/setup-uv@v3
|
||||
with:
|
||||
cache-dependency-path: pyproject.toml
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
- run: uv pip install -e ".[dev]" --system
|
||||
env:
|
||||
UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
|
||||
@@ -86,10 +87,11 @@ jobs:
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
- uses: hynek/setup-cached-uv@v2
|
||||
- uses: astral-sh/setup-uv@v3
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
cache-suffix: ${{ matrix.pyv }}
|
||||
cache-dependency-path: pyproject.toml
|
||||
- run: uv pip install -e ".[dev]" pytest-split --system
|
||||
env:
|
||||
UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
|
||||
@@ -120,7 +122,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Slack Notification
|
||||
uses: rtCamp/[email protected].0
|
||||
uses: rtCamp/[email protected].1
|
||||
env:
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||
SLACK_COLOR: ${{ job.status }}
|
||||
|
||||
@@ -6,7 +6,7 @@ ci:
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.6.0
|
||||
rev: v5.0.0
|
||||
hooks:
|
||||
- id: check-added-large-files
|
||||
- id: check-case-conflict
|
||||
@@ -22,7 +22,7 @@ repos:
|
||||
- id: sort-simple-yaml
|
||||
- id: trailing-whitespace
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: 'v0.6.2'
|
||||
rev: 'v0.7.0'
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: [--fix, --exit-non-zero-on-fix]
|
||||
@@ -50,7 +50,7 @@ repos:
|
||||
language: system
|
||||
name: DVC pre-commit
|
||||
stages:
|
||||
- commit
|
||||
- pre-commit
|
||||
verbose: true
|
||||
require_serial: true
|
||||
- args:
|
||||
@@ -61,7 +61,7 @@ repos:
|
||||
language: system
|
||||
name: DVC pre-push
|
||||
stages:
|
||||
- push
|
||||
- pre-push
|
||||
require_serial: true
|
||||
- always_run: true
|
||||
args:
|
||||
|
||||
+1
-1
@@ -54,7 +54,7 @@ class CmdGC(CmdBase):
|
||||
msg += " and all experiments"
|
||||
|
||||
if self.args.not_in_remote:
|
||||
msg += " that are present in the DVC remote"
|
||||
msg += " that are not present in the DVC remote"
|
||||
|
||||
if self.args.repos:
|
||||
msg += " of the current and the following repos:"
|
||||
|
||||
+180
-32
@@ -1,3 +1,5 @@
|
||||
from typing import Callable
|
||||
|
||||
from dvc.cli import completion, formatter
|
||||
from dvc.cli.command import CmdBaseNoRepo
|
||||
from dvc.cli.utils import DictAction, append_doc_link
|
||||
@@ -9,54 +11,181 @@ from dvc.ui import ui
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
def _format_entry(entry, fmt):
|
||||
from dvc.utils.humanize import naturalsize
|
||||
def _get_formatter(with_color: bool = False) -> Callable[[dict], str]:
|
||||
def fmt(entry: dict) -> str:
|
||||
return entry["path"]
|
||||
|
||||
size = entry.get("size")
|
||||
if size is None:
|
||||
size = ""
|
||||
else:
|
||||
size = naturalsize(size)
|
||||
return size, fmt(entry)
|
||||
|
||||
|
||||
def show_entries(entries, with_color=False, with_size=False):
|
||||
if with_color:
|
||||
ls_colors = LsColors()
|
||||
fmt = ls_colors.format
|
||||
else:
|
||||
return ls_colors.format
|
||||
|
||||
def fmt(entry):
|
||||
return entry["path"]
|
||||
return fmt
|
||||
|
||||
|
||||
def _format_entry(entry, name, with_size=True, with_hash=False):
|
||||
from dvc.utils.humanize import naturalsize
|
||||
|
||||
ret = []
|
||||
if with_size:
|
||||
ui.table([_format_entry(entry, fmt) for entry in entries])
|
||||
size = entry.get("size")
|
||||
if size is None:
|
||||
size = ""
|
||||
else:
|
||||
size = naturalsize(size)
|
||||
ret.append(size)
|
||||
if with_hash:
|
||||
md5 = entry.get("md5", "")
|
||||
ret.append(md5)
|
||||
ret.append(name)
|
||||
return ret
|
||||
|
||||
|
||||
def show_entries(entries, with_color=False, with_size=False, with_hash=False):
|
||||
fmt = _get_formatter(with_color)
|
||||
if with_size or with_hash:
|
||||
colalign = ("right",) if with_size else None
|
||||
ui.table(
|
||||
[
|
||||
_format_entry(
|
||||
entry,
|
||||
fmt(entry),
|
||||
with_size=with_size,
|
||||
with_hash=with_hash,
|
||||
)
|
||||
for entry in entries
|
||||
],
|
||||
colalign=colalign,
|
||||
)
|
||||
return
|
||||
|
||||
# NOTE: this is faster than ui.table for very large number of entries
|
||||
ui.write("\n".join(fmt(entry) for entry in entries))
|
||||
|
||||
|
||||
class TreePart:
|
||||
Edge = "├── "
|
||||
Line = "│ "
|
||||
Corner = "└── "
|
||||
Blank = " "
|
||||
|
||||
|
||||
def _build_tree_structure(
|
||||
entries, with_color=False, with_size=False, with_hash=False, _depth=0, _prefix=""
|
||||
):
|
||||
rows = []
|
||||
fmt = _get_formatter(with_color)
|
||||
|
||||
num_entries = len(entries)
|
||||
for i, (name, entry) in enumerate(entries.items()):
|
||||
# show full path for root, otherwise only show the name
|
||||
if _depth > 0:
|
||||
entry["path"] = name
|
||||
|
||||
is_last = i >= num_entries - 1
|
||||
tree_part = ""
|
||||
if _depth > 0:
|
||||
tree_part = TreePart.Corner if is_last else TreePart.Edge
|
||||
|
||||
row = _format_entry(
|
||||
entry,
|
||||
_prefix + tree_part + fmt(entry),
|
||||
with_size=with_size,
|
||||
with_hash=with_hash,
|
||||
)
|
||||
rows.append(row)
|
||||
|
||||
if contents := entry.get("contents"):
|
||||
new_prefix = _prefix
|
||||
if _depth > 0:
|
||||
new_prefix += TreePart.Blank if is_last else TreePart.Line
|
||||
new_rows = _build_tree_structure(
|
||||
contents,
|
||||
with_color=with_color,
|
||||
with_size=with_size,
|
||||
with_hash=with_hash,
|
||||
_depth=_depth + 1,
|
||||
_prefix=new_prefix,
|
||||
)
|
||||
rows.extend(new_rows)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def show_tree(entries, with_color=False, with_size=False, with_hash=False):
|
||||
import tabulate
|
||||
|
||||
rows = _build_tree_structure(
|
||||
entries,
|
||||
with_color=with_color,
|
||||
with_size=with_size,
|
||||
with_hash=with_hash,
|
||||
)
|
||||
|
||||
colalign = ("right",) if with_size else None
|
||||
|
||||
_orig = tabulate.PRESERVE_WHITESPACE
|
||||
tabulate.PRESERVE_WHITESPACE = True
|
||||
try:
|
||||
ui.table(rows, colalign=colalign)
|
||||
finally:
|
||||
tabulate.PRESERVE_WHITESPACE = _orig
|
||||
|
||||
|
||||
class CmdList(CmdBaseNoRepo):
|
||||
def run(self):
|
||||
def _show_tree(self):
|
||||
from dvc.repo.ls import ls_tree
|
||||
|
||||
entries = ls_tree(
|
||||
self.args.url,
|
||||
self.args.path,
|
||||
rev=self.args.rev,
|
||||
dvc_only=self.args.dvc_only,
|
||||
config=self.args.config,
|
||||
remote=self.args.remote,
|
||||
remote_config=self.args.remote_config,
|
||||
maxdepth=self.args.level,
|
||||
)
|
||||
show_tree(
|
||||
entries,
|
||||
with_color=True,
|
||||
with_size=self.args.size,
|
||||
with_hash=self.args.show_hash,
|
||||
)
|
||||
return 0
|
||||
|
||||
def _show_list(self):
|
||||
from dvc.repo import Repo
|
||||
|
||||
try:
|
||||
entries = Repo.ls(
|
||||
self.args.url,
|
||||
self.args.path,
|
||||
rev=self.args.rev,
|
||||
recursive=self.args.recursive,
|
||||
dvc_only=self.args.dvc_only,
|
||||
config=self.args.config,
|
||||
remote=self.args.remote,
|
||||
remote_config=self.args.remote_config,
|
||||
entries = Repo.ls(
|
||||
self.args.url,
|
||||
self.args.path,
|
||||
rev=self.args.rev,
|
||||
recursive=self.args.recursive,
|
||||
dvc_only=self.args.dvc_only,
|
||||
config=self.args.config,
|
||||
remote=self.args.remote,
|
||||
remote_config=self.args.remote_config,
|
||||
maxdepth=self.args.level,
|
||||
)
|
||||
if self.args.json:
|
||||
ui.write_json(entries)
|
||||
elif entries:
|
||||
show_entries(
|
||||
entries,
|
||||
with_color=True,
|
||||
with_size=self.args.size,
|
||||
with_hash=self.args.show_hash,
|
||||
)
|
||||
if self.args.json:
|
||||
ui.write_json(entries)
|
||||
elif entries:
|
||||
show_entries(entries, with_color=True, with_size=self.args.size)
|
||||
return 0
|
||||
return 0
|
||||
|
||||
def run(self):
|
||||
if self.args.tree and self.args.json:
|
||||
raise DvcException("Cannot use --tree and --json options together.")
|
||||
|
||||
try:
|
||||
if self.args.tree:
|
||||
return self._show_tree()
|
||||
return self._show_list()
|
||||
except FileNotFoundError:
|
||||
logger.exception("")
|
||||
return 1
|
||||
@@ -85,6 +214,19 @@ def add_parser(subparsers, parent_parser):
|
||||
action="store_true",
|
||||
help="Recursively list files.",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"-T",
|
||||
"--tree",
|
||||
action="store_true",
|
||||
help="Recurse into directories as a tree.",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"-L",
|
||||
"--level",
|
||||
metavar="depth",
|
||||
type=int,
|
||||
help="Limit the depth of recursion.",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"--dvc-only", action="store_true", help="Show only DVC outputs."
|
||||
)
|
||||
@@ -123,6 +265,12 @@ def add_parser(subparsers, parent_parser):
|
||||
),
|
||||
)
|
||||
list_parser.add_argument("--size", action="store_true", help="Show sizes.")
|
||||
list_parser.add_argument(
|
||||
"--show-hash",
|
||||
help="Display hash value for each item.",
|
||||
action="store_true",
|
||||
default=False,
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"path",
|
||||
nargs="?",
|
||||
|
||||
@@ -32,7 +32,9 @@ class LsColors:
|
||||
if entry.get("isexec", False):
|
||||
return self._format(text, code="ex")
|
||||
|
||||
_, ext = os.path.splitext(text)
|
||||
stem, ext = os.path.splitext(text)
|
||||
if not ext and stem.startswith("."):
|
||||
ext = stem
|
||||
return self._format(text, ext=ext)
|
||||
|
||||
def _format(self, text, code=None, ext=None):
|
||||
|
||||
+1
-1
@@ -571,7 +571,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
||||
|
||||
os.makedirs(lpath, exist_ok=True)
|
||||
for d in _dirs:
|
||||
os.mkdir(d)
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
def get_file(arg: tuple[FileSystem, tuple[str, str, Optional[dict]]]):
|
||||
fs, (src, dest, info) = arg
|
||||
|
||||
+6
-3
@@ -752,8 +752,9 @@ class Output:
|
||||
granular = (
|
||||
self.is_dir_checksum and filter_info and filter_info != self.fs_path
|
||||
)
|
||||
hardlink = relink and next(iter(self.cache.cache_types), None) == "hardlink"
|
||||
if granular:
|
||||
obj = self._commit_granular_dir(filter_info, hardlink=False)
|
||||
obj = self._commit_granular_dir(filter_info, hardlink=hardlink)
|
||||
else:
|
||||
staging, _, obj = self._build(
|
||||
self.cache,
|
||||
@@ -771,7 +772,7 @@ class Output:
|
||||
self.cache,
|
||||
{obj.hash_info},
|
||||
shallow=False,
|
||||
hardlink=False,
|
||||
hardlink=hardlink,
|
||||
callback=cb,
|
||||
)
|
||||
if relink:
|
||||
@@ -1399,12 +1400,14 @@ class Output:
|
||||
|
||||
assert staging
|
||||
assert obj.hash_info
|
||||
|
||||
hardlink = relink and next(iter(self.cache.cache_types), None) == "hardlink"
|
||||
with TqdmCallback(desc=f"Adding {self} to cache", unit="file") as cb:
|
||||
otransfer(
|
||||
staging,
|
||||
self.cache,
|
||||
{obj.hash_info},
|
||||
hardlink=False,
|
||||
hardlink=hardlink,
|
||||
shallow=False,
|
||||
callback=cb,
|
||||
)
|
||||
|
||||
@@ -176,8 +176,9 @@ class Container(Node, ABC):
|
||||
return value
|
||||
if isinstance(value, (list, dict)):
|
||||
assert meta
|
||||
container = CtxDict if isinstance(value, dict) else CtxList
|
||||
return container(value, meta=meta)
|
||||
if isinstance(value, dict):
|
||||
return CtxDict(value, meta=meta)
|
||||
return CtxList(value, meta=meta)
|
||||
msg = f"Unsupported value of type '{type(value).__name__}' in '{meta}'"
|
||||
raise TypeError(msg)
|
||||
|
||||
|
||||
@@ -205,7 +205,7 @@ class Experiments:
|
||||
Experiment will be derived from the current workspace.
|
||||
"""
|
||||
|
||||
name = kwargs.get("name", None)
|
||||
name = kwargs.get("name")
|
||||
baseline_sha = kwargs.get("baseline_rev") or self.repo.scm.get_rev()
|
||||
|
||||
if name:
|
||||
|
||||
+103
-45
@@ -4,8 +4,44 @@ from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
|
||||
|
||||
def _open_repo(
|
||||
url: str,
|
||||
rev: Optional[str] = None,
|
||||
config: Union[None, dict[str, Any], str] = None,
|
||||
remote: Optional[str] = None,
|
||||
remote_config: Optional[dict] = None,
|
||||
):
|
||||
from dvc.config import Config
|
||||
|
||||
from . import Repo
|
||||
|
||||
if config and not isinstance(config, dict):
|
||||
config_dict = Config.load_file(config)
|
||||
else:
|
||||
config_dict = None
|
||||
|
||||
return Repo.open(
|
||||
url,
|
||||
rev=rev,
|
||||
subrepos=True,
|
||||
uninitialized=True,
|
||||
config=config_dict,
|
||||
remote=remote,
|
||||
remote_config=remote_config,
|
||||
)
|
||||
|
||||
|
||||
def _adapt_info(info: dict[str, Any]) -> dict[str, Any]:
|
||||
dvc_info = info.get("dvc_info", {})
|
||||
return {
|
||||
"isout": dvc_info.get("isout", False),
|
||||
"isdir": info["type"] == "directory",
|
||||
"isexec": info.get("isexec", False),
|
||||
"size": info.get("size"),
|
||||
"md5": dvc_info.get("md5") or dvc_info.get("md5-dos2unix"),
|
||||
}
|
||||
|
||||
|
||||
def ls(
|
||||
url: str,
|
||||
@@ -16,6 +52,7 @@ def ls(
|
||||
config: Union[None, dict[str, Any], str] = None,
|
||||
remote: Optional[str] = None,
|
||||
remote_config: Optional[dict] = None,
|
||||
maxdepth: Optional[int] = None,
|
||||
):
|
||||
"""Methods for getting files and outputs for the repo.
|
||||
|
||||
@@ -41,60 +78,58 @@ def ls(
|
||||
"isexec": bool,
|
||||
}
|
||||
"""
|
||||
from dvc.config import Config
|
||||
|
||||
from . import Repo
|
||||
|
||||
if config and not isinstance(config, dict):
|
||||
config_dict = Config.load_file(config)
|
||||
else:
|
||||
config_dict = None
|
||||
|
||||
with Repo.open(
|
||||
url,
|
||||
rev=rev,
|
||||
subrepos=True,
|
||||
uninitialized=True,
|
||||
config=config_dict,
|
||||
remote=remote,
|
||||
remote_config=remote_config,
|
||||
) as repo:
|
||||
with _open_repo(url, rev, config, remote, remote_config) as repo:
|
||||
path = path or ""
|
||||
fs: DVCFileSystem = repo.dvcfs
|
||||
fs_path = fs.from_os_path(path)
|
||||
return _ls(fs, fs_path, recursive, dvc_only, maxdepth)
|
||||
|
||||
ret = _ls(repo, path, recursive, dvc_only)
|
||||
|
||||
ret_list = []
|
||||
for path, info in ret.items():
|
||||
info["path"] = path
|
||||
ret_list.append(info)
|
||||
ret_list.sort(key=lambda f: f["path"])
|
||||
return ret_list
|
||||
def ls_tree(
|
||||
url: str,
|
||||
path: Optional[str] = None,
|
||||
rev: Optional[str] = None,
|
||||
dvc_only: bool = False,
|
||||
config: Union[None, dict[str, Any], str] = None,
|
||||
remote: Optional[str] = None,
|
||||
remote_config: Optional[dict] = None,
|
||||
maxdepth: Optional[int] = None,
|
||||
):
|
||||
with _open_repo(url, rev, config, remote, remote_config) as repo:
|
||||
path = path or ""
|
||||
fs: DVCFileSystem = repo.dvcfs
|
||||
fs_path = fs.from_os_path(path)
|
||||
return _ls_tree(fs, fs_path, dvc_only, maxdepth)
|
||||
|
||||
|
||||
def _ls(
|
||||
repo: "Repo",
|
||||
fs: "DVCFileSystem",
|
||||
path: str,
|
||||
recursive: Optional[bool] = None,
|
||||
dvc_only: bool = False,
|
||||
maxdepth: Optional[int] = None,
|
||||
):
|
||||
fs: DVCFileSystem = repo.dvcfs
|
||||
fs_path = fs.from_os_path(path)
|
||||
|
||||
fs_path = fs.info(fs_path)["name"]
|
||||
fs_path = fs.info(path)["name"]
|
||||
|
||||
infos = {}
|
||||
if fs.isfile(fs_path):
|
||||
infos[os.path.basename(path)] = fs.info(fs_path)
|
||||
|
||||
# ignore maxdepth only if recursive is not set
|
||||
maxdepth = maxdepth if recursive else None
|
||||
if maxdepth == 0 or fs.isfile(fs_path):
|
||||
infos[os.path.basename(path) or os.curdir] = fs.info(fs_path)
|
||||
else:
|
||||
for root, dirs, files in fs.walk(
|
||||
fs_path, dvcfiles=True, dvc_only=dvc_only, detail=True
|
||||
fs_path,
|
||||
dvcfiles=True,
|
||||
dvc_only=dvc_only,
|
||||
detail=True,
|
||||
maxdepth=maxdepth,
|
||||
):
|
||||
if not recursive:
|
||||
files.update(dirs)
|
||||
|
||||
parts = fs.relparts(root, fs_path)
|
||||
if parts == (".",):
|
||||
parts = ()
|
||||
if not recursive or (maxdepth and len(parts) >= maxdepth - 1):
|
||||
files.update(dirs)
|
||||
|
||||
for name, entry in files.items():
|
||||
infos[os.path.join(*parts, name)] = entry
|
||||
@@ -102,14 +137,37 @@ def _ls(
|
||||
if not recursive:
|
||||
break
|
||||
|
||||
ret = {}
|
||||
for name, info in infos.items():
|
||||
dvc_info = info.get("dvc_info", {})
|
||||
ret[name] = {
|
||||
"isout": dvc_info.get("isout", False),
|
||||
"isdir": info["type"] == "directory",
|
||||
"isexec": info.get("isexec", False),
|
||||
"size": info.get("size"),
|
||||
}
|
||||
ret_list = []
|
||||
for p, info in sorted(infos.items(), key=lambda x: x[0]):
|
||||
_info = _adapt_info(info)
|
||||
_info["path"] = p
|
||||
ret_list.append(_info)
|
||||
return ret_list
|
||||
|
||||
|
||||
def _ls_tree(
|
||||
fs, path, dvc_only: bool = False, maxdepth: Optional[int] = None, _info=None
|
||||
):
|
||||
ret = {}
|
||||
info = _info or fs.info(path)
|
||||
|
||||
path = info["name"].rstrip(fs.sep) or os.curdir
|
||||
name = path.rsplit("/", 1)[-1]
|
||||
ls_info = _adapt_info(info)
|
||||
ls_info["path"] = path
|
||||
|
||||
recurse = maxdepth is None or maxdepth > 0
|
||||
if recurse and info["type"] == "directory":
|
||||
infos = fs.ls(path, dvcfiles=True, dvc_only=dvc_only, detail=True)
|
||||
infos.sort(key=lambda f: f["name"])
|
||||
maxdepth = maxdepth - 1 if maxdepth is not None else None
|
||||
contents = {}
|
||||
for info in infos:
|
||||
d = _ls_tree(
|
||||
fs, info["name"], dvc_only=dvc_only, maxdepth=maxdepth, _info=info
|
||||
)
|
||||
contents.update(d)
|
||||
ls_info["contents"] = contents
|
||||
|
||||
ret[name] = ls_info
|
||||
return ret
|
||||
|
||||
@@ -83,7 +83,7 @@ class RawData:
|
||||
def create_stage(cls: type[_T], repo, path, **kwargs) -> _T:
|
||||
from dvc.dvcfile import check_dvcfile_path
|
||||
|
||||
wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
|
||||
wdir = os.path.abspath(kwargs.get("wdir") or os.curdir)
|
||||
path = os.path.abspath(path)
|
||||
|
||||
check_dvcfile_path(repo, path)
|
||||
@@ -435,7 +435,7 @@ class Stage(params.StageParams):
|
||||
elif not self.changed(True, upstream) and pull:
|
||||
try:
|
||||
logger.info("Pulling data for %s", self)
|
||||
self.repo.pull(self.addressing, jobs=kwargs.get("jobs", None))
|
||||
self.repo.pull(self.addressing, jobs=kwargs.get("jobs"))
|
||||
self.checkout()
|
||||
return None
|
||||
except CheckoutError:
|
||||
@@ -609,7 +609,7 @@ class Stage(params.StageParams):
|
||||
self.remove_outs(ignore_remove=False, force=False)
|
||||
|
||||
if (self.is_import and not self.frozen) or self.is_partial_import:
|
||||
self._sync_import(dry, force, kwargs.get("jobs", None), no_download)
|
||||
self._sync_import(dry, force, kwargs.get("jobs"), no_download)
|
||||
elif not self.frozen and self.cmd:
|
||||
self._run_stage(dry, force, **kwargs)
|
||||
elif not dry:
|
||||
|
||||
+8
-1
@@ -300,6 +300,7 @@ class Console:
|
||||
header_styles: Optional[Union[dict[str, "Styles"], Sequence["Styles"]]] = None,
|
||||
row_styles: Optional[Sequence["Styles"]] = None,
|
||||
borders: Union[bool, str] = False,
|
||||
colalign: Optional[tuple[str, ...]] = None,
|
||||
) -> None:
|
||||
from dvc.ui import table as t
|
||||
|
||||
@@ -321,7 +322,13 @@ class Console:
|
||||
return
|
||||
|
||||
return t.plain_table(
|
||||
self, data, headers, markdown=markdown, pager=pager, force=force
|
||||
self,
|
||||
data,
|
||||
headers,
|
||||
markdown=markdown,
|
||||
pager=pager,
|
||||
force=force,
|
||||
colalign=colalign,
|
||||
)
|
||||
|
||||
def status(self, status: str, **kwargs: Any) -> "Status":
|
||||
|
||||
@@ -29,6 +29,7 @@ def plain_table(
|
||||
markdown: bool = False,
|
||||
pager: bool = False,
|
||||
force: bool = True,
|
||||
colalign: Optional[tuple[str, ...]] = None,
|
||||
) -> None:
|
||||
from funcy import nullcontext
|
||||
from tabulate import tabulate
|
||||
@@ -40,6 +41,7 @@ def plain_table(
|
||||
disable_numparse=True,
|
||||
# None will be shown as "" by default, overriding
|
||||
missingval="-",
|
||||
colalign=colalign,
|
||||
)
|
||||
if markdown:
|
||||
# NOTE: md table is incomplete without the trailing newline
|
||||
|
||||
@@ -382,7 +382,7 @@ def glob_targets(targets, glob=True, recursive=True):
|
||||
|
||||
def error_handler(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
onerror = kwargs.get("onerror", None)
|
||||
onerror = kwargs.get("onerror")
|
||||
result = {}
|
||||
|
||||
try:
|
||||
|
||||
@@ -61,7 +61,7 @@ def _load_data(
|
||||
):
|
||||
open_fn = fs.open if fs else open
|
||||
encoding = "utf-8"
|
||||
with open_fn(path, encoding=encoding, **kwargs) as fd: # type: ignore[operator]
|
||||
with open_fn(path, encoding=encoding, **kwargs) as fd: # type: ignore[arg-type]
|
||||
with reraise(UnicodeDecodeError, EncodingError(path, encoding)):
|
||||
return parser(fd.read(), path)
|
||||
|
||||
@@ -74,7 +74,7 @@ def _dump_data(
|
||||
**dumper_args,
|
||||
):
|
||||
open_fn = fs.open if fs else open
|
||||
with open_fn(path, "w+", encoding="utf-8") as fd: # type: ignore[operator]
|
||||
with open_fn(path, "w+", encoding="utf-8") as fd: # type: ignore[call-overload]
|
||||
dumper(data, fd, **dumper_args)
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -65,7 +65,7 @@ dependencies = [
|
||||
"requests>=2.22",
|
||||
"rich>=12",
|
||||
"ruamel.yaml>=0.17.11",
|
||||
"scmrepo>=3.3.4,<4",
|
||||
"scmrepo>=3.3.8,<4",
|
||||
"shortuuid>=0.5",
|
||||
"shtab<2,>=1.3.4",
|
||||
"tabulate>=0.8.7",
|
||||
@@ -83,7 +83,7 @@ gdrive = ["dvc-gdrive>=3,<4"]
|
||||
gs = ["dvc-gs>=3,<4"]
|
||||
hdfs = ["dvc-hdfs>=3,<4"]
|
||||
lint = [
|
||||
"mypy==1.11.2",
|
||||
"mypy==1.12.1",
|
||||
"pandas-stubs",
|
||||
"types-colorama",
|
||||
"types-psutil",
|
||||
|
||||
+357
-11
@@ -2,10 +2,12 @@ import os
|
||||
import shutil
|
||||
import textwrap
|
||||
from operator import itemgetter
|
||||
from os.path import join
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.ls import ls_tree
|
||||
from dvc.scm import CloneError
|
||||
|
||||
FS_STRUCTURE = {
|
||||
@@ -222,7 +224,12 @@ def test_ls_repo_with_missed_path_dvc_only(tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
Repo.ls(os.fspath(tmp_dir), path="missed_path", recursive=True, dvc_only=True)
|
||||
Repo.ls(
|
||||
os.fspath(tmp_dir),
|
||||
path="missed_path",
|
||||
recursive=True,
|
||||
dvc_only=True,
|
||||
)
|
||||
|
||||
|
||||
def test_ls_repo_with_removed_dvc_dir(tmp_dir, dvc, scm):
|
||||
@@ -467,20 +474,49 @@ def test_ls_granular(erepo_dir, M):
|
||||
|
||||
entries = Repo.ls(os.fspath(erepo_dir), os.path.join("dir", "subdir"))
|
||||
assert entries == [
|
||||
{"isout": True, "isdir": False, "isexec": False, "path": "bar", "size": 3},
|
||||
{"isout": True, "isdir": False, "isexec": False, "path": "foo", "size": 3},
|
||||
{
|
||||
"isout": True,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"path": "bar",
|
||||
"size": 3,
|
||||
"md5": "37b51d194a7513e45b56f6524f2d51f2",
|
||||
},
|
||||
{
|
||||
"isout": True,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"path": "foo",
|
||||
"size": 3,
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
},
|
||||
]
|
||||
|
||||
entries = Repo.ls(os.fspath(erepo_dir), "dir")
|
||||
assert entries == [
|
||||
{"isout": True, "isdir": False, "isexec": False, "path": "1", "size": 1},
|
||||
{"isout": True, "isdir": False, "isexec": False, "path": "2", "size": 1},
|
||||
{
|
||||
"isout": True,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"path": "1",
|
||||
"size": 1,
|
||||
"md5": "c4ca4238a0b923820dcc509a6f75849b",
|
||||
},
|
||||
{
|
||||
"isout": True,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"path": "2",
|
||||
"size": 1,
|
||||
"md5": "c81e728d9d4c2f636f067f89cc14862c",
|
||||
},
|
||||
{
|
||||
"isout": True,
|
||||
"isdir": True,
|
||||
"isexec": False,
|
||||
"path": "subdir",
|
||||
"size": M.instance_of(int),
|
||||
"md5": None,
|
||||
},
|
||||
]
|
||||
|
||||
@@ -506,14 +542,42 @@ def test_ls_target(erepo_dir, use_scm):
|
||||
return Repo.ls(os.fspath(erepo_dir), path)
|
||||
|
||||
assert _ls(os.path.join("dir", "1")) == [
|
||||
{"isout": isout, "isdir": False, "isexec": False, "path": "1", "size": 1}
|
||||
{
|
||||
"isout": isout,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"path": "1",
|
||||
"size": 1,
|
||||
"md5": "c4ca4238a0b923820dcc509a6f75849b" if not use_scm else None,
|
||||
}
|
||||
]
|
||||
assert _ls(os.path.join("dir", "subdir", "foo")) == [
|
||||
{"isout": isout, "isdir": False, "isexec": False, "path": "foo", "size": 3}
|
||||
{
|
||||
"isout": isout,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"path": "foo",
|
||||
"size": 3,
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8" if not use_scm else None,
|
||||
}
|
||||
]
|
||||
assert _ls(os.path.join("dir", "subdir")) == [
|
||||
{"isdir": False, "isexec": 0, "isout": isout, "path": "bar", "size": 3},
|
||||
{"isdir": False, "isexec": 0, "isout": isout, "path": "foo", "size": 3},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": isout,
|
||||
"path": "bar",
|
||||
"size": 3,
|
||||
"md5": "37b51d194a7513e45b56f6524f2d51f2" if not use_scm else None,
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": isout,
|
||||
"path": "foo",
|
||||
"size": 3,
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8" if not use_scm else None,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@@ -576,6 +640,7 @@ def test_broken_symlink(tmp_dir, dvc, M):
|
||||
"isexec": False,
|
||||
"path": ".dvcignore",
|
||||
"size": M.instance_of(int),
|
||||
"md5": None,
|
||||
},
|
||||
{
|
||||
"isout": False,
|
||||
@@ -583,6 +648,7 @@ def test_broken_symlink(tmp_dir, dvc, M):
|
||||
"isexec": False,
|
||||
"path": "link",
|
||||
"size": 0,
|
||||
"md5": None,
|
||||
},
|
||||
]
|
||||
|
||||
@@ -614,14 +680,23 @@ def test_ls_broken_dir(tmp_dir, dvc, M):
|
||||
"isout": False,
|
||||
"path": ".dvcignore",
|
||||
"size": M.instance_of(int),
|
||||
"md5": None,
|
||||
},
|
||||
{
|
||||
"isdir": True,
|
||||
"isexec": False,
|
||||
"isout": True,
|
||||
"path": "broken",
|
||||
"size": 3,
|
||||
"md5": "630bd47b538d2a513c7d267d07e0bc44.dir",
|
||||
},
|
||||
{"isdir": True, "isexec": False, "isout": True, "path": "broken", "size": 3},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"isout": False,
|
||||
"path": "broken.dvc",
|
||||
"size": M.instance_of(int),
|
||||
"md5": None,
|
||||
},
|
||||
{
|
||||
"isdir": True,
|
||||
@@ -629,6 +704,7 @@ def test_ls_broken_dir(tmp_dir, dvc, M):
|
||||
"isout": True,
|
||||
"path": "dir",
|
||||
"size": M.instance_of(int),
|
||||
"md5": "91aaa9bb58b657d623ef143b195a67e4.dir",
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
@@ -636,14 +712,23 @@ def test_ls_broken_dir(tmp_dir, dvc, M):
|
||||
"isout": False,
|
||||
"path": "dir.dvc",
|
||||
"size": M.instance_of(int),
|
||||
"md5": None,
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"isout": True,
|
||||
"path": "foo",
|
||||
"size": 3,
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
},
|
||||
{"isdir": False, "isexec": False, "isout": True, "path": "foo", "size": 3},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"isout": False,
|
||||
"path": "foo.dvc",
|
||||
"size": M.instance_of(int),
|
||||
"md5": None,
|
||||
},
|
||||
]
|
||||
|
||||
@@ -652,3 +737,264 @@ def test_ls_broken_dir(tmp_dir, dvc, M):
|
||||
|
||||
with pytest.raises(DataIndexDirError):
|
||||
Repo.ls(os.fspath(tmp_dir), recursive=True)
|
||||
|
||||
|
||||
def test_ls_maxdepth(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
files = Repo.ls(os.fspath(tmp_dir), "structure.xml", maxdepth=0, recursive=True)
|
||||
match_files(files, ((("structure.xml",), True),))
|
||||
|
||||
files = Repo.ls(os.fspath(tmp_dir), maxdepth=0, recursive=True)
|
||||
match_files(files, (((os.curdir,), False),))
|
||||
|
||||
files = Repo.ls(os.fspath(tmp_dir), maxdepth=1, recursive=True)
|
||||
match_files(
|
||||
files,
|
||||
(
|
||||
((".dvcignore",), False),
|
||||
((".gitignore",), False),
|
||||
(("README.md",), False),
|
||||
(("structure.xml.dvc",), False),
|
||||
(("model",), False),
|
||||
(("data",), False),
|
||||
(("structure.xml",), True),
|
||||
),
|
||||
)
|
||||
files = Repo.ls(os.fspath(tmp_dir), maxdepth=2, recursive=True)
|
||||
match_files(
|
||||
files,
|
||||
(
|
||||
((".dvcignore",), False),
|
||||
((".gitignore",), False),
|
||||
(("README.md",), False),
|
||||
((join("data", "subcontent"),), False),
|
||||
((join("model", ".gitignore"),), False),
|
||||
((join("model", "people.csv"),), True),
|
||||
((join("model", "people.csv.dvc"),), False),
|
||||
((join("model", "script.py"),), False),
|
||||
((join("model", "train.py"),), False),
|
||||
(("structure.xml",), True),
|
||||
(("structure.xml.dvc",), False),
|
||||
),
|
||||
)
|
||||
|
||||
files = Repo.ls(os.fspath(tmp_dir), maxdepth=3, recursive=True)
|
||||
match_files(
|
||||
files,
|
||||
(
|
||||
((".dvcignore",), False),
|
||||
((".gitignore",), False),
|
||||
(("README.md",), False),
|
||||
((join("data", "subcontent", ".gitignore"),), False),
|
||||
((join("data", "subcontent", "data.xml"),), True),
|
||||
((join("data", "subcontent", "data.xml.dvc"),), False),
|
||||
((join("data", "subcontent", "statistics"),), False),
|
||||
((join("model", ".gitignore"),), False),
|
||||
((join("model", "people.csv"),), True),
|
||||
((join("model", "people.csv.dvc"),), False),
|
||||
((join("model", "script.py"),), False),
|
||||
((join("model", "train.py"),), False),
|
||||
((join("structure.xml"),), True),
|
||||
((join("structure.xml.dvc"),), False),
|
||||
),
|
||||
)
|
||||
|
||||
files = Repo.ls(os.fspath(tmp_dir), maxdepth=4, recursive=True)
|
||||
match_files(
|
||||
files,
|
||||
(
|
||||
((".dvcignore",), False),
|
||||
((".gitignore",), False),
|
||||
(("README.md",), False),
|
||||
((join("data", "subcontent", ".gitignore"),), False),
|
||||
((join("data", "subcontent", "data.xml"),), True),
|
||||
((join("data", "subcontent", "data.xml.dvc"),), False),
|
||||
((join("data", "subcontent", "statistics", ".gitignore"),), False),
|
||||
((join("data", "subcontent", "statistics", "data.csv"),), True),
|
||||
((join("data", "subcontent", "statistics", "data.csv.dvc"),), False),
|
||||
((join("model", ".gitignore"),), False),
|
||||
((join("model", "people.csv"),), True),
|
||||
((join("model", "people.csv.dvc"),), False),
|
||||
((join("model", "script.py"),), False),
|
||||
((join("model", "train.py"),), False),
|
||||
(("structure.xml",), True),
|
||||
(("structure.xml.dvc",), False),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _simplify_tree(files):
|
||||
ret = {}
|
||||
for path, info in files.items():
|
||||
if content := info.get("contents"):
|
||||
ret[path] = _simplify_tree(content)
|
||||
else:
|
||||
ret[path] = None
|
||||
return ret
|
||||
|
||||
|
||||
def test_ls_tree(M, tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), "structure.xml")
|
||||
assert _simplify_tree(files) == {"structure.xml": None}
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir))
|
||||
|
||||
expected = {
|
||||
".": {
|
||||
".dvcignore": None,
|
||||
".gitignore": None,
|
||||
"README.md": None,
|
||||
"data": {
|
||||
"subcontent": {
|
||||
".gitignore": None,
|
||||
"data.xml": None,
|
||||
"data.xml.dvc": None,
|
||||
"statistics": {
|
||||
".gitignore": None,
|
||||
"data.csv": None,
|
||||
"data.csv.dvc": None,
|
||||
},
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
".gitignore": None,
|
||||
"people.csv": None,
|
||||
"people.csv.dvc": None,
|
||||
"script.py": None,
|
||||
"train.py": None,
|
||||
},
|
||||
"structure.xml": None,
|
||||
"structure.xml.dvc": None,
|
||||
}
|
||||
}
|
||||
assert _simplify_tree(files) == expected
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), "model")
|
||||
assert _simplify_tree(files) == {
|
||||
"model": {
|
||||
".gitignore": None,
|
||||
"people.csv": None,
|
||||
"people.csv.dvc": None,
|
||||
"script.py": None,
|
||||
"train.py": None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_ls_tree_dvc_only(M, tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), dvc_only=True)
|
||||
|
||||
expected = {
|
||||
".": {
|
||||
"data": {
|
||||
"subcontent": {"data.xml": None, "statistics": {"data.csv": None}}
|
||||
},
|
||||
"model": {"people.csv": None},
|
||||
"structure.xml": None,
|
||||
}
|
||||
}
|
||||
assert _simplify_tree(files) == expected
|
||||
|
||||
|
||||
def test_ls_tree_maxdepth(M, tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), maxdepth=0)
|
||||
assert _simplify_tree(files) == {".": None}
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), maxdepth=1)
|
||||
assert _simplify_tree(files) == {
|
||||
".": {
|
||||
".dvcignore": None,
|
||||
".gitignore": None,
|
||||
"README.md": None,
|
||||
"data": None,
|
||||
"model": None,
|
||||
"structure.xml": None,
|
||||
"structure.xml.dvc": None,
|
||||
}
|
||||
}
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), maxdepth=2)
|
||||
assert _simplify_tree(files) == {
|
||||
".": {
|
||||
".dvcignore": None,
|
||||
".gitignore": None,
|
||||
"README.md": None,
|
||||
"data": {"subcontent": None},
|
||||
"model": {
|
||||
".gitignore": None,
|
||||
"people.csv": None,
|
||||
"people.csv.dvc": None,
|
||||
"script.py": None,
|
||||
"train.py": None,
|
||||
},
|
||||
"structure.xml": None,
|
||||
"structure.xml.dvc": None,
|
||||
}
|
||||
}
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), maxdepth=3)
|
||||
assert _simplify_tree(files) == {
|
||||
".": {
|
||||
".dvcignore": None,
|
||||
".gitignore": None,
|
||||
"README.md": None,
|
||||
"data": {
|
||||
"subcontent": {
|
||||
".gitignore": None,
|
||||
"data.xml": None,
|
||||
"data.xml.dvc": None,
|
||||
"statistics": None,
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
".gitignore": None,
|
||||
"people.csv": None,
|
||||
"people.csv.dvc": None,
|
||||
"script.py": None,
|
||||
"train.py": None,
|
||||
},
|
||||
"structure.xml": None,
|
||||
"structure.xml.dvc": None,
|
||||
}
|
||||
}
|
||||
|
||||
files = ls_tree(os.fspath(tmp_dir), maxdepth=4)
|
||||
assert _simplify_tree(files) == {
|
||||
".": {
|
||||
".dvcignore": None,
|
||||
".gitignore": None,
|
||||
"README.md": None,
|
||||
"data": {
|
||||
"subcontent": {
|
||||
".gitignore": None,
|
||||
"data.xml": None,
|
||||
"data.xml.dvc": None,
|
||||
"statistics": {
|
||||
".gitignore": None,
|
||||
"data.csv": None,
|
||||
"data.csv.dvc": None,
|
||||
},
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
".gitignore": None,
|
||||
"people.csv": None,
|
||||
"people.csv.dvc": None,
|
||||
"script.py": None,
|
||||
"train.py": None,
|
||||
},
|
||||
"structure.xml": None,
|
||||
"structure.xml.dvc": None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import json
|
||||
|
||||
from dvc.cli import parse_args
|
||||
from dvc.commands.ls import CmdList
|
||||
from dvc.commands.ls import CmdList, show_tree
|
||||
|
||||
|
||||
def _test_cli(mocker, *args):
|
||||
@@ -27,6 +27,7 @@ def test_list(mocker):
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -42,6 +43,7 @@ def test_list_recursive(mocker):
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -57,6 +59,7 @@ def test_list_git_ssh_rev(mocker):
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -73,6 +76,7 @@ def test_list_targets(mocker):
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -88,6 +92,7 @@ def test_list_outputs_only(mocker):
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -114,6 +119,65 @@ def test_list_config(mocker):
|
||||
config="myconfig",
|
||||
remote="myremote",
|
||||
remote_config={"k1": "v1", "k2": "v2"},
|
||||
maxdepth=None,
|
||||
)
|
||||
|
||||
|
||||
def test_list_level(mocker):
|
||||
url = "local_dir"
|
||||
m = _test_cli(mocker, url, None, "--level", "1")
|
||||
m.assert_called_once_with(
|
||||
url,
|
||||
None,
|
||||
recursive=False,
|
||||
rev=None,
|
||||
dvc_only=False,
|
||||
config=None,
|
||||
remote=None,
|
||||
remote_config=None,
|
||||
maxdepth=1,
|
||||
)
|
||||
|
||||
|
||||
def test_list_tree(mocker):
|
||||
url = "[email protected]:repo"
|
||||
cli_args = parse_args(
|
||||
[
|
||||
"list",
|
||||
url,
|
||||
"local_dir",
|
||||
"--rev",
|
||||
"123",
|
||||
"--tree",
|
||||
"--show-hash",
|
||||
"--size",
|
||||
"--level",
|
||||
"2",
|
||||
"--dvc-only",
|
||||
"--config",
|
||||
"myconfig",
|
||||
"--remote",
|
||||
"myremote",
|
||||
"--remote-config",
|
||||
"k1=v1",
|
||||
"k2=v2",
|
||||
]
|
||||
)
|
||||
assert cli_args.func == CmdList
|
||||
|
||||
cmd = cli_args.func(cli_args)
|
||||
m = mocker.patch("dvc.repo.ls.ls_tree")
|
||||
|
||||
assert cmd.run() == 0
|
||||
m.assert_called_once_with(
|
||||
url,
|
||||
"local_dir",
|
||||
rev="123",
|
||||
dvc_only=True,
|
||||
config="myconfig",
|
||||
remote="myremote",
|
||||
remote_config={"k1": "v1", "k2": "v2"},
|
||||
maxdepth=2,
|
||||
)
|
||||
|
||||
|
||||
@@ -165,6 +229,268 @@ def test_show_colors(mocker, capsys, monkeypatch):
|
||||
]
|
||||
|
||||
|
||||
def test_show_size(mocker, capsys):
|
||||
cli_args = parse_args(["list", "local_dir", "--size"])
|
||||
assert cli_args.func == CmdList
|
||||
cmd = cli_args.func(cli_args)
|
||||
|
||||
result = [
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".dvcignore",
|
||||
"size": 100,
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".gitignore",
|
||||
"size": 200,
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": "README.md",
|
||||
"size": 100_000,
|
||||
},
|
||||
]
|
||||
mocker.patch("dvc.repo.Repo.ls", return_value=result)
|
||||
|
||||
assert cmd.run() == 0
|
||||
out, _ = capsys.readouterr()
|
||||
entries = out.splitlines()
|
||||
|
||||
assert entries == [" 100 .dvcignore", " 200 .gitignore", "97.7k README.md"]
|
||||
|
||||
|
||||
def test_show_hash(mocker, capsys):
|
||||
cli_args = parse_args(["list", "local_dir", "--show-hash"])
|
||||
assert cli_args.func == CmdList
|
||||
cmd = cli_args.func(cli_args)
|
||||
|
||||
result = [
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".dvcignore",
|
||||
"md5": "123",
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".gitignore",
|
||||
"md5": "456",
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": "README.md",
|
||||
"md5": "789",
|
||||
},
|
||||
]
|
||||
mocker.patch("dvc.repo.Repo.ls", return_value=result)
|
||||
|
||||
assert cmd.run() == 0
|
||||
out, _ = capsys.readouterr()
|
||||
entries = out.splitlines()
|
||||
|
||||
assert entries == ["123 .dvcignore", "456 .gitignore", "789 README.md"]
|
||||
|
||||
|
||||
def test_show_size_and_hash(mocker, capsys):
|
||||
cli_args = parse_args(["list", "local_dir", "--size", "--show-hash"])
|
||||
assert cli_args.func == CmdList
|
||||
cmd = cli_args.func(cli_args)
|
||||
|
||||
result = [
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".dvcignore",
|
||||
"size": 100,
|
||||
"md5": "123",
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": ".gitignore",
|
||||
"size": 200,
|
||||
"md5": "456",
|
||||
},
|
||||
{
|
||||
"isdir": False,
|
||||
"isexec": 0,
|
||||
"isout": False,
|
||||
"path": "README.md",
|
||||
"size": 100_000,
|
||||
"md5": "789",
|
||||
},
|
||||
]
|
||||
mocker.patch("dvc.repo.Repo.ls", return_value=result)
|
||||
|
||||
assert cmd.run() == 0
|
||||
out, _ = capsys.readouterr()
|
||||
entries = out.splitlines()
|
||||
|
||||
assert entries == [
|
||||
" 100 123 .dvcignore",
|
||||
" 200 456 .gitignore",
|
||||
"97.7k 789 README.md",
|
||||
]
|
||||
|
||||
|
||||
def test_show_tree(capsys):
|
||||
entries = {
|
||||
"data": {
|
||||
"isout": True,
|
||||
"isdir": True,
|
||||
"isexec": False,
|
||||
"size": 192,
|
||||
"path": "data",
|
||||
"md5": "3fb071066d5d5b282f56a0169340346d.dir",
|
||||
"contents": {
|
||||
"dir": {
|
||||
"isout": False,
|
||||
"isdir": True,
|
||||
"isexec": False,
|
||||
"size": 96,
|
||||
"path": "data/dir",
|
||||
"md5": None,
|
||||
"contents": {
|
||||
"subdir": {
|
||||
"isout": True,
|
||||
"isdir": True,
|
||||
"isexec": False,
|
||||
"size": 96,
|
||||
"md5": None,
|
||||
"path": "data/dir/subdir",
|
||||
"contents": {
|
||||
"foobar": {
|
||||
"isout": True,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"size": 4,
|
||||
"md5": "d3b07384d113edec49eaa6238ad5ff00",
|
||||
}
|
||||
},
|
||||
},
|
||||
"foo": {
|
||||
"isout": False,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"size": 4,
|
||||
"path": "data/dir/foo",
|
||||
"md5": None,
|
||||
},
|
||||
},
|
||||
},
|
||||
"bar": {
|
||||
"isout": True,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"size": 4,
|
||||
"path": "data/bar",
|
||||
"md5": "c157a79031e1c40f85931829bc5fc552",
|
||||
},
|
||||
"large-file": {
|
||||
"isout": False,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"size": 1073741824,
|
||||
"path": "data/large-file",
|
||||
"md5": None,
|
||||
},
|
||||
"dir2": {
|
||||
"isout": True,
|
||||
"isdir": True,
|
||||
"isexec": False,
|
||||
"size": 96,
|
||||
"md5": None,
|
||||
"path": "data/dir2",
|
||||
"contents": {
|
||||
"foo": {
|
||||
"isout": True,
|
||||
"isdir": False,
|
||||
"isexec": False,
|
||||
"size": 4,
|
||||
"path": "data/dir2/foo",
|
||||
"md5": "d3b07384d113edec49eaa6238ad5ff00",
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
show_tree(entries, with_color=False)
|
||||
out, _ = capsys.readouterr()
|
||||
expected = """\
|
||||
data
|
||||
├── dir
|
||||
│ ├── subdir
|
||||
│ │ └── foobar
|
||||
│ └── foo
|
||||
├── bar
|
||||
├── large-file
|
||||
└── dir2
|
||||
└── foo
|
||||
"""
|
||||
assert out == expected
|
||||
|
||||
show_tree(entries, with_color=False, with_size=True)
|
||||
out, _ = capsys.readouterr()
|
||||
expected = """\
|
||||
192 data
|
||||
96 ├── dir
|
||||
96 │ ├── subdir
|
||||
4 │ │ └── foobar
|
||||
4 │ └── foo
|
||||
4 ├── bar
|
||||
1.00G ├── large-file
|
||||
96 └── dir2
|
||||
4 └── foo
|
||||
"""
|
||||
assert out == expected
|
||||
|
||||
show_tree(entries, with_color=False, with_hash=True)
|
||||
out, _ = capsys.readouterr()
|
||||
expected = """\
|
||||
3fb071066d5d5b282f56a0169340346d.dir data
|
||||
- ├── dir
|
||||
- │ ├── subdir
|
||||
d3b07384d113edec49eaa6238ad5ff00 │ │ └── foobar
|
||||
- │ └── foo
|
||||
c157a79031e1c40f85931829bc5fc552 ├── bar
|
||||
- ├── large-file
|
||||
- └── dir2
|
||||
d3b07384d113edec49eaa6238ad5ff00 └── foo
|
||||
"""
|
||||
assert out == expected
|
||||
|
||||
show_tree(entries, with_color=False, with_hash=True, with_size=True)
|
||||
out, _ = capsys.readouterr()
|
||||
expected = """\
|
||||
192 3fb071066d5d5b282f56a0169340346d.dir data
|
||||
96 - ├── dir
|
||||
96 - │ ├── subdir
|
||||
4 d3b07384d113edec49eaa6238ad5ff00 │ │ └── foobar
|
||||
4 - │ └── foo
|
||||
4 c157a79031e1c40f85931829bc5fc552 ├── bar
|
||||
1.00G - ├── large-file
|
||||
96 - └── dir2
|
||||
4 d3b07384d113edec49eaa6238ad5ff00 └── foo
|
||||
"""
|
||||
assert out == expected
|
||||
|
||||
|
||||
def test_list_alias():
|
||||
cli_args = parse_args(["ls", "local_dir"])
|
||||
assert cli_args.func == CmdList
|
||||
|
||||
@@ -133,7 +133,7 @@ def test_show_dot(repo):
|
||||
"\"stage: 'b.dvc'\" -> \"stage: '2'\";",
|
||||
"\"stage: 'b.dvc'\" -> \"stage: '3'\";",
|
||||
"\"stage: 'b.dvc'\";",
|
||||
"strict digraph {",
|
||||
"strict digraph {",
|
||||
"}",
|
||||
]
|
||||
actual = sorted(line.rstrip() for line in _show_dot(repo.index.graph).splitlines())
|
||||
@@ -153,7 +153,7 @@ def test_show_dot_properly_escapes():
|
||||
)
|
||||
|
||||
expected = {
|
||||
"strict digraph {",
|
||||
"strict digraph {",
|
||||
'"data\\raw\\2.dvc";',
|
||||
'"prepare";',
|
||||
'"4";',
|
||||
|
||||
Reference in New Issue
Block a user