repo: try to support opening uninitialized/broken repositories (#10736)

This `_get_remote_config()` should use `uninitialized=True` so that it supports
more broader kinds of broken/partially-initialized repositories, including some that have
`.dvc` directory missing, or `.git` directory missing.

This partially reverts #10719. #10608 is also fixed, and no longer requires `core.no_scm` to be passed.
This was already supported by `dvc.api.get_url` as it uses `uninitialized=True`, but this was not respected
in `_get_remote_config()` where it would fail before.

That said, this whole `open_repo`/`_get_remote_config` is terribly broken. For one, it is opening
a local repository, and forcing it's remote config to a repository opened with `Repo(rev=...)`,
where the config may be different.

.github/.test_durations

@@ -835,9 +835,9 @@
     "tests/func/test_dataset.py::test_dvc": 0.46001330000012786,
     "tests/func/test_dataset.py::test_dvc_dataset_pipeline": 0.7948393000000351,
     "tests/func/test_dataset.py::test_dvc_dump": 0.17996560000005957,
-    "tests/func/test_dataset.py::test_dvcx": 0.22482930000001033,
-    "tests/func/test_dataset.py::test_dvcx_dataset_pipeline": 0.43405649999999696,
-    "tests/func/test_dataset.py::test_dvcx_dump": 0.1858166000000665,
+    "tests/func/test_dataset.py::test_datachain": 0.22482930000001033,
+    "tests/func/test_dataset.py::test_datachain_dataset_pipeline": 0.43405649999999696,
+    "tests/func/test_dataset.py::test_datachain_dump": 0.1858166000000665,
     "tests/func/test_dataset.py::test_invalidation": 0.16291839999996682,
     "tests/func/test_dataset.py::test_parametrized": 0.19277699999997822,
     "tests/func/test_dataset.py::test_pipeline_when_not_in_sync": 0.2063928999999689,
@@ -1627,7 +1627,7 @@
     "tests/unit/command/test_data_sync.py::test_pull": 0.17549940000003517,
     "tests/unit/command/test_data_sync.py::test_push": 0.16576090000000931,
     "tests/unit/command/test_dataset.py::test_add[spec0-lock0-Adding ds (url:/path @ main)\\n]": 0.22054290000005494,
-    "tests/unit/command/test_dataset.py::test_add[spec1-lock1-Adding mydataset (dvcx://dataset @ v1)\\n]": 0.22979979999990974,
+    "tests/unit/command/test_dataset.py::test_add[spec1-lock1-Adding mydataset (dc://dataset @ v1)\\n]": 0.22979979999990974,
     "tests/unit/command/test_dataset.py::test_add[spec2-lock2-Adding mydataset (s3://bucket/path)\\n]": 0.20264819999988504,
     "tests/unit/command/test_dataset.py::test_add_already_exists": 0.2456899000001158,
     "tests/unit/command/test_dataset.py::test_update[spec0-old_lock0-new_lock0-expected_outputs0-missing]": 0.2354315999998562,

.github/workflows/benchmarks.yaml

@@ -16,14 +16,18 @@ jobs:
     steps:
       - uses: actions/setup-python@v5
         with:
-            python-version: "3.12"
+            python-version: "3.13"
 
-      - uses: hynek/setup-cached-uv@v2
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.base.sha }}
           fetch-depth: 0
 
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: pyproject.toml
+
       - run: uv pip install '.[tests]' --system
       - name: run benchmarks on base branch
         run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'

.github/workflows/build.yaml

@@ -25,25 +25,27 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: 3.12
+        python-version: 3.13
 
-    - uses: hynek/setup-cached-uv@v2
-    - run: uv pip install --upgrade setuptools_scm build twine --system
+    - uses: astral-sh/setup-uv@v6
 
-    - name: Force version for Test PyPI upload
-      if: ${{ !startsWith(github.ref, 'refs/tags') }}
-      run: echo version=$(python -m setuptools_scm | awk -F+ '{print $1}' | tail -1) >> $GITHUB_ENV
+    - name: Omit local version for Test PyPI upload
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+      run: echo SETUPTOOLS_SCM_OVERRIDES_FOR_DVC='{local_scheme="no-local-version"}' >> $GITHUB_ENV
 
-    - run: echo 'PKG = "pip"'>dvc/_build.py
-    - run: python -m build --installer=uv
-      env:
-        SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DVC: ${{ env.version }}
-    - run: twine check --strict dist/*
+    - name: Build Python Package
+      run: |
+        echo 'PKG = "pip"'>dvc/_build.py
+        uv build
+
+    - name: Check dist
+      run: uv tool run twine check --strict dist/*
 
     - uses: actions/upload-artifact@v4
       with:
         name: Packages
         path: dist/
+        if-no-files-found: error
 
   test-pypi-publish:
     name: Publish dev package to test.pypi.org

.github/workflows/plugin_tests.yaml

@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        pyv: ["3.12"]
+        pyv: ["3.13"]
         plugin: ["dvc-s3"]
 
     steps:
@@ -38,11 +38,11 @@ jobs:
       with:
         python-version: ${{ matrix.pyv }}
 
-    - uses: hynek/setup-cached-uv@v2
+    - uses: astral-sh/setup-uv@v6
       with:
-        cache-dependency-path: |
-          dvc/pyproject.toml
-          ${{ matrix.plugin }}/pyproject.toml
+        enable-cache: true
+        cache-dependency-glob: "**/pyproject.toml"
+        cache-suffix: ${{ matrix.pyv }}
 
     - name: Install plugin + DVC@PR
       run: |

.github/workflows/tests.yaml

@@ -19,6 +19,8 @@ jobs:
   lint:
     timeout-minutes: 10
     runs-on: ${{ matrix.os }}
+    env:
+       DVC_NO_ANALYTICS: true
     strategy:
       fail-fast: false
       matrix:
@@ -30,10 +32,11 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
-    - uses: hynek/setup-cached-uv@v2
+        python-version: '3.13'
+    - uses: astral-sh/setup-uv@v6
       with:
-        cache-dependency-path: pyproject.toml
+        enable-cache: true
+        cache-dependency-glob: pyproject.toml
     - run: uv pip install -e ".[dev]" --system
       env:
         UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
@@ -52,7 +55,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        pyv: ["3.9", "3.10", "3.11", "3.12"]
+        pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         include:
         - os: windows-latest
           pyv: "3.9"
@@ -66,6 +69,30 @@ jobs:
         - os: windows-latest
           pyv: "3.9"
           pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
         - os: windows-latest
           pyv: "3.12"
           pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
@@ -78,6 +105,18 @@ jobs:
         - os: windows-latest
           pyv: "3.12"
           pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
     steps:
     - uses: actions/checkout@v4
       with:
@@ -86,10 +125,11 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.pyv }}
-    - uses: hynek/setup-cached-uv@v2
+    - uses: astral-sh/setup-uv@v6
       with:
+        enable-cache: true
+        cache-dependency-glob: pyproject.toml
         cache-suffix: ${{ matrix.pyv }}
-        cache-dependency-path: pyproject.toml
     - run: uv pip install -e ".[dev]" pytest-split --system
       env:
         UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
@@ -102,7 +142,7 @@ jobs:
         pytest ${{ matrix.pytestargs }} -n=logical --dist=worksteal --timeout=300 --durations=0
         --cov --cov-report=xml --cov-report=term  --durations-path=./.github/.test_durations
     - name: upload coverage report
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v5
       with:
         file: ./coverage.xml
         fail_ci_if_error: false
@@ -120,7 +160,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Slack Notification
-      uses: rtCamp/[email protected].0
+      uses: rtCamp/[email protected].3
       env:
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_COLOR: ${{ job.status }}

.gitignore

@@ -43,7 +43,7 @@ scripts/ci/gcp-creds.json
 .gcp-creds.json
 
 *~
-/dvc/_dvc_version.py
+/dvc/_version.py
 
 .mypy_cache/
 .pytest_cache/

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ ci:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -22,13 +22,13 @@ repos:
       - id: sort-simple-yaml
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.6.2'
+    rev: 'v0.11.8'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.1
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]
@@ -50,7 +50,7 @@ repos:
         language: system
         name: DVC pre-commit
         stages:
-          - commit
+          - pre-commit
         verbose: true
         require_serial: true
       - args:
@@ -61,7 +61,7 @@ repos:
         language: system
         name: DVC pre-push
         stages:
-          - push
+          - pre-push
         require_serial: true
       - always_run: true
         args:

.pre-commit-hooks.yaml

@@ -8,7 +8,7 @@
   name: DVC pre-commit
   require_serial: true
   stages:
-  - commit
+  - pre-commit
   verbose: true
 - args:
   - git-hook
@@ -20,7 +20,7 @@
   name: DVC pre-push
   require_serial: true
   stages:
-  - push
+  - pre-push
 - always_run: true
   args:
   - git-hook
@@ -29,7 +29,7 @@
   id: dvc-post-checkout
   language: python
   language_version: python3
-  minimum_pre_commit_version: 2.2.0
+  minimum_pre_commit_version: 3.2.0
   name: DVC post-checkout
   require_serial: true
   stages:

README.rst

@@ -1,6 +1,4 @@
-🚀 Check out our new product `DataChain <https://github.com/iterative/datachain>`_ (and give it a ⭐!) if you need to version and process a large number of files. Contact us at [email protected] to discuss commercial solutions and support for AI reproducibility and data management scenarios.
-
---------------------------
+|Banner|
 
 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_

dvc/api/__init__.py

@@ -1,7 +1,7 @@
 from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem
 
 from .artifacts import artifacts_show
-from .data import get_url, open, read
+from .data import get_url, open, read  # noqa: A004
 from .dataset import get as get_dataset
 from .experiments import exp_save, exp_show
 from .scm import all_branches, all_commits, all_tags

dvc/api/dataset.py

@@ -1,8 +1,8 @@
 from typing import Literal, TypedDict, Union
 
 
-class DVCXDataset(TypedDict):
-    type: Literal["dvcx"]
+class DatachainDataset(TypedDict):
+    type: Literal["dc"]
     name: str
     version: int
 
@@ -20,7 +20,7 @@ class URLDataset(TypedDict):
     path: str
 
 
-def get(name: str) -> Union[DVCXDataset, DVCDataset, URLDataset]:
+def get(name: str) -> Union[DatachainDataset, DVCDataset, URLDataset]:
     from difflib import get_close_matches
 
     from dvc.fs import get_cloud_fs
@@ -46,9 +46,9 @@ def get(name: str) -> Union[DVCXDataset, DVCDataset, URLDataset]:
             path=dataset.lock.path,
             sha=dataset.lock.rev_lock,
         )
-    if dataset.type == "dvcx":
-        return DVCXDataset(
-            type="dvcx", name=dataset.name_version[0], version=dataset.lock.version
+    if dataset.type == "dc":
+        return DatachainDataset(
+            type="dc", name=dataset.name_version[0], version=dataset.lock.version
         )
     if dataset.type == "url":
         fs_cls, _, path = get_cloud_fs(repo.config, url=dataset.lock.url)

dvc/api/show.py

@@ -27,8 +27,7 @@ def _postprocess(results):
             }
             processed[rev] = processed[rev] | to_merge
 
-    if "workspace" in processed:
-        del processed["workspace"]
+    processed.pop("workspace", None)
 
     return processed
 

dvc/cli/utils.py

@@ -40,7 +40,7 @@ def hide_subparsers_from_help(subparsers):
     # from the 'positional arguments' choices list
     # see: https://bugs.python.org/issue22848
     # Need to set `add_help=False`, but avoid setting `help`
-    # (not even to `argparse.SUPPPRESS`).
+    # (not even to `argparse.SUPPRESS`).
     # NOTE: The argument is the parent subparser, not the subcommand parser.
     cmds = [cmd for cmd, parser in subparsers.choices.items() if parser.add_help]
     subparsers.metavar = "{{{}}}".format(",".join(cmds))

dvc/commands/dataset.py

@@ -34,7 +34,7 @@ class CmdDatasetAdd(CmdBase):
 
         url = dataset.spec.url
         ver: str = ""
-        if dataset.type == "dvcx":
+        if dataset.type == "dc":
             ver = f"v{dataset.lock.version}"
         if dataset.type == "dvc":
             if dataset.lock.path:
@@ -55,7 +55,7 @@ class CmdDatasetAdd(CmdBase):
             raise DvcException("--path can't be used without --dvc")
 
         d = vars(self.args)
-        for key in ["dvc", "dvcx", "url"]:
+        for key in ["dvc", "dc", "url"]:
             if url := d.pop(key, None):
                 d.update({"type": key, "url": url})
                 break
@@ -88,8 +88,8 @@ class CmdDatasetUpdate(CmdBase):
         assert new.lock
 
         v: Optional[tuple[str, str]] = None
-        if dataset.type == "dvcx":
-            assert new.type == "dvcx"
+        if dataset.type == "dc":
+            assert new.type == "dc"
             if new.lock.version < dataset.lock.version:
                 action = "Downgrading"
 
@@ -168,7 +168,7 @@ def add_parser(subparsers, parent_parser):
 
     url_exclusive_group = ds_add_parser.add_mutually_exclusive_group(required=True)
     url_exclusive_group.add_argument(
-        "--dvcx", metavar="name", help="Name of the dvcx dataset to track"
+        "--dc", metavar="name", help="Name of the DataChain dataset to track"
     )
     url_exclusive_group.add_argument(
         "--dvc",
@@ -219,7 +219,7 @@ remote://remote_name/path/to/file/or/dir (see `dvc remote`)
         "--rev",
         "--version",
         nargs="?",
-        help="DVCX dataset version or Git revision (e.g. SHA, branch, tag)",
+        help="DataChain dataset version or Git revision (e.g. SHA, branch, tag)",
         metavar="<version>",
     )
     ds_update_parser.set_defaults(func=CmdDatasetUpdate)

dvc/commands/experiments/__init__.py

@@ -58,6 +58,15 @@ def add_parser(subparsers, parent_parser):
     hide_subparsers_from_help(experiments_subparsers)
 
 
+def add_keep_selection_flag(experiments_subcmd_parser):
+    experiments_subcmd_parser.add_argument(
+        "--keep",
+        action="store_true",
+        default=False,
+        help="Keep the selected experiments instead of removing them.",
+    )
+
+
 def add_rev_selection_flags(
     experiments_subcmd_parser, command: str, default: bool = True
 ):

dvc/commands/experiments/remove.py

@@ -34,6 +34,7 @@ class CmdExperimentsRemove(CmdBase):
             num=self.args.num,
             queue=self.args.queue,
             git_remote=self.args.git_remote,
+            keep=self.args.keep,
         )
         if removed:
             ui.write(f"Removed experiments: {humanize.join(map(repr, removed))}")
@@ -44,7 +45,7 @@ class CmdExperimentsRemove(CmdBase):
 
 
 def add_parser(experiments_subparsers, parent_parser):
-    from . import add_rev_selection_flags
+    from . import add_keep_selection_flag, add_rev_selection_flags
 
     EXPERIMENTS_REMOVE_HELP = "Remove experiments."
     experiments_remove_parser = experiments_subparsers.add_parser(
@@ -57,6 +58,7 @@ def add_parser(experiments_subparsers, parent_parser):
     )
     remove_group = experiments_remove_parser.add_mutually_exclusive_group()
     add_rev_selection_flags(experiments_remove_parser, "Remove", False)
+    add_keep_selection_flag(experiments_remove_parser)
     remove_group.add_argument(
         "--queue", action="store_true", help="Remove all queued experiments."
     )

dvc/commands/gc.py

@@ -54,7 +54,7 @@ class CmdGC(CmdBase):
             msg += " and all experiments"
 
         if self.args.not_in_remote:
-            msg += " that are present in the DVC remote"
+            msg += " that are not present in the DVC remote"
 
         if self.args.repos:
             msg += " of the current and the following repos:"

dvc/commands/ls/__init__.py

@@ -1,3 +1,5 @@
+from typing import Callable
+
 from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBaseNoRepo
 from dvc.cli.utils import DictAction, append_doc_link
@@ -9,54 +11,178 @@ from dvc.ui import ui
 logger = logger.getChild(__name__)
 
 
-def _format_entry(entry, fmt):
-    from dvc.utils.humanize import naturalsize
+def _get_formatter(with_color: bool = False) -> Callable[[dict], str]:
+    def fmt(entry: dict) -> str:
+        return entry["path"]
 
-    size = entry.get("size")
-    if size is None:
-        size = ""
-    else:
-        size = naturalsize(size)
-    return size, fmt(entry)
-
-
-def show_entries(entries, with_color=False, with_size=False):
     if with_color:
         ls_colors = LsColors()
-        fmt = ls_colors.format
-    else:
+        return ls_colors.format
 
-        def fmt(entry):
-            return entry["path"]
+    return fmt
 
+
+def _format_entry(entry, name, with_size=True, with_hash=False):
+    from dvc.utils.humanize import naturalsize
+
+    ret = []
     if with_size:
-        ui.table([_format_entry(entry, fmt) for entry in entries])
+        size = entry.get("size")
+        if size is None or (size <= 0 and entry.get("isdir")):
+            size = ""
+        else:
+            size = naturalsize(size)
+        ret.append(size)
+    if with_hash:
+        md5 = entry.get("md5", "")
+        ret.append(md5)
+    ret.append(name)
+    return ret
+
+
+def show_entries(entries, with_color=False, with_size=False, with_hash=False):
+    fmt = _get_formatter(with_color)
+    if with_size or with_hash:
+        colalign = ("right",) if with_size else None
+        ui.table(
+            [
+                _format_entry(
+                    entry,
+                    fmt(entry),
+                    with_size=with_size,
+                    with_hash=with_hash,
+                )
+                for entry in entries
+            ],
+            colalign=colalign,
+        )
         return
 
     # NOTE: this is faster than ui.table for very large number of entries
     ui.write("\n".join(fmt(entry) for entry in entries))
 
 
+class TreePart:
+    Edge = "├── "
+    Line = "│   "
+    Corner = "└── "
+    Blank = "    "
+
+
+def _build_tree_structure(
+    entries, with_color=False, with_size=False, with_hash=False, _depth=0, _prefix=""
+):
+    rows = []
+    fmt = _get_formatter(with_color)
+
+    num_entries = len(entries)
+    for i, (name, entry) in enumerate(entries.items()):
+        entry["path"] = name
+        is_last = i >= num_entries - 1
+        tree_part = ""
+        if _depth > 0:
+            tree_part = TreePart.Corner if is_last else TreePart.Edge
+
+        row = _format_entry(
+            entry,
+            _prefix + tree_part + fmt(entry),
+            with_size=with_size,
+            with_hash=with_hash,
+        )
+        rows.append(row)
+
+        if contents := entry.get("contents"):
+            new_prefix = _prefix
+            if _depth > 0:
+                new_prefix += TreePart.Blank if is_last else TreePart.Line
+            new_rows = _build_tree_structure(
+                contents,
+                with_color=with_color,
+                with_size=with_size,
+                with_hash=with_hash,
+                _depth=_depth + 1,
+                _prefix=new_prefix,
+            )
+            rows.extend(new_rows)
+
+    return rows
+
+
+def show_tree(entries, with_color=False, with_size=False, with_hash=False):
+    import tabulate
+
+    rows = _build_tree_structure(
+        entries,
+        with_color=with_color,
+        with_size=with_size,
+        with_hash=with_hash,
+    )
+
+    colalign = ("right",) if with_size else None
+
+    _orig = tabulate.PRESERVE_WHITESPACE
+    tabulate.PRESERVE_WHITESPACE = True
+    try:
+        ui.table(rows, colalign=colalign)
+    finally:
+        tabulate.PRESERVE_WHITESPACE = _orig
+
+
 class CmdList(CmdBaseNoRepo):
-    def run(self):
+    def _show_tree(self):
+        from dvc.repo.ls import ls_tree
+
+        entries = ls_tree(
+            self.args.url,
+            self.args.path,
+            rev=self.args.rev,
+            dvc_only=self.args.dvc_only,
+            config=self.args.config,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+            maxdepth=self.args.level,
+        )
+        show_tree(
+            entries,
+            with_color=True,
+            with_size=self.args.size,
+            with_hash=self.args.show_hash,
+        )
+        return 0
+
+    def _show_list(self):
         from dvc.repo import Repo
 
-        try:
-            entries = Repo.ls(
-                self.args.url,
-                self.args.path,
-                rev=self.args.rev,
-                recursive=self.args.recursive,
-                dvc_only=self.args.dvc_only,
-                config=self.args.config,
-                remote=self.args.remote,
-                remote_config=self.args.remote_config,
+        entries = Repo.ls(
+            self.args.url,
+            self.args.path,
+            rev=self.args.rev,
+            recursive=self.args.recursive,
+            dvc_only=self.args.dvc_only,
+            config=self.args.config,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+            maxdepth=self.args.level,
+        )
+        if self.args.json:
+            ui.write_json(entries)
+        elif entries:
+            show_entries(
+                entries,
+                with_color=True,
+                with_size=self.args.size,
+                with_hash=self.args.show_hash,
             )
-            if self.args.json:
-                ui.write_json(entries)
-            elif entries:
-                show_entries(entries, with_color=True, with_size=self.args.size)
-            return 0
+        return 0
+
+    def run(self):
+        if self.args.tree and self.args.json:
+            raise DvcException("Cannot use --tree and --json options together.")
+
+        try:
+            if self.args.tree:
+                return self._show_tree()
+            return self._show_list()
         except FileNotFoundError:
             logger.exception("")
             return 1
@@ -85,6 +211,19 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Recursively list files.",
     )
+    list_parser.add_argument(
+        "-T",
+        "--tree",
+        action="store_true",
+        help="Recurse into directories as a tree.",
+    )
+    list_parser.add_argument(
+        "-L",
+        "--level",
+        metavar="depth",
+        type=int,
+        help="Limit the depth of recursion.",
+    )
     list_parser.add_argument(
         "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
@@ -123,6 +262,12 @@ def add_parser(subparsers, parent_parser):
         ),
     )
     list_parser.add_argument("--size", action="store_true", help="Show sizes.")
+    list_parser.add_argument(
+        "--show-hash",
+        help="Display hash value for each item.",
+        action="store_true",
+        default=False,
+    )
     list_parser.add_argument(
         "path",
         nargs="?",

dvc/commands/ls/ls_colors.py

@@ -32,7 +32,9 @@ class LsColors:
         if entry.get("isexec", False):
             return self._format(text, code="ex")
 
-        _, ext = os.path.splitext(text)
+        stem, ext = os.path.splitext(text)
+        if not ext and stem.startswith("."):
+            ext = stem
         return self._format(text, ext=ext)
 
     def _format(self, text, code=None, ext=None):

dvc/commands/ls_url.py

@@ -3,26 +3,45 @@ from dvc.cli.command import CmdBaseNoRepo
 from dvc.cli.utils import DictAction, append_doc_link
 from dvc.log import logger
 
-from .ls import show_entries
+from .ls import show_entries, show_tree
 
 logger = logger.getChild(__name__)
 
 
 class CmdListUrl(CmdBaseNoRepo):
-    def run(self):
-        from dvc.config import Config
+    def _show_tree(self, config):
+        from dvc.fs import parse_external_url
+        from dvc.repo.ls import _ls_tree
+
+        fs, fs_path = parse_external_url(
+            self.args.url, fs_config=self.args.fs_config, config=config
+        )
+        entries = _ls_tree(fs, fs_path, maxdepth=self.args.level)
+        show_tree(entries, with_color=True, with_size=self.args.size)
+        return 0
+
+    def _show_list(self, config):
         from dvc.repo import Repo
 
         entries = Repo.ls_url(
             self.args.url,
             recursive=self.args.recursive,
+            maxdepth=self.args.level,
             fs_config=self.args.fs_config,
-            config=Config.from_cwd(),
+            config=config,
         )
         if entries:
             show_entries(entries, with_color=True, with_size=self.args.size)
         return 0
 
+    def run(self):
+        from dvc.config import Config
+
+        config = Config.from_cwd()
+        if self.args.tree:
+            return self._show_tree(config=config)
+        return self._show_list(config=config)
+
 
 def add_parser(subparsers, parent_parser):
     LS_HELP = "List directory contents from URL."
@@ -40,6 +59,19 @@ def add_parser(subparsers, parent_parser):
     lsurl_parser.add_argument(
         "-R", "--recursive", action="store_true", help="Recursively list files."
     )
+    lsurl_parser.add_argument(
+        "-T",
+        "--tree",
+        action="store_true",
+        help="Recurse into directories as a tree.",
+    )
+    lsurl_parser.add_argument(
+        "-L",
+        "--level",
+        metavar="depth",
+        type=int,
+        help="Limit the depth of recursion.",
+    )
     lsurl_parser.add_argument("--size", action="store_true", help="Show sizes.")
     lsurl_parser.add_argument(
         "--fs-config",

dvc/commands/remote.py

@@ -110,8 +110,16 @@ class CmdRemoteDefault(CmdRemote):
 class CmdRemoteList(CmdRemote):
     def run(self):
         conf = self.config.read(self.args.level)
+        default_remote = conf["core"].get("remote")
+
         for name, remote_conf in conf["remote"].items():
-            ui.write(name, remote_conf["url"], sep="\t")
+            if name == default_remote:
+                text = f"{name}\t{remote_conf['url']}\t(default)"
+                color = "green"
+            else:
+                text = f"{name}\t{remote_conf['url']}"
+                color = ""
+            ui.write(ui.rich_text(text, style=color), styled=True)
         return 0
 
 

dvc/config.py

@@ -1,6 +1,8 @@
 """DVC config objects."""
 
+import ntpath
 import os
+import posixpath
 import re
 from contextlib import contextmanager
 from functools import partial
@@ -268,12 +270,17 @@ class Config(dict):
         if re.match(r"\w+://", path):
             return path
 
+        if os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path:
+            return path
+
         if os.path.isabs(path):
             return path
 
         # on windows convert slashes to backslashes
         # to have path compatible with abs_conf_dir
         if os.path.sep == "\\" and "/" in path:
+            if path.startswith("/"):
+                path = path.replace("/", "\\\\", 1)
             path = path.replace("/", "\\")
 
         expanded = os.path.expanduser(path)
@@ -305,6 +312,9 @@ class Config(dict):
         if os.path.expanduser(path) != path:
             return localfs.as_posix(path)
 
+        if os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path:
+            return path
+
         if isinstance(path, RelPath) or not os.path.isabs(path):
             path = relpath(path, conf_dir)
             return localfs.as_posix(path)

dvc/daemon.py

@@ -164,7 +164,7 @@ def _spawn(
         return _detached_subprocess(executable + args, env=env, **kwargs)
 
 
-def daemonize(args: list[str], executable: Union[None, str, list[str]] = None) -> None:
+def daemonize(args: list[str], executable: Union[str, list[str], None] = None) -> None:
     if os.name not in ("posix", "nt"):
         return
 

dvc/dagascii.py

@@ -104,21 +104,21 @@ class AsciiCanvas:
                 if dx == 0:
                     y = y0
                 else:
-                    y = y0 + int(round((x - x0) * dy / float(dx)))
+                    y = y0 + round((x - x0) * dy / float(dx))
                 self.point(x, y, char)
         elif y0 < y1:
             for y in range(y0, y1 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x0 + int(round((y - y0) * dx / float(dy)))
+                    x = x0 + round((y - y0) * dx / float(dy))
                 self.point(x, y, char)
         else:
             for y in range(y1, y0 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x1 + int(round((y - y1) * dx / float(dy)))
+                    x = x1 + round((y - y1) * dx / float(dy))
                 self.point(x, y, char)
 
     def text(self, x, y, text):
@@ -258,8 +258,8 @@ def draw(vertices, edges):
     maxx = max(Xs)
     maxy = max(Ys)
 
-    canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
-    canvas_lines = int(round(maxy - miny))
+    canvas_cols = math.ceil(math.ceil(maxx) - math.floor(minx)) + 1
+    canvas_lines = round(maxy - miny)
 
     canvas = AsciiCanvas(canvas_cols, canvas_lines)
 
@@ -270,10 +270,10 @@ def draw(vertices, edges):
             start = edge.view._pts[index - 1]
             end = edge.view._pts[index]
 
-            start_x = int(round(start[0] - minx))
-            start_y = int(round(start[1] - miny))
-            end_x = int(round(end[0] - minx))
-            end_y = int(round(end[1] - miny))
+            start_x = round(start[0] - minx)
+            start_y = round(start[1] - miny)
+            end_x = round(end[0] - minx)
+            end_y = round(end[1] - miny)
 
             assert start_x >= 0
             assert start_y >= 0
@@ -288,12 +288,12 @@ def draw(vertices, edges):
         y = vertex.view.xy[1]
 
         canvas.box(
-            int(round(x - minx)),
-            int(round(y - miny)),
+            round(x - minx),
+            round(y - miny),
             vertex.view.w,
             vertex.view.h,
         )
 
-        canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data)
+        canvas.text(round(x - minx) + 1, round(y - miny) + 1, vertex.data)
 
     return canvas.draw()

dvc/dependency/repo.py

@@ -101,9 +101,9 @@ class RepoDependency(Dependency):
             return
 
         hashes: list[tuple[str, HashInfo, dict[str, Any]]] = []
-        for src_path, dest_path, *rest in files:
+        for src_path, dest_path, maybe_info in files:
             try:
-                info = rest[0] if rest else self.fs.info(src_path)
+                info = maybe_info or self.fs.info(src_path)
                 hash_info = info["dvc_info"]["entry"].hash_info
                 dest_info = to.fs.info(dest_path)
             except (KeyError, AttributeError):
@@ -146,6 +146,10 @@ class RepoDependency(Dependency):
         else:
             config = Config.load_file(conf)
 
+        # Setup config to the new DVCFileSystem to use the remote repo, but rely on the
+        # local cache instead of the remote's cache. This avoids re-streaming of data,
+        # but messes up the call to `_get_remote_config()` downstream, which will need
+        # to ignore cache parameters.
         config["cache"] = self.repo.config["cache"]
         config["cache"]["dir"] = self.repo.cache.local_cache_dir
 

dvc/dirs.py

@@ -21,6 +21,19 @@ def global_config_dir():
 
 
 def site_cache_dir():
+    from platformdirs import PlatformDirs
+    from platformdirs.unix import Unix
+
+    if issubclass(Unix, PlatformDirs):
+        # Return the cache directory shared by users, e.g. `/var/tmp/$appname`
+        # NOTE: platformdirs>=5 changed `site_cache_dir` to return /var/cache/$appname.
+        # as the following path is considered insecure.
+        # For details, see: https://github.com/tox-dev/platformdirs/pull/239
+
+        # FIXME: keeping the old behavior temporarily to avoid dependency conflict.
+        #        In the future, consider migrating to a more secure directory.
+        return f"/var/tmp/{APPNAME}"  # noqa: S108
+
     return os.getenv(env.DVC_SITE_CACHE_DIR) or platformdirs.site_cache_dir(
         APPNAME, APPAUTHOR, opinion=True
     )

dvc/fs/__init__.py

@@ -1,5 +1,6 @@
 import glob
-from typing import Optional, Union
+from itertools import repeat
+from typing import Optional
 from urllib.parse import urlparse
 
 from dvc.config import ConfigError as RepoConfigError
@@ -19,7 +20,7 @@ from dvc_objects.fs import (  # noqa: F401
     system,
     utils,
 )
-from dvc_objects.fs.base import AnyFSPath, FileSystem  # noqa: F401, TCH001
+from dvc_objects.fs.base import AnyFSPath, FileSystem  # noqa: F401, TC001
 from dvc_objects.fs.errors import (  # noqa: F401
     AuthError,
     ConfigError,
@@ -47,7 +48,7 @@ known_implementations.update(
 
 def download(
     fs: "FileSystem", fs_path: str, to: str, jobs: Optional[int] = None
-) -> list[Union[tuple[str, str], tuple[str, str, dict]]]:
+) -> list[tuple[str, str, Optional[dict]]]:
     from dvc.scm import lfs_prefetch
 
     from .callbacks import TqdmCallback
@@ -84,7 +85,7 @@ def download(
         cb.set_size(len(from_infos))
         jobs = jobs or fs.jobs
         generic.copy(fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs)
-        return list(zip(from_infos, to_infos))
+        return list(zip(from_infos, to_infos, repeat(None)))
 
 
 def parse_external_url(url, fs_config=None, config=None):

dvc/fs/dvc.py

@@ -4,8 +4,8 @@ import ntpath
 import os
 import posixpath
 import threading
-from collections import deque
-from contextlib import ExitStack, suppress
+from collections import defaultdict, deque
+from contextlib import ExitStack, nullcontext, suppress
 from glob import has_magic
 from typing import TYPE_CHECKING, Any, Callable, Optional, Union
 
@@ -19,6 +19,8 @@ from dvc_objects.fs.base import AnyFSPath, FileSystem
 from .data import DataFileSystem
 
 if TYPE_CHECKING:
+    from contextlib import AbstractContextManager
+
     from dvc.repo import Repo
     from dvc.types import DictStrAny, StrPath
 
@@ -498,7 +500,7 @@ class _DVCFileSystem(AbstractFileSystem):
             **kwargs,
         )
 
-    def _get(  # noqa: C901
+    def _get(  # noqa: C901, PLR0912, PLR0915
         self,
         rpath,
         lpath,
@@ -507,7 +509,7 @@ class _DVCFileSystem(AbstractFileSystem):
         maxdepth=None,
         batch_size=None,
         **kwargs,
-    ) -> list[Union[tuple[str, str], tuple[str, str, dict]]]:
+    ) -> list[tuple[str, str, Optional[dict]]]:
         if (
             isinstance(rpath, list)
             or isinstance(lpath, list)
@@ -531,10 +533,13 @@ class _DVCFileSystem(AbstractFileSystem):
         if self.isfile(rpath):
             with callback.branched(rpath, lpath) as child:
                 self.get_file(rpath, lpath, callback=child, **kwargs)
-                return [(rpath, lpath)]
+                return [(rpath, lpath, None)]
 
-        _files = []
+        result: list[tuple[str, str, Optional[dict]]] = []
         _dirs: list[str] = []
+        _files: dict[FileSystem, list[tuple[str, str, Optional[dict]]]]
+        _files = defaultdict(list)
+
         for root, dirs, files in self.walk(rpath, maxdepth=maxdepth, detail=True):
             if files:
                 callback.set_size((callback.size or 0) + len(files))
@@ -550,33 +555,48 @@ class _DVCFileSystem(AbstractFileSystem):
             _, dvc_fs, _ = self._get_subrepo_info(key)
 
             for name, info in files.items():
+                dvc_info = info.get("dvc_info")
+                fs_info = info.get("fs_info")
+                if dvc_fs and dvc_info and not fs_info:
+                    fs = dvc_fs
+                    fs_path = dvc_info["name"]
+                else:
+                    fs = self.repo.fs
+                    fs_path = fs_info["name"]
+
                 src_path = f"{root}{self.sep}{name}"
                 dest_path = f"{dest_root}{os.path.sep}{name}"
-                _files.append((dvc_fs, src_path, dest_path, info))
+                _files[fs].append((fs_path, dest_path, dvc_info))
+                result.append((src_path, dest_path, info))
 
         os.makedirs(lpath, exist_ok=True)
         for d in _dirs:
-            os.mkdir(d)
+            os.makedirs(d, exist_ok=True)
 
-        def _get_file(arg):
-            dvc_fs, src, dest, info = arg
-            dvc_info = info.get("dvc_info")
-            if dvc_info and dvc_fs:
-                dvc_path = dvc_info["name"]
-                dvc_fs.get_file(
-                    dvc_path, dest, callback=callback, info=dvc_info, **kwargs
-                )
-            else:
-                self.get_file(src, dest, callback=callback, **kwargs)
-            return src, dest, info
+        def get_file(arg: tuple[FileSystem, tuple[str, str, Optional[dict]]]):
+            fs, (src, dest, info) = arg
+            kw = kwargs
+            if isinstance(fs, DataFileSystem):
+                kw = kw | {"info": info}
+            with callback.branched(src, dest) as child:
+                fs.get_file(src, dest, callback=child, **kw)
 
-        with ThreadPoolExecutor(max_workers=batch_size) as executor:
-            return list(executor.imap_unordered(_get_file, _files))
+        if batch_size == 1:
+            ctx: AbstractContextManager = nullcontext()
+            map_fn: Callable = map
+        else:
+            ctx = ThreadPoolExecutor(max_workers=batch_size)
+            map_fn = ctx.imap_unordered
+
+        with ctx:
+            it = ((fs, f) for fs, files in _files.items() for f in files)
+            deque(callback.wrap(map_fn(get_file, it)), maxlen=0)
+        return result
 
     def get_file(self, rpath, lpath, **kwargs):
+        dvc_info = kwargs.pop("info", {}).pop("dvc_info", None)
         key = self._get_key_from_relative(rpath)
         fs_path = self._from_key(key)
-
         dirpath = os.path.dirname(lpath)
         if dirpath:
             # makedirs raises error if the string is empty
@@ -590,7 +610,7 @@ class _DVCFileSystem(AbstractFileSystem):
                 raise
 
         dvc_path = _get_dvc_path(dvc_fs, subkey)
-        return dvc_fs.get_file(dvc_path, lpath, **kwargs)
+        return dvc_fs.get_file(dvc_path, lpath, info=dvc_info, **kwargs)
 
     def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs):
         if maxdepth is not None:
@@ -666,7 +686,7 @@ class DVCFileSystem(FileSystem):
         recursive: bool = False,
         batch_size: Optional[int] = None,
         **kwargs,
-    ) -> list[Union[tuple[str, str], tuple[str, str, dict]]]:
+    ) -> list[tuple[str, str, Optional[dict]]]:
         # FileSystem.get is non-recursive by default if arguments are lists
         # otherwise, it's recursive.
         recursive = not (isinstance(from_info, list) and isinstance(to_info, list))
@@ -713,9 +733,10 @@ class DVCFileSystem(FileSystem):
         return self.fs.repo_url
 
     def from_os_path(self, path: str) -> str:
-        if os.path.isabs(path):
+        if os.path.isabs(path) or (
+            os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path
+        ):
             path = os.path.relpath(path, self.repo.root_dir)
-
         return as_posix(path)
 
     def close(self):

dvc/logger.py

@@ -36,7 +36,7 @@ def add_logging_level(level_name, level_num, method_name=None):
             self._log(level_num, message, args, **kwargs)
 
     def log_to_root(message, *args, **kwargs):
-        logging.log(level_num, message, *args, **kwargs)
+        logging.log(level_num, message, *args, **kwargs)  # noqa: LOG015
 
     # getLevelName resolves the numeric log level if already defined,
     # otherwise returns a string

dvc/output.py

@@ -431,7 +431,7 @@ class Output:
             self.meta.nfiles = len(self.files)
             self.meta.size = sum(filter(None, (f.get("size") for f in self.files)))
             self.meta.remote = first(f.get("remote") for f in self.files)
-        elif self.meta.nfiles or self.hash_info and self.hash_info.isdir:
+        elif self.meta.nfiles or (self.hash_info and self.hash_info.isdir):
             self.meta.isdir = True
             if not self.hash_info and self.hash_name not in ("md5", "md5-dos2unix"):
                 md5 = getattr(self.meta, "md5", None)
@@ -752,8 +752,9 @@ class Output:
             granular = (
                 self.is_dir_checksum and filter_info and filter_info != self.fs_path
             )
+            hardlink = relink and next(iter(self.cache.cache_types), None) == "hardlink"
             if granular:
-                obj = self._commit_granular_dir(filter_info, hardlink=False)
+                obj = self._commit_granular_dir(filter_info, hardlink=hardlink)
             else:
                 staging, _, obj = self._build(
                     self.cache,
@@ -771,7 +772,7 @@ class Output:
                         self.cache,
                         {obj.hash_info},
                         shallow=False,
-                        hardlink=False,
+                        hardlink=hardlink,
                         callback=cb,
                     )
             if relink:
@@ -1399,12 +1400,14 @@ class Output:
 
         assert staging
         assert obj.hash_info
+
+        hardlink = relink and next(iter(self.cache.cache_types), None) == "hardlink"
         with TqdmCallback(desc=f"Adding {self} to cache", unit="file") as cb:
             otransfer(
                 staging,
                 self.cache,
                 {obj.hash_info},
-                hardlink=False,
+                hardlink=hardlink,
                 shallow=False,
                 callback=cb,
             )

dvc/parsing/context.py

@@ -105,7 +105,7 @@ class Meta:
 
     @staticmethod
     def update_path(meta: "Meta", path: Union[str, int]):
-        dpaths = meta.dpaths[:] + [str(path)]
+        dpaths = [*meta.dpaths, str(path)]
         return replace(meta, dpaths=dpaths)
 
     def __str__(self):
@@ -176,8 +176,9 @@ class Container(Node, ABC):
             return value
         if isinstance(value, (list, dict)):
             assert meta
-            container = CtxDict if isinstance(value, dict) else CtxList
-            return container(value, meta=meta)
+            if isinstance(value, dict):
+                return CtxDict(value, meta=meta)
+            return CtxList(value, meta=meta)
         msg = f"Unsupported value of type '{type(value).__name__}' in '{meta}'"
         raise TypeError(msg)
 

dvc/pathspec_math.py

@@ -27,11 +27,9 @@ def _remove_slash(rule):
 
 def _match_all_level(rule):
     if rule[:-1].find("/") >= 0 and not rule.startswith("**/"):
-        if rule.startswith("/"):
-            rule = rule[1:]
+        rule = rule.removeprefix("/")
         return False, rule
-    if rule.startswith("**/"):
-        rule = rule[3:]
+    rule = rule.removeprefix("**/")
     return True, rule
 
 

dvc/repo/datasets.py

@@ -16,7 +16,7 @@ from dvc.types import StrPath
 from dvc_data.hashfile.meta import Meta
 
 if TYPE_CHECKING:
-    from dvcx.dataset import DatasetRecord, DatasetVersion  # type: ignore[import]
+    from datachain.dataset import DatasetRecord, DatasetVersion  # type: ignore[import]
     from typing_extensions import Self
 
     from dvc.repo import Repo
@@ -29,10 +29,10 @@ def _get_dataset_record(name: str) -> "DatasetRecord":
     from dvc.exceptions import DvcException
 
     try:
-        from dvcx.catalog import get_catalog  # type: ignore[import]
+        from datachain.catalog import get_catalog  # type: ignore[import]
 
     except ImportError as exc:
-        raise DvcException("dvcx is not installed") from exc
+        raise DvcException("datachain is not installed") from exc
 
     catalog = get_catalog()
     return catalog.get_remote_dataset(name)
@@ -86,7 +86,7 @@ class SerDe:
 class DatasetSpec(SerDe):
     name: str
     url: str
-    type: Literal["dvc", "dvcx", "url"]
+    type: Literal["dvc", "dc", "url"]
 
 
 @frozen(kw_only=True)
@@ -108,7 +108,7 @@ class DVCDatasetLock(DVCDatasetSpec):
 
 
 @frozen(kw_only=True)
-class DVCXDatasetLock(DatasetSpec):
+class DatachainDatasetLock(DatasetSpec):
     version: int
     created_at: datetime = field(converter=to_datetime)
 
@@ -160,13 +160,13 @@ class DVCDataset:
 
 
 @frozen(kw_only=True)
-class DVCXDataset:
+class DatachainDataset:
     manifest_path: str
     spec: "DatasetSpec"
-    lock: "Optional[DVCXDatasetLock]" = field(default=None)
+    lock: "Optional[DatachainDatasetLock]" = field(default=None)
     _invalidated: bool = field(default=False, eq=False, repr=False)
 
-    type: ClassVar[Literal["dvcx"]] = "dvcx"
+    type: ClassVar[Literal["dc"]] = "dc"
 
     @property
     def pinned(self) -> bool:
@@ -193,7 +193,7 @@ class DVCXDataset:
         name, _version = self.name_version
         version = version if version is not None else _version
         version_info = _get_dataset_info(name, record=record, version=version)
-        lock = DVCXDatasetLock(
+        lock = DatachainDatasetLock(
             **self.spec.to_dict(),
             version=version_info.version,
             created_at=version_info.created_at,
@@ -226,9 +226,9 @@ class URLDataset:
         return evolve(self, lock=lock)
 
 
-Lock = Union[DVCDatasetLock, DVCXDatasetLock, URLDatasetLock]
+Lock = Union[DVCDatasetLock, DatachainDatasetLock, URLDatasetLock]
 Spec = Union[DatasetSpec, DVCDatasetSpec]
-Dataset = Union[DVCDataset, DVCXDataset, URLDataset]
+Dataset = Union[DVCDataset, DatachainDataset, URLDataset]
 
 
 class DatasetNotFoundError(DvcException, KeyError):
@@ -307,13 +307,13 @@ class Datasets(Mapping[str, Dataset]):
             raise ValueError("type should be present in spec")
         if typ == "dvc":
             return DVCDatasetSpec.from_dict(spec)
-        if typ in {"dvcx", "url"}:
+        if typ in {"dc", "url"}:
             return DatasetSpec.from_dict(spec)
         raise ValueError(f"unknown dataset type: {spec.get('type', '')}")
 
     @staticmethod
     def _lock_from_info(lock: Optional[dict[str, Any]]) -> Optional[Lock]:
-        kl = {"dvc": DVCDatasetLock, "dvcx": DVCXDatasetLock, "url": URLDatasetLock}
+        kl = {"dvc": DVCDatasetLock, "dc": DatachainDatasetLock, "url": URLDatasetLock}
         if lock and (cls := kl.get(lock.get("type", ""))):  # type: ignore[assignment]
             return cls.from_dict(lock)  # type: ignore[attr-defined]
         return None
@@ -356,9 +356,9 @@ class Datasets(Mapping[str, Dataset]):
                 lock=lock,
                 invalidated=_invalidated,
             )
-        if spec.type == "dvcx":
-            assert lock is None or isinstance(lock, DVCXDatasetLock)
-            return DVCXDataset(
+        if spec.type == "dc":
+            assert lock is None or isinstance(lock, DatachainDatasetLock)
+            return DatachainDataset(
                 manifest_path=manifest_path,
                 spec=spec,
                 lock=lock,
@@ -374,7 +374,7 @@ class Datasets(Mapping[str, Dataset]):
         manifest_path: StrPath = "dvc.yaml",
         **kwargs: Any,
     ) -> Dataset:
-        assert type in {"dvc", "dvcx", "url"}
+        assert type in {"dvc", "dc", "url"}
         kwargs.update({"name": name, "url": url, "type": type})
         dataset = self._build_dataset(os.path.abspath(manifest_path), kwargs)
         dataset = dataset.update(self.repo)
@@ -389,13 +389,16 @@ class Datasets(Mapping[str, Dataset]):
 
         if dataset.type == "url" and (version or kwargs.get("rev")):
             raise ValueError("cannot update version/revision for a url")
-        if dataset.type == "dvcx" and version is not None:
+        if dataset.type == "dc" and version is not None:
             if not isinstance(version, int):
                 raise TypeError(
-                    f"dvcx version has to be an integer, got {type(version).__name__!r}"
+                    "DataChain dataset version has to be an integer, "
+                    f"got {type(version).__name__!r}"
                 )
             if version < 1:
-                raise ValueError(f"dvcx version should be >=1, got {version}")
+                raise ValueError(
+                    f"DataChain dataset version should be >=1, got {version}"
+                )
 
         new = dataset.update(self.repo, **kwargs)
 

dvc/repo/du.py

@@ -6,7 +6,7 @@ def du(
     path: Optional[str] = None,
     rev: Optional[str] = None,
     summarize: bool = False,
-    config: Union[None, dict[str, Any], str] = None,
+    config: Union[dict[str, Any], str, None] = None,
     remote: Optional[str] = None,
     remote_config: Optional[dict] = None,
 ):

dvc/repo/experiments/__init__.py

@@ -205,7 +205,7 @@ class Experiments:
         Experiment will be derived from the current workspace.
         """
 
-        name = kwargs.get("name", None)
+        name = kwargs.get("name")
         baseline_sha = kwargs.get("baseline_rev") or self.repo.scm.get_rev()
 
         if name:

dvc/repo/experiments/queue/base.py

@@ -372,7 +372,7 @@ class BaseStashQueue(ABC):
         )
 
     def _stash_commit_deps(self, *args, **kwargs):
-        if len(args):
+        if args:
             targets = args[0]
         else:
             targets = kwargs.get("targets")
@@ -609,9 +609,9 @@ class BaseStashQueue(ABC):
                 result[exp_name] = entry_name_dict[exp_name]
                 continue
             if self.scm.is_sha(exp_name):
-                for rev in entry_rev_dict:
+                for rev, entry in entry_rev_dict.items():
                     if rev.startswith(exp_name.lower()):
-                        result[exp_name] = entry_rev_dict[rev]
+                        result[exp_name] = entry
                         break
 
         return result

dvc/repo/experiments/refs.py

@@ -67,3 +67,12 @@ class ExpRefInfo:
         baseline_sha = parts[2] + parts[3]
         name = parts[4] if len(parts) == 5 else None
         return cls(baseline_sha, name)
+
+    def __eq__(self, other):
+        if not isinstance(other, ExpRefInfo):
+            return False
+
+        return self.baseline_sha == other.baseline_sha and self.name == other.name
+
+    def __hash__(self):
+        return hash((self.baseline_sha, self.name))

dvc/repo/experiments/remove.py

@@ -6,7 +6,7 @@ from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
 from dvc.scm import Git, iter_revs
 
-from .exceptions import UnresolvedExpNamesError
+from .exceptions import InvalidArgumentError, UnresolvedExpNamesError
 from .utils import exp_refs, exp_refs_by_baseline, push_refspec
 
 if TYPE_CHECKING:
@@ -24,16 +24,22 @@ logger = logger.getChild(__name__)
 @scm_context
 def remove(  # noqa: C901, PLR0912
     repo: "Repo",
-    exp_names: Union[None, str, list[str]] = None,
+    exp_names: Union[str, list[str], None] = None,
     rev: Optional[Union[list[str], str]] = None,
     all_commits: bool = False,
     num: int = 1,
     queue: bool = False,
     git_remote: Optional[str] = None,
+    keep: bool = False,
 ) -> list[str]:
     removed: list[str] = []
+
+    if all([keep, queue]):
+        raise InvalidArgumentError("Cannot use both `--keep` and `--queue`.")
+
     if not any([exp_names, queue, all_commits, rev]):
         return removed
+
     celery_queue: LocalCeleryQueue = repo.experiments.celery_queue
 
     if queue:
@@ -43,6 +49,7 @@ def remove(  # noqa: C901, PLR0912
 
     exp_ref_list: list[ExpRefInfo] = []
     queue_entry_list: list[QueueEntry] = []
+
     if exp_names:
         results: dict[str, ExpRefAndQueueEntry] = (
             celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
@@ -68,6 +75,10 @@ def remove(  # noqa: C901, PLR0912
         exp_ref_list.extend(exp_ref_dict.values())
     elif all_commits:
         exp_ref_list.extend(exp_refs(repo.scm, git_remote))
+        removed.extend([ref.name for ref in exp_ref_list])
+
+    if keep:
+        exp_ref_list = list(set(exp_refs(repo.scm, git_remote)) - set(exp_ref_list))
         removed = [ref.name for ref in exp_ref_list]
 
     if exp_ref_list:
@@ -83,6 +94,7 @@ def remove(  # noqa: C901, PLR0912
 
         removed_refs = [str(r) for r in exp_ref_list]
         notify_refs_to_studio(repo, git_remote, removed=removed_refs)
+
     return removed
 
 

dvc/repo/experiments/run.py

@@ -93,7 +93,7 @@ def run(  # noqa: C901, PLR0912
 
     for idx, sweep_overrides in enumerate(sweeps):
         if hydra_sweep and name_prefix is not None:
-            kwargs["name"] = f"{name_prefix}-{idx+1}"
+            kwargs["name"] = f"{name_prefix}-{idx + 1}"
         queue_entry = repo.experiments.queue_one(
             repo.experiments.celery_queue,
             targets=targets,

dvc/repo/install.py

@@ -1,5 +1,6 @@
 from typing import TYPE_CHECKING
 
+from dvc import version_tuple
 from dvc.exceptions import DvcException
 
 if TYPE_CHECKING:
@@ -16,19 +17,19 @@ def pre_commit_install(scm: "Git") -> None:
     with modify_yaml(config_path) as config:
         entry = {
             "repo": "https://github.com/iterative/dvc",
-            "rev": "main",
+            "rev": ".".join(map(str, version_tuple[:3])),
             "hooks": [
                 {
                     "id": "dvc-pre-commit",
                     "additional_dependencies": [".[all]"],
                     "language_version": "python3",
-                    "stages": ["commit"],
+                    "stages": ["pre-commit"],
                 },
                 {
                     "id": "dvc-pre-push",
                     "additional_dependencies": [".[all]"],
                     "language_version": "python3",
-                    "stages": ["push"],
+                    "stages": ["pre-push"],
                 },
                 {
                     "id": "dvc-post-checkout",

dvc/repo/ls.py

@@ -4,8 +4,44 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 if TYPE_CHECKING:
     from dvc.fs.dvc import DVCFileSystem
 
+
+def _open_repo(
+    url: str,
+    rev: Optional[str] = None,
+    config: Union[dict[str, Any], str, None] = None,
+    remote: Optional[str] = None,
+    remote_config: Optional[dict] = None,
+):
+    from dvc.config import Config
+
     from . import Repo
 
+    if config and not isinstance(config, dict):
+        config_dict = Config.load_file(config)
+    else:
+        config_dict = None
+
+    return Repo.open(
+        url,
+        rev=rev,
+        subrepos=True,
+        uninitialized=True,
+        config=config_dict,
+        remote=remote,
+        remote_config=remote_config,
+    )
+
+
+def _adapt_info(info: dict[str, Any]) -> dict[str, Any]:
+    dvc_info = info.get("dvc_info", {})
+    return {
+        "isout": dvc_info.get("isout", False),
+        "isdir": info["type"] == "directory",
+        "isexec": info.get("isexec", False),
+        "size": info.get("size"),
+        "md5": dvc_info.get("md5") or dvc_info.get("md5-dos2unix"),
+    }
+
 
 def ls(
     url: str,
@@ -13,9 +49,10 @@ def ls(
     rev: Optional[str] = None,
     recursive: Optional[bool] = None,
     dvc_only: bool = False,
-    config: Union[None, dict[str, Any], str] = None,
+    config: Union[dict[str, Any], str, None] = None,
     remote: Optional[str] = None,
     remote_config: Optional[dict] = None,
+    maxdepth: Optional[int] = None,
 ):
     """Methods for getting files and outputs for the repo.
 
@@ -41,60 +78,60 @@ def ls(
             "isexec": bool,
         }
     """
-    from dvc.config import Config
-
-    from . import Repo
-
-    if config and not isinstance(config, dict):
-        config_dict = Config.load_file(config)
-    else:
-        config_dict = None
-
-    with Repo.open(
-        url,
-        rev=rev,
-        subrepos=True,
-        uninitialized=True,
-        config=config_dict,
-        remote=remote,
-        remote_config=remote_config,
-    ) as repo:
+    with _open_repo(url, rev, config, remote, remote_config) as repo:
         path = path or ""
+        fs: DVCFileSystem = repo.dvcfs
+        fs_path = fs.from_os_path(path)
+        return _ls(fs, fs_path, recursive, dvc_only, maxdepth)
 
-        ret = _ls(repo, path, recursive, dvc_only)
 
-        ret_list = []
-        for path, info in ret.items():
-            info["path"] = path
-            ret_list.append(info)
-        ret_list.sort(key=lambda f: f["path"])
-        return ret_list
+def ls_tree(
+    url: str,
+    path: Optional[str] = None,
+    rev: Optional[str] = None,
+    dvc_only: bool = False,
+    config: Union[dict[str, Any], str, None] = None,
+    remote: Optional[str] = None,
+    remote_config: Optional[dict] = None,
+    maxdepth: Optional[int] = None,
+):
+    with _open_repo(url, rev, config, remote, remote_config) as repo:
+        path = path or ""
+        fs: DVCFileSystem = repo.dvcfs
+        fs_path = fs.from_os_path(path)
+        return _ls_tree(
+            fs, fs_path, maxdepth=maxdepth, dvc_only=dvc_only, dvcfiles=True
+        )
 
 
 def _ls(
-    repo: "Repo",
+    fs: "DVCFileSystem",
     path: str,
     recursive: Optional[bool] = None,
     dvc_only: bool = False,
+    maxdepth: Optional[int] = None,
 ):
-    fs: DVCFileSystem = repo.dvcfs
-    fs_path = fs.from_os_path(path)
-
-    fs_path = fs.info(fs_path)["name"]
+    fs_path = fs.info(path)["name"]
 
     infos = {}
-    if fs.isfile(fs_path):
-        infos[os.path.basename(path)] = fs.info(fs_path)
+
+    # ignore maxdepth only if recursive is not set
+    maxdepth = maxdepth if recursive else None
+    if maxdepth == 0 or fs.isfile(fs_path):
+        infos[os.path.basename(path) or os.curdir] = fs.info(fs_path)
     else:
         for root, dirs, files in fs.walk(
-            fs_path, dvcfiles=True, dvc_only=dvc_only, detail=True
+            fs_path,
+            dvcfiles=True,
+            dvc_only=dvc_only,
+            detail=True,
+            maxdepth=maxdepth,
         ):
-            if not recursive:
-                files.update(dirs)
-
             parts = fs.relparts(root, fs_path)
             if parts == (".",):
                 parts = ()
+            if not recursive or (maxdepth and len(parts) >= maxdepth - 1):
+                files.update(dirs)
 
             for name, entry in files.items():
                 infos[os.path.join(*parts, name)] = entry
@@ -102,14 +139,44 @@ def _ls(
             if not recursive:
                 break
 
-    ret = {}
-    for name, info in infos.items():
-        dvc_info = info.get("dvc_info", {})
-        ret[name] = {
-            "isout": dvc_info.get("isout", False),
-            "isdir": info["type"] == "directory",
-            "isexec": info.get("isexec", False),
-            "size": info.get("size"),
-        }
+    ret_list = []
+    for p, info in sorted(infos.items(), key=lambda x: x[0]):
+        _info = _adapt_info(info)
+        _info["path"] = p
+        ret_list.append(_info)
+    return ret_list
 
+
+def _ls_tree(fs, path, maxdepth=None, _info=None, **fs_kwargs):
+    info = _info or fs.info(path)
+    if _info is None:
+        # preserve the original path name
+        name = path
+        if not name:
+            name = os.curdir if fs.protocol == "local" else fs.root_marker
+        path = info["name"]
+    else:
+        name = path.rsplit(fs.sep, 1)[-1]
+
+    ret = {}
+    ls_info = _adapt_info(info)
+    ls_info["path"] = path
+
+    recurse = maxdepth is None or maxdepth > 0
+    if recurse and info["type"] == "directory":
+        try:
+            infos = fs.ls(path, detail=True, **fs_kwargs)
+        except FileNotFoundError:
+            # broken symlink?
+            infos = []
+
+        infos.sort(key=lambda f: f["name"])
+        maxdepth = maxdepth - 1 if maxdepth is not None else None
+        contents = {}
+        for info in infos:
+            d = _ls_tree(fs, info["name"], maxdepth=maxdepth, _info=info, **fs_kwargs)
+            contents.update(d)
+        ls_info["contents"] = contents
+
+    ret[name] = ls_info
     return ret

dvc/repo/ls_url.py

@@ -1,19 +1,30 @@
+from fsspec.implementations.local import LocalFileSystem as _LocalFileSystem
+
 from dvc.exceptions import URLMissingError
-from dvc.fs import parse_external_url
+from dvc.fs import LocalFileSystem, parse_external_url
 
 
-def ls_url(url, *, fs_config=None, recursive=False, config=None):
+def ls_url(url, *, fs_config=None, recursive=False, maxdepth=None, config=None):
     fs, fs_path = parse_external_url(url, fs_config=fs_config, config=config)
     try:
         info = fs.info(fs_path)
     except FileNotFoundError as exc:
         raise URLMissingError(url) from exc
-    if info["type"] != "directory":
+    if maxdepth == 0 or info["type"] != "directory":
         return [{"path": info["name"], "isdir": False}]
 
+    if isinstance(fs, LocalFileSystem):
+        # dvc's LocalFileSystem does not support maxdepth yet
+        walk = _LocalFileSystem().walk
+    else:
+        walk = fs.walk
+
     ret = []
-    for _, dirs, files in fs.walk(fs_path, detail=True):
-        if not recursive:
+    for root, dirs, files in walk(fs_path, detail=True, maxdepth=maxdepth):
+        parts = fs.relparts(root, fs_path)
+        if parts == (".",):
+            parts = ()
+        if not recursive or (maxdepth and len(parts) >= maxdepth - 1):
             files.update(dirs)
 
         for info in files.values():

dvc/repo/open_repo.py

@@ -99,7 +99,7 @@ def clean_repos():
 
 def _get_remote_config(url):
     try:
-        repo = Repo(url)
+        repo = Repo(url, uninitialized=True)
     except NotDvcRepoError:
         return {}
 
@@ -109,10 +109,16 @@ def _get_remote_config(url):
             # Fill the empty upstream entry with a new remote pointing to the
             # original repo's cache location.
             name = "auto-generated-upstream"
-            return {
-                "core": {"remote": name},
-                "remote": {name: {"url": repo.cache.local_cache_dir}},
-            }
+            try:
+                local_cache_dir = repo.cache.local_cache_dir
+            except AttributeError:
+                # if the `.dvc` dir is missing, we get an AttributeError
+                return {}
+            else:
+                return {
+                    "core": {"remote": name},
+                    "remote": {name: {"url": local_cache_dir}},
+                }
 
         # Use original remote to make sure that we are using correct url,
         # credential paths, etc if they are relative to the config location.

dvc/stage/__init__.py

@@ -83,7 +83,7 @@ class RawData:
 def create_stage(cls: type[_T], repo, path, **kwargs) -> _T:
     from dvc.dvcfile import check_dvcfile_path
 
-    wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
+    wdir = os.path.abspath(kwargs.get("wdir") or os.curdir)
     path = os.path.abspath(path)
 
     check_dvcfile_path(repo, path)
@@ -435,7 +435,7 @@ class Stage(params.StageParams):
         elif not self.changed(True, upstream) and pull:
             try:
                 logger.info("Pulling data for %s", self)
-                self.repo.pull(self.addressing, jobs=kwargs.get("jobs", None))
+                self.repo.pull(self.addressing, jobs=kwargs.get("jobs"))
                 self.checkout()
                 return None
             except CheckoutError:
@@ -609,7 +609,7 @@ class Stage(params.StageParams):
             self.remove_outs(ignore_remove=False, force=False)
 
         if (self.is_import and not self.frozen) or self.is_partial_import:
-            self._sync_import(dry, force, kwargs.get("jobs", None), no_download)
+            self._sync_import(dry, force, kwargs.get("jobs"), no_download)
         elif not self.frozen and self.cmd:
             self._run_stage(dry, force, **kwargs)
         elif not dry:

dvc/testing/benchmarks/fixtures.py

@@ -138,8 +138,7 @@ def dvc_bin(request, make_dvc_bin):
         version = Version(parse(dvc_version).base_version)
         if version not in spec:
             pytest.skip(
-                f"Version {dvc_version} "
-                f"does not satisfy requirement {spec!r}: {reason}"
+                f"Version {dvc_version} does not satisfy requirement {spec!r}: {reason}"
             )
     return make_dvc_bin
 
@@ -150,7 +149,8 @@ def make_bench(request):
         import pytest_benchmark.plugin
 
         # hack from https://github.com/ionelmc/pytest-benchmark/issues/166
-        bench = pytest_benchmark.plugin.benchmark.__pytest_wrapped__.obj(request)
+        bench_gen = pytest_benchmark.plugin.benchmark.__pytest_wrapped__.obj(request)
+        bench = next(bench_gen)
 
         suffix = f"-{name}"
 
@@ -172,7 +172,7 @@ def bench_dvc(request, dvc_bin, make_bench):
         name = kwargs.pop("name", None)
         name = f"-{name}" if name else ""
         bench = make_bench(args[0] + name)
-        if request.config.getoption("--benchmark-cprofile-dump") or kwargs.pop(
+        if request.config.getoption("--dvc-benchmark-cprofile-dump") or kwargs.pop(
             "cprofile", False
         ):
             cprofile_results = request.config.invocation_params.dir / "prof"

dvc/testing/benchmarks/plugin.py

@@ -73,7 +73,7 @@ def pytest_addoption(parser):
     )
 
     parser.addoption(
-        "--benchmark-cprofile-dump",
+        "--dvc-benchmark-cprofile-dump",
         action="store_true",
         default=False,
         help="Save cprofile results",

dvc/testing/workspace_tests.py

@@ -225,6 +225,39 @@ class TestLsUrl:
             ],
         )
 
+        result = ls_url(
+            str(cloud / "dir"), fs_config=cloud.config, recursive=True, maxdepth=0
+        )
+        match_files(
+            fs,
+            result,
+            [{"path": (cloud / "dir").fs_path, "isdir": False}],
+        )
+
+        result = ls_url(
+            str(cloud / "dir"), fs_config=cloud.config, recursive=True, maxdepth=1
+        )
+        match_files(
+            fs,
+            result,
+            [
+                {"path": "foo", "isdir": False},
+                {"path": "subdir", "isdir": True},
+            ],
+        )
+
+        result = ls_url(
+            str(cloud / "dir"), fs_config=cloud.config, recursive=True, maxdepth=2
+        )
+        match_files(
+            fs,
+            result,
+            [
+                {"path": "foo", "isdir": False},
+                {"path": "subdir/bar", "isdir": False},
+            ],
+        )
+
     def test_nonexistent(self, cloud):
         with pytest.raises(URLMissingError):
             ls_url(str(cloud / "dir"), fs_config=cloud.config)

dvc/ui/__init__.py

@@ -300,6 +300,7 @@ class Console:
         header_styles: Optional[Union[dict[str, "Styles"], Sequence["Styles"]]] = None,
         row_styles: Optional[Sequence["Styles"]] = None,
         borders: Union[bool, str] = False,
+        colalign: Optional[tuple[str, ...]] = None,
     ) -> None:
         from dvc.ui import table as t
 
@@ -321,7 +322,13 @@ class Console:
             return
 
         return t.plain_table(
-            self, data, headers, markdown=markdown, pager=pager, force=force
+            self,
+            data,
+            headers,
+            markdown=markdown,
+            pager=pager,
+            force=force,
+            colalign=colalign,
         )
 
     def status(self, status: str, **kwargs: Any) -> "Status":

dvc/ui/table.py

@@ -29,6 +29,7 @@ def plain_table(
     markdown: bool = False,
     pager: bool = False,
     force: bool = True,
+    colalign: Optional[tuple[str, ...]] = None,
 ) -> None:
     from funcy import nullcontext
     from tabulate import tabulate
@@ -40,6 +41,7 @@ def plain_table(
         disable_numparse=True,
         # None will be shown as "" by default, overriding
         missingval="-",
+        colalign=colalign,
     )
     if markdown:
         # NOTE: md table is incomplete without the trailing newline

dvc/utils/__init__.py

@@ -382,7 +382,7 @@ def glob_targets(targets, glob=True, recursive=True):
 
 def error_handler(func):
     def wrapper(*args, **kwargs):
-        onerror = kwargs.get("onerror", None)
+        onerror = kwargs.get("onerror")
         result = {}
 
         try:

dvc/utils/serialize/_common.py

@@ -33,7 +33,7 @@ class LoaderFn(Protocol):
     def __call__(self, path: "StrPath", fs: Optional["FileSystem"] = None) -> Any: ...
 
 
-ReadType = Union[bytes, None, str]
+ReadType = Union[bytes, str, None]
 ParserFn = Callable[[ReadType, "StrPath"], dict]
 
 
@@ -61,7 +61,7 @@ def _load_data(
 ):
     open_fn = fs.open if fs else open
     encoding = "utf-8"
-    with open_fn(path, encoding=encoding, **kwargs) as fd:  # type: ignore[operator]
+    with open_fn(path, encoding=encoding, **kwargs) as fd:  # type: ignore[arg-type]
         with reraise(UnicodeDecodeError, EncodingError(path, encoding)):
             return parser(fd.read(), path)
 
@@ -74,7 +74,7 @@ def _dump_data(
     **dumper_args,
 ):
     open_fn = fs.open if fs else open
-    with open_fn(path, "w+", encoding="utf-8") as fd:  # type: ignore[operator]
+    with open_fn(path, "w+", encoding="utf-8") as fd:  # type: ignore[call-overload]
         dumper(data, fd, **dumper_args)
 
 

dvc/utils/studio.py

@@ -23,7 +23,7 @@ if TYPE_CHECKING:
 
 logger = logger.getChild(__name__)
 
-STUDIO_URL = "https://studio.dvc.ai"
+STUDIO_URL = "https://studio.datachain.ai"
 
 
 def post(

dvc/version.py

@@ -1,6 +1,6 @@
 try:
-    from ._dvc_version import version as __version__  # type: ignore[import]
-    from ._dvc_version import version_tuple  # type: ignore[import]
+    from ._version import version as __version__  # type: ignore[import]
+    from ._version import version_tuple  # type: ignore[import]
 except ImportError:
     __version__ = "UNKNOWN"
     version_tuple = (0, 0, __version__)  # type: ignore[assignment]

pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 build-backend = "setuptools.build_meta"
-requires = ["setuptools>=61", "setuptools_scm[toml]>=7"]
+requires = ["setuptools>=77", "setuptools_scm[toml]>=8"]
 
 [project]
 name = "dvc"
@@ -16,7 +16,8 @@ keywords = [
     "machine-learning",
     "reproducibility",
 ]
-license = { text = "Apache License 2.0" }
+license = "Apache-2.0"
+license-files = ["LICENSE"]
 maintainers = [{ name = "Iterative", email = "[email protected]" }]
 authors = [{ name = "Dmitry Petrov", email = "[email protected]" }]
 requires-python = ">=3.9"
@@ -27,6 +28,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 dynamic = ["version"]
 
@@ -34,7 +36,7 @@ dependencies = [
     "attrs>=22.2.0",
     "celery",
     "colorama>=0.3.9",
-    "configobj>=5.0.6",
+    "configobj>=5.0.9",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
     "dulwich",
@@ -45,7 +47,7 @@ dependencies = [
     "dvc-studio-client>=0.21,<1",
     "dvc-task>=0.3.0,<1",
     "flatten_dict<1,>=0.4.1",
-    "flufl.lock>=8.1.0,<9", # https://github.com/iterative/dvc/issues/9654
+    "flufl.lock>=8.1.0,<9",
     "fsspec>=2024.2.0",
     "funcy>=1.14",
     "grandalf<1,>=0.7",
@@ -57,7 +59,7 @@ dependencies = [
     "omegaconf",
     "packaging>=19",
     "pathspec>=0.10.3",
-    "platformdirs<4,>=3.1.1",
+    "platformdirs<5,>=3.1.1",
     "psutil>=5.8",
     "pydot>=1.2.4",
     "pygtrie>=2.3.2",
@@ -65,7 +67,7 @@ dependencies = [
     "requests>=2.22",
     "rich>=12",
     "ruamel.yaml>=0.17.11",
-    "scmrepo>=3.3.4,<4",
+    "scmrepo>=3.3.8,<4",
     "shortuuid>=0.5",
     "shtab<2,>=1.3.4",
     "tabulate>=0.8.7",
@@ -83,7 +85,7 @@ gdrive = ["dvc-gdrive>=3,<4"]
 gs = ["dvc-gs>=3,<4"]
 hdfs = ["dvc-hdfs>=3,<4"]
 lint = [
-    "mypy==1.11.2",
+    "mypy==1.15.0",
     "pandas-stubs",
     "types-colorama",
     "types-psutil",
@@ -99,7 +101,7 @@ s3 = ["dvc-s3>=3.2.0,<4"]
 ssh = ["dvc-ssh>=4,<5"]
 ssh_gssapi = ["dvc-ssh[gssapi]>=4,<5"]
 testing = [
-  "pytest-benchmark[histogram]",
+  "pytest-benchmark[histogram]>=5,<6",
   "pytest-test-utils",
   "uv",
 ]
@@ -148,15 +150,12 @@ dvc = "dvc.fs.dvc_path:DVCPath"
 hook-dirs = "dvc.__pyinstaller:get_hook_dirs"
 tests = "dvc.__pyinstaller:get_PyInstaller_tests"
 
-[tool.setuptools]
-license-files = ["LICENSE"]
-
 [tool.setuptools.packages.find]
 exclude = ["tests", "tests.*"]
 namespaces = false
 
 [tool.setuptools_scm]
-write_to = "dvc/_dvc_version.py"
+write_to = "dvc/_version.py"
 
 [tool.pytest.ini_options]
 addopts = "-ra --cov-config pyproject.toml"
@@ -172,6 +171,8 @@ filterwarnings = [
     "ignore:Deprecated call to `pkg_resources.declare_namespace:DeprecationWarning",
     # see https://github.com/networkx/networkx/issues/5723.
     "ignore:nx.nx_pydot.* depends on the pydot package, which has.*known issues and is not actively maintained:DeprecationWarning",
+    # TODO: investigate where we are not closing sqlite3.Connection
+    "ignore:unclosed database.*sqlite3.Connection:ResourceWarning",
 ]
 log_level = "debug"
 markers = [
@@ -264,14 +265,14 @@ show-fixes = true
 
 [tool.ruff.lint]
 ignore = [
-    "N818", "S101", "ISC001", "PT007", "RET502", "RET503", "SIM105", "SIM108", "SIM117",
+    "N818", "S101", "A005", "PT007", "RET502", "RET503", "SIM105", "SIM108", "SIM117",
     "TRY003", "TRY300", "PERF203", "PLR2004", "PLW2901", "LOG007",
 ]
 select = [
     "F", "E", "W", "C90", "I", "N", "UP", "YTT", "ASYNC", "S", "BLE", "B", "A", "C4", "DTZ", "T10",
     "EXE", "ISC", "ICN", "LOG", "G", "INP", "PIE", "T20", "PYI", "PT", "Q", "RSE", "RET",
-    "SLOT", "SIM", "TID", "TCH", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY",
-    "FLY", "PERF", "FURB", "RUF", "RUF022", "RUF023", "RUF024", "RUF026", "RUF027", "RUF029", "RUF030",
+    "SLOT", "SIM", "TID", "TC", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY",
+    "FLY", "PERF", "FURB", "RUF", "RUF027", "RUF029", "RUF036",
 ]
 preview = true
 explicit-preview-rules = true

tests/func/api/test_data.py

@@ -5,6 +5,7 @@ from funcy import first, get_in
 
 from dvc import api
 from dvc.exceptions import OutputNotFoundError, PathMissingError
+from dvc.scm import CloneError
 from dvc.testing.api_tests import TestAPI  # noqa: F401
 from dvc.testing.tmp_dir import make_subrepo
 from dvc.utils.fs import remove
@@ -65,6 +66,34 @@ def test_get_url_from_remote(tmp_dir, erepo_dir, cloud, local_cloud):
     )
 
 
+def test_get_url_ignore_scm(tmp_dir, dvc, cloud, scm):
+    tmp_dir.add_remote(config=cloud.config)
+    tmp_dir.dvc_gen("foo", "foo", commit="add foo")
+
+    repo_posix = tmp_dir.as_posix()
+    expected_url = (cloud / "files" / "md5" / "ac/bd18db4cc2f85cedef654fccc4a4d8").url
+
+    # Test baseline with scm
+    assert api.get_url("foo", repo=repo_posix) == expected_url
+
+    # Simulate gitless environment (e.g. deployed container)
+    (tmp_dir / ".git").rename(tmp_dir / "gitless_environment")
+
+    assert api.get_url("foo", repo=repo_posix) == expected_url
+    assert (
+        api.get_url("foo", repo=repo_posix, config={"core": {"no_scm": True}})
+        == expected_url
+    )
+
+    # Addressing repos with `file://` triggers git, so it fails in a gitless environment
+    repo_url = f"file://{repo_posix}"
+    with pytest.raises(
+        CloneError,
+        match="SCM error",
+    ):
+        api.get_url("foo", repo=repo_url, config={"core": {"no_scm": True}})
+
+
 def test_open_external(tmp_dir, erepo_dir, cloud):
     erepo_dir.add_remote(config=cloud.config)
 

tests/func/experiments/test_remove.py

@@ -18,7 +18,7 @@ def test_remove_experiments_by_ref(tmp_dir, scm, dvc, exp_stage, caplog):
         ref_name_list.append(str(ref_info))
 
     with pytest.raises(InvalidArgumentError):
-        dvc.experiments.remove(ref_name_list[:2] + ["non-exist"])
+        dvc.experiments.remove([*ref_name_list[:2], "non-exist"])
     assert scm.get_ref(ref_name_list[0]) is not None
     assert scm.get_ref(ref_name_list[1]) is not None
     assert scm.get_ref(ref_name_list[2]) is not None
@@ -43,6 +43,26 @@ def test_remove_all_queued_experiments(tmp_dir, scm, dvc, exp_stage):
     assert scm.get_ref(str(ref_info)) is not None
 
 
+def test_remove_all_experiments_queued_and_completed(tmp_dir, scm, dvc, exp_stage):
+    queue_length = 3
+    for i in range(queue_length):
+        dvc.experiments.run(
+            exp_stage.addressing, params=[f"foo={i}"], name=f"exp{i}", queue=True
+        )
+
+    results = dvc.experiments.run(
+        exp_stage.addressing, params=[f"foo={queue_length}"], name=f"exp{queue_length}"
+    )
+    ref_info = first(exp_refs_by_rev(scm, first(results)))
+
+    removed = sorted(dvc.experiments.remove(all_commits=True, queue=True))
+
+    assert len(removed) == queue_length + 1
+    assert removed == [f"exp{i}" for i in range(queue_length)] + [ref_info.name]
+    assert len(dvc.experiments.stash_revs) == 0
+    assert scm.get_ref(str(ref_info)) is None
+
+
 def test_remove_special_queued_experiments(tmp_dir, scm, dvc, exp_stage):
     dvc.experiments.run(
         exp_stage.addressing, params=["foo=1"], queue=True, name="queue1"
@@ -179,3 +199,85 @@ def test_remove_multi_rev(tmp_dir, scm, dvc, exp_stage):
 
     assert scm.get_ref(str(baseline_exp_ref)) is None
     assert scm.get_ref(str(new_exp_ref)) is None
+
+
+@pytest.mark.parametrize(
+    "keep, expected_removed",
+    [
+        [["exp1"], ["exp2", "exp3"]],
+        [["exp1", "exp2"], ["exp3"]],
+        [["exp1", "exp2", "exp3"], []],
+        [[], []],  # remove does nothing if no experiments are specified
+    ],
+)
+def test_keep_selected_by_name(tmp_dir, scm, dvc, exp_stage, keep, expected_removed):
+    # Setup: Run experiments
+    refs = {}
+    for i in range(1, len(keep) + len(expected_removed) + 1):
+        results = dvc.experiments.run(
+            exp_stage.addressing, params=[f"foo={i}"], name=f"exp{i}"
+        )
+        refs[f"exp{i}"] = first(exp_refs_by_rev(scm, first(results)))
+        assert scm.get_ref(str(refs[f"exp{i}"])) is not None
+
+    removed = dvc.experiments.remove(exp_names=keep, keep=True)
+    assert sorted(removed) == sorted(expected_removed)
+
+    for exp in expected_removed:
+        assert scm.get_ref(str(refs[exp])) is None
+
+    for exp in keep:
+        assert scm.get_ref(str(refs[exp])) is not None
+
+
+def test_keep_selected_by_nonexistent_name(tmp_dir, scm, dvc, exp_stage):
+    # non existent name should raise an error
+    with pytest.raises(UnresolvedExpNamesError):
+        dvc.experiments.remove(exp_names=["nonexistent"], keep=True)
+
+
+@pytest.mark.parametrize(
+    "num_exps, rev, num, expected_removed",
+    [
+        [2, "exp1", 1, ["exp2"]],
+        [3, "exp3", 1, ["exp1", "exp2"]],
+        [3, "exp3", 2, ["exp1"]],
+        [3, "exp3", 3, []],
+        [3, "exp2", 2, ["exp3"]],
+        [4, "exp2", 2, ["exp3", "exp4"]],
+        [4, "exp4", 2, ["exp1", "exp2"]],
+        [1, None, 1, []],  # remove does nothing if no experiments are specified
+    ],
+)
+def test_keep_selected_by_rev(
+    tmp_dir, scm, dvc, exp_stage, num_exps, rev, num, expected_removed
+):
+    refs = {}
+    revs = {}
+    # Setup: Run experiments and commit
+    for i in range(1, num_exps + 1):
+        scm.commit(f"commit{i}")
+        results = dvc.experiments.run(
+            exp_stage.addressing, params=[f"foo={i}"], name=f"exp{i}"
+        )
+        refs[f"exp{i}"] = first(exp_refs_by_rev(scm, first(results)))
+        revs[f"exp{i}"] = scm.get_rev()
+        assert scm.get_ref(str(refs[f"exp{i}"])) is not None
+
+    # Keep the experiment from the new revision
+    removed = dvc.experiments.remove(rev=revs.get(rev), num=num, keep=True)
+    assert sorted(removed) == sorted(expected_removed)
+
+    # Check remaining experiments
+    for exp in expected_removed:
+        assert scm.get_ref(str(refs[exp])) is None
+
+    for exp, ref in refs.items():
+        if exp not in expected_removed:
+            assert scm.get_ref(str(ref)) is not None
+
+
+def test_remove_with_queue_and_keep(tmp_dir, scm, dvc, exp_stage):
+    # This should raise an exception, until decided otherwise
+    with pytest.raises(InvalidArgumentError):
+        dvc.experiments.remove(queue=True, keep=True)

tests/func/experiments/test_set_params.py

@@ -133,7 +133,7 @@ def test_hydra_sweep_prefix_name(tmp_dir, params_repo, dvc):
     db_values = ["mysql", "postgresql"]
     param = "+db=" + ",".join(db_values)
     dvc.experiments.run(params=[param], queue=True, name=prefix)
-    expected_names = [f"{prefix}-{i+1}" for i, _ in enumerate(db_values)]
+    expected_names = [f"{prefix}-{i + 1}" for i, _ in enumerate(db_values)]
     exp_names = [entry.name for entry in dvc.experiments.celery_queue.iter_queued()]
     for name, expected in zip(exp_names, expected_names):
         assert name == expected

tests/func/parsing/test_top_level.py

@@ -121,7 +121,7 @@ def test_artifacts(tmp_dir, dvc):
 def test_datasets(tmp_dir, dvc):
     template = {
         "datasets": [
-            {"name": "${ds1.name}", "url": "${ds1.url}", "type": "dvcx"},
+            {"name": "${ds1.name}", "url": "${ds1.url}", "type": "dc"},
             {
                 "name": "${ds2.name}",
                 "url": "${ds2.url}",
@@ -138,7 +138,7 @@ def test_datasets(tmp_dir, dvc):
 
     (tmp_dir / "params.yaml").dump(
         {
-            "ds1": {"name": "dogs", "url": "dvcx://dogs"},
+            "ds1": {"name": "dogs", "url": "dc://dogs"},
             "ds2": {
                 "name": "example-get-started",
                 "url": "[email protected]:iterative/example-get-started.git",
@@ -153,7 +153,7 @@ def test_datasets(tmp_dir, dvc):
 
     resolver = DataResolver(dvc, tmp_dir, template)
     assert resolver.resolve_datasets() == [
-        {"name": "dogs", "url": "dvcx://dogs", "type": "dvcx"},
+        {"name": "dogs", "url": "dc://dogs", "type": "dc"},
         {
             "name": "example-get-started",
             "url": "[email protected]:iterative/example-get-started.git",

tests/func/plots/test_show.py

@@ -205,11 +205,8 @@ def test_plots_show_nested_x_dict(tmp_dir, dvc, scm):
 
     dvclive_metrics_dir.gen(
         {
-            "Error.tsv": "step\tError\n" "0\t0.11\n" "1\t0.22\n" "2\t0.44\n",
-            "Max_Leaf_Nodes.tsv": "step\tMax_Leaf_Nodes\n"
-            "0\t5\n"
-            "1\t50\n"
-            "2\t500\n",
+            "Error.tsv": "step\tError\n0\t0.11\n1\t0.22\n2\t0.44\n",
+            "Max_Leaf_Nodes.tsv": "step\tMax_Leaf_Nodes\n0\t5\n1\t50\n2\t500\n",
         }
     )
 

tests/func/repro/test_repro.py

@@ -759,7 +759,7 @@ def test_repro_dry(tmp_dir, dvc, copy_script):
 
     stages = dvc.reproduce(stage.addressing, dry=True)
 
-    assert len(stages), 2
+    assert len(stages) == 2
     assert not filecmp.cmp("file1", "bar", shallow=False)
 
     ret = main(["repro", "--dry", stage.addressing])

tests/func/test_checkout.py

@@ -744,3 +744,12 @@ def test_checkout_dir_compat(tmp_dir, dvc):
     remove("data")
     dvc.checkout()
     assert (tmp_dir / "data").read_text() == {"foo": "foo"}
+
+
+def test_checkout_cleanup_properly_on_untracked_nested_directories(tmp_dir, scm, dvc):
+    tmp_dir.dvc_gen({"datasets": {"dir1": {"file1": "file1"}}})
+    tmp_dir.gen({"datasets": {"dir2": {"dir3": {"file2": "file2"}}}})
+
+    dvc.checkout(force=True)
+
+    assert (tmp_dir / "datasets").read_text() == {"dir1": {"file1": "file1"}}

tests/func/test_dataset.py

@@ -8,13 +8,13 @@ from attrs import define, evolve, field, has
 from dvc.dependency.base import Dependency
 from dvc.exceptions import ReproductionError
 from dvc.repo.datasets import (
+    DatachainDataset,
+    DatachainDatasetLock,
     DatasetNotFoundError,
     DatasetSpec,
     DVCDataset,
     DVCDatasetLock,
     DVCDatasetSpec,
-    DVCXDataset,
-    DVCXDatasetLock,
     FileInfo,
     URLDataset,
     URLDatasetLock,
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
 
 
 @define
-class MockedDVCXVersionInfo:
+class MockedDatachainVersionInfo:
     version: int
     created_at: datetime = field(factory=lambda: datetime.now(timezone.utc))
 
@@ -77,21 +77,21 @@ def test_dvc(tmp_dir, scm, dvc: "Repo"):
     assert old == new
 
 
-def test_dvcx(tmp_dir, dvc, mocker):
+def test_datachain(tmp_dir, dvc, mocker):
     datasets = dvc.datasets
 
-    version_info = [MockedDVCXVersionInfo(1), MockedDVCXVersionInfo(2)]
+    version_info = [MockedDatachainVersionInfo(1), MockedDatachainVersionInfo(2)]
     version_info.append(version_info[1])
     mocker.patch("dvc.repo.datasets._get_dataset_info", side_effect=version_info)
 
-    dataset = datasets.add("mydataset", "dataset", "dvcx")
-    expected = DVCXDataset(
+    dataset = datasets.add("mydataset", "dataset", "dc")
+    expected = DatachainDataset(
         manifest_path=(tmp_dir / "dvc.yaml").fs_path,
-        spec=DatasetSpec(name="mydataset", url="dataset", type="dvcx"),
-        lock=DVCXDatasetLock(
+        spec=DatasetSpec(name="mydataset", url="dataset", type="dc"),
+        lock=DatachainDatasetLock(
             name="mydataset",
             url="dataset",
-            type="dvcx",
+            type="dc",
             version=1,
             created_at=version_info[0].created_at,
         ),
@@ -189,16 +189,16 @@ def test_dvc_dump(tmp_dir, dvc):
     assert dvc.datasets["mydataset"] == dataset
 
 
-def test_dvcx_dump(tmp_dir, dvc):
+def test_datachain_dump(tmp_dir, dvc):
     manifest_path = os.path.join(tmp_dir, "dvc.yaml")
-    spec = DatasetSpec(name="mydataset", url="dataset", type="dvcx")
+    spec = DatasetSpec(name="mydataset", url="dataset", type="dc")
     dt = datetime.now(tz=timezone.utc)
-    lock = DVCXDatasetLock(version=1, created_at=dt, **spec.to_dict())
-    dataset = DVCXDataset(manifest_path=manifest_path, spec=spec, lock=lock)
+    lock = DatachainDatasetLock(version=1, created_at=dt, **spec.to_dict())
+    dataset = DatachainDataset(manifest_path=manifest_path, spec=spec, lock=lock)
 
     dvc.datasets.dump(dataset)
 
-    spec_d = {"name": "mydataset", "type": "dvcx", "url": "dataset"}
+    spec_d = {"name": "mydataset", "type": "dc", "url": "dataset"}
     assert (tmp_dir / "dvc.yaml").parse() == {"datasets": [spec_d]}
     assert (tmp_dir / "dvc.lock").parse() == {
         "schema": "2.0",
@@ -244,10 +244,10 @@ def test_url_dump(tmp_dir, dvc):
 def test_invalidation(tmp_dir, dvc):
     manifest_path = os.path.join(tmp_dir, "dvc.yaml")
     spec = DatasetSpec(name="mydataset", url="url1", type="url")
-    lock = DVCXDatasetLock(
+    lock = DatachainDatasetLock(
         name="mydataset",
         url="dataset",
-        type="dvcx",
+        type="dc",
         version=1,
         created_at=datetime.now(tz=timezone.utc),
     )
@@ -289,15 +289,15 @@ def test_dvc_dataset_pipeline(tmp_dir, dvc, scm):
     assert dvc.reproduce() == [stage]
 
 
-def test_dvcx_dataset_pipeline(mocker, tmp_dir, dvc):
-    version_info = [MockedDVCXVersionInfo(1), MockedDVCXVersionInfo(2)]
+def test_datachain_dataset_pipeline(mocker, tmp_dir, dvc):
+    version_info = [MockedDatachainVersionInfo(1), MockedDatachainVersionInfo(2)]
     mocker.patch("dvc.repo.datasets._get_dataset_info", side_effect=version_info)
 
-    dvc.datasets.add("mydataset", "dataset", "dvcx")
+    dvc.datasets.add("mydataset", "dataset", "dc")
 
     stage = dvc.stage.add(cmd="echo", name="train", deps=["ds://mydataset"])
     assert (tmp_dir / "dvc.yaml").parse() == {
-        "datasets": [{"name": "mydataset", "url": "dataset", "type": "dvcx"}],
+        "datasets": [{"name": "mydataset", "url": "dataset", "type": "dc"}],
         "stages": {"train": {"cmd": "echo", "deps": ["ds://mydataset"]}},
     }
 
@@ -360,10 +360,10 @@ def test_url_dataset_pipeline(mocker, tmp_dir, dvc):
 def test_pipeline_when_not_in_sync(tmp_dir, dvc):
     manifest_path = os.path.join(tmp_dir, "dvc.yaml")
     spec = DatasetSpec(name="mydataset", url="url1", type="url")
-    lock = DVCXDatasetLock(
+    lock = DatachainDatasetLock(
         name="mydataset",
         url="dataset",
-        type="dvcx",
+        type="dc",
         version=1,
         created_at=datetime.now(tz=timezone.utc),
     )
@@ -382,9 +382,9 @@ def test_pipeline_when_not_in_sync(tmp_dir, dvc):
 def test_collect(tmp_dir, dvc):
     manifest_path1 = os.path.join(tmp_dir, "dvc.yaml")
     dt = datetime.now(tz=timezone.utc)
-    spec = DatasetSpec(name="mydataset1", url="url1", type="dvcx")
-    lock = DVCXDatasetLock(version=1, created_at=dt, **spec.to_dict())
-    mydataset1 = DVCXDataset(manifest_path=manifest_path1, spec=spec, lock=lock)
+    spec = DatasetSpec(name="mydataset1", url="url1", type="dc")
+    lock = DatachainDatasetLock(version=1, created_at=dt, **spec.to_dict())
+    mydataset1 = DatachainDataset(manifest_path=manifest_path1, spec=spec, lock=lock)
     dvc.datasets.dump(mydataset1)
 
     (tmp_dir / "sub").mkdir()
@@ -418,7 +418,7 @@ def test_parametrized(tmp_dir, dvc):
     (tmp_dir / "dvc.yaml").dump(
         {
             "datasets": [
-                {"name": "${ds1.name}", "url": "${ds1.url}", "type": "dvcx"},
+                {"name": "${ds1.name}", "url": "${ds1.url}", "type": "dc"},
                 {
                     "name": "${ds2.name}",
                     "url": "${ds2.url}",
@@ -435,7 +435,7 @@ def test_parametrized(tmp_dir, dvc):
     )
     (tmp_dir / "params.yaml").dump(
         {
-            "ds1": {"name": "dogs", "url": "dvcx://dogs"},
+            "ds1": {"name": "dogs", "url": "dc://dogs"},
             "ds2": {
                 "name": "example-get-started",
                 "url": "[email protected]:iterative/example-get-started.git",
@@ -450,9 +450,9 @@ def test_parametrized(tmp_dir, dvc):
 
     path = (tmp_dir / "dvc.yaml").fs_path
     assert dict(dvc.datasets.items()) == {
-        "dogs": DVCXDataset(
+        "dogs": DatachainDataset(
             manifest_path=path,
-            spec=DatasetSpec(name="dogs", url="dvcx://dogs", type="dvcx"),
+            spec=DatasetSpec(name="dogs", url="dc://dogs", type="dc"),
         ),
         "example-get-started": DVCDataset(
             manifest_path=path,

tests/func/test_import.py

@@ -12,7 +12,7 @@ from dvc.scm import Git
 from dvc.stage.exceptions import StagePathNotFoundError
 from dvc.testing.tmp_dir import make_subrepo
 from dvc.utils.fs import remove
-from dvc_data.hashfile import hash
+from dvc_data.hashfile import hash as _hash
 from dvc_data.index.index import DataIndex, DataIndexDirError
 
 
@@ -747,7 +747,7 @@ def test_import_no_hash(
     with erepo_dir.chdir():
         erepo_dir.dvc_gen(files, commit="create foo")
 
-    file_md5_spy = mocker.spy(hash, "file_md5")
+    file_md5_spy = mocker.spy(_hash, "file_md5")
     index_info_spy = mocker.spy(DataIndex, "info")
     name = next(iter(files))
 

tests/func/test_ls.py

@@ -2,10 +2,13 @@ import os
 import shutil
 import textwrap
 from operator import itemgetter
+from os.path import join
 
 import pytest
 
+from dvc.fs import MemoryFileSystem
 from dvc.repo import Repo
+from dvc.repo.ls import _ls_tree, ls_tree
 from dvc.scm import CloneError
 
 FS_STRUCTURE = {
@@ -222,7 +225,12 @@ def test_ls_repo_with_missed_path_dvc_only(tmp_dir, dvc, scm):
     tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
 
     with pytest.raises(FileNotFoundError):
-        Repo.ls(os.fspath(tmp_dir), path="missed_path", recursive=True, dvc_only=True)
+        Repo.ls(
+            os.fspath(tmp_dir),
+            path="missed_path",
+            recursive=True,
+            dvc_only=True,
+        )
 
 
 def test_ls_repo_with_removed_dvc_dir(tmp_dir, dvc, scm):
@@ -467,20 +475,49 @@ def test_ls_granular(erepo_dir, M):
 
     entries = Repo.ls(os.fspath(erepo_dir), os.path.join("dir", "subdir"))
     assert entries == [
-        {"isout": True, "isdir": False, "isexec": False, "path": "bar", "size": 3},
-        {"isout": True, "isdir": False, "isexec": False, "path": "foo", "size": 3},
+        {
+            "isout": True,
+            "isdir": False,
+            "isexec": False,
+            "path": "bar",
+            "size": 3,
+            "md5": "37b51d194a7513e45b56f6524f2d51f2",
+        },
+        {
+            "isout": True,
+            "isdir": False,
+            "isexec": False,
+            "path": "foo",
+            "size": 3,
+            "md5": "acbd18db4cc2f85cedef654fccc4a4d8",
+        },
     ]
 
     entries = Repo.ls(os.fspath(erepo_dir), "dir")
     assert entries == [
-        {"isout": True, "isdir": False, "isexec": False, "path": "1", "size": 1},
-        {"isout": True, "isdir": False, "isexec": False, "path": "2", "size": 1},
+        {
+            "isout": True,
+            "isdir": False,
+            "isexec": False,
+            "path": "1",
+            "size": 1,
+            "md5": "c4ca4238a0b923820dcc509a6f75849b",
+        },
+        {
+            "isout": True,
+            "isdir": False,
+            "isexec": False,
+            "path": "2",
+            "size": 1,
+            "md5": "c81e728d9d4c2f636f067f89cc14862c",
+        },
         {
             "isout": True,
             "isdir": True,
             "isexec": False,
             "path": "subdir",
             "size": M.instance_of(int),
+            "md5": None,
         },
     ]
 
@@ -506,14 +543,42 @@ def test_ls_target(erepo_dir, use_scm):
         return Repo.ls(os.fspath(erepo_dir), path)
 
     assert _ls(os.path.join("dir", "1")) == [
-        {"isout": isout, "isdir": False, "isexec": False, "path": "1", "size": 1}
+        {
+            "isout": isout,
+            "isdir": False,
+            "isexec": False,
+            "path": "1",
+            "size": 1,
+            "md5": "c4ca4238a0b923820dcc509a6f75849b" if not use_scm else None,
+        }
     ]
     assert _ls(os.path.join("dir", "subdir", "foo")) == [
-        {"isout": isout, "isdir": False, "isexec": False, "path": "foo", "size": 3}
+        {
+            "isout": isout,
+            "isdir": False,
+            "isexec": False,
+            "path": "foo",
+            "size": 3,
+            "md5": "acbd18db4cc2f85cedef654fccc4a4d8" if not use_scm else None,
+        }
     ]
     assert _ls(os.path.join("dir", "subdir")) == [
-        {"isdir": False, "isexec": 0, "isout": isout, "path": "bar", "size": 3},
-        {"isdir": False, "isexec": 0, "isout": isout, "path": "foo", "size": 3},
+        {
+            "isdir": False,
+            "isexec": 0,
+            "isout": isout,
+            "path": "bar",
+            "size": 3,
+            "md5": "37b51d194a7513e45b56f6524f2d51f2" if not use_scm else None,
+        },
+        {
+            "isdir": False,
+            "isexec": 0,
+            "isout": isout,
+            "path": "foo",
+            "size": 3,
+            "md5": "acbd18db4cc2f85cedef654fccc4a4d8" if not use_scm else None,
+        },
     ]
 
 
@@ -576,6 +641,7 @@ def test_broken_symlink(tmp_dir, dvc, M):
             "isexec": False,
             "path": ".dvcignore",
             "size": M.instance_of(int),
+            "md5": None,
         },
         {
             "isout": False,
@@ -583,6 +649,7 @@ def test_broken_symlink(tmp_dir, dvc, M):
             "isexec": False,
             "path": "link",
             "size": 0,
+            "md5": None,
         },
     ]
 
@@ -614,14 +681,23 @@ def test_ls_broken_dir(tmp_dir, dvc, M):
             "isout": False,
             "path": ".dvcignore",
             "size": M.instance_of(int),
+            "md5": None,
+        },
+        {
+            "isdir": True,
+            "isexec": False,
+            "isout": True,
+            "path": "broken",
+            "size": 3,
+            "md5": "630bd47b538d2a513c7d267d07e0bc44.dir",
         },
-        {"isdir": True, "isexec": False, "isout": True, "path": "broken", "size": 3},
         {
             "isdir": False,
             "isexec": False,
             "isout": False,
             "path": "broken.dvc",
             "size": M.instance_of(int),
+            "md5": None,
         },
         {
             "isdir": True,
@@ -629,6 +705,7 @@ def test_ls_broken_dir(tmp_dir, dvc, M):
             "isout": True,
             "path": "dir",
             "size": M.instance_of(int),
+            "md5": "91aaa9bb58b657d623ef143b195a67e4.dir",
         },
         {
             "isdir": False,
@@ -636,14 +713,23 @@ def test_ls_broken_dir(tmp_dir, dvc, M):
             "isout": False,
             "path": "dir.dvc",
             "size": M.instance_of(int),
+            "md5": None,
+        },
+        {
+            "isdir": False,
+            "isexec": False,
+            "isout": True,
+            "path": "foo",
+            "size": 3,
+            "md5": "acbd18db4cc2f85cedef654fccc4a4d8",
         },
-        {"isdir": False, "isexec": False, "isout": True, "path": "foo", "size": 3},
         {
             "isdir": False,
             "isexec": False,
             "isout": False,
             "path": "foo.dvc",
             "size": M.instance_of(int),
+            "md5": None,
         },
     ]
 
@@ -652,3 +738,325 @@ def test_ls_broken_dir(tmp_dir, dvc, M):
 
     with pytest.raises(DataIndexDirError):
         Repo.ls(os.fspath(tmp_dir), recursive=True)
+
+
+def test_ls_maxdepth(tmp_dir, scm, dvc):
+    tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
+    tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
+
+    files = Repo.ls(os.fspath(tmp_dir), "structure.xml", maxdepth=0, recursive=True)
+    match_files(files, ((("structure.xml",), True),))
+
+    files = Repo.ls(os.fspath(tmp_dir), maxdepth=0, recursive=True)
+    match_files(files, (((os.curdir,), False),))
+
+    files = Repo.ls(os.fspath(tmp_dir), maxdepth=1, recursive=True)
+    match_files(
+        files,
+        (
+            ((".dvcignore",), False),
+            ((".gitignore",), False),
+            (("README.md",), False),
+            (("structure.xml.dvc",), False),
+            (("model",), False),
+            (("data",), False),
+            (("structure.xml",), True),
+        ),
+    )
+    files = Repo.ls(os.fspath(tmp_dir), maxdepth=2, recursive=True)
+    match_files(
+        files,
+        (
+            ((".dvcignore",), False),
+            ((".gitignore",), False),
+            (("README.md",), False),
+            ((join("data", "subcontent"),), False),
+            ((join("model", ".gitignore"),), False),
+            ((join("model", "people.csv"),), True),
+            ((join("model", "people.csv.dvc"),), False),
+            ((join("model", "script.py"),), False),
+            ((join("model", "train.py"),), False),
+            (("structure.xml",), True),
+            (("structure.xml.dvc",), False),
+        ),
+    )
+
+    files = Repo.ls(os.fspath(tmp_dir), maxdepth=3, recursive=True)
+    match_files(
+        files,
+        (
+            ((".dvcignore",), False),
+            ((".gitignore",), False),
+            (("README.md",), False),
+            ((join("data", "subcontent", ".gitignore"),), False),
+            ((join("data", "subcontent", "data.xml"),), True),
+            ((join("data", "subcontent", "data.xml.dvc"),), False),
+            ((join("data", "subcontent", "statistics"),), False),
+            ((join("model", ".gitignore"),), False),
+            ((join("model", "people.csv"),), True),
+            ((join("model", "people.csv.dvc"),), False),
+            ((join("model", "script.py"),), False),
+            ((join("model", "train.py"),), False),
+            ((join("structure.xml"),), True),
+            ((join("structure.xml.dvc"),), False),
+        ),
+    )
+
+    files = Repo.ls(os.fspath(tmp_dir), maxdepth=4, recursive=True)
+    match_files(
+        files,
+        (
+            ((".dvcignore",), False),
+            ((".gitignore",), False),
+            (("README.md",), False),
+            ((join("data", "subcontent", ".gitignore"),), False),
+            ((join("data", "subcontent", "data.xml"),), True),
+            ((join("data", "subcontent", "data.xml.dvc"),), False),
+            ((join("data", "subcontent", "statistics", ".gitignore"),), False),
+            ((join("data", "subcontent", "statistics", "data.csv"),), True),
+            ((join("data", "subcontent", "statistics", "data.csv.dvc"),), False),
+            ((join("model", ".gitignore"),), False),
+            ((join("model", "people.csv"),), True),
+            ((join("model", "people.csv.dvc"),), False),
+            ((join("model", "script.py"),), False),
+            ((join("model", "train.py"),), False),
+            (("structure.xml",), True),
+            (("structure.xml.dvc",), False),
+        ),
+    )
+
+
+def _simplify_tree(files):
+    ret = {}
+    for path, info in files.items():
+        if content := info.get("contents"):
+            ret[path] = _simplify_tree(content)
+        else:
+            ret[path] = None
+    return ret
+
+
+def test_ls_tree(M, tmp_dir, scm, dvc):
+    tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
+    tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
+
+    files = ls_tree(os.fspath(tmp_dir), "structure.xml")
+    assert _simplify_tree(files) == {"structure.xml": None}
+
+    files = ls_tree(os.fspath(tmp_dir))
+
+    expected = {
+        ".": {
+            ".dvcignore": None,
+            ".gitignore": None,
+            "README.md": None,
+            "data": {
+                "subcontent": {
+                    ".gitignore": None,
+                    "data.xml": None,
+                    "data.xml.dvc": None,
+                    "statistics": {
+                        ".gitignore": None,
+                        "data.csv": None,
+                        "data.csv.dvc": None,
+                    },
+                }
+            },
+            "model": {
+                ".gitignore": None,
+                "people.csv": None,
+                "people.csv.dvc": None,
+                "script.py": None,
+                "train.py": None,
+            },
+            "structure.xml": None,
+            "structure.xml.dvc": None,
+        }
+    }
+    assert _simplify_tree(files) == expected
+
+    files = ls_tree(os.fspath(tmp_dir), "model")
+    assert _simplify_tree(files) == {
+        "model": {
+            ".gitignore": None,
+            "people.csv": None,
+            "people.csv.dvc": None,
+            "script.py": None,
+            "train.py": None,
+        }
+    }
+
+
+def test_ls_tree_dvc_only(M, tmp_dir, scm, dvc):
+    tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
+    tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
+
+    files = ls_tree(os.fspath(tmp_dir), dvc_only=True)
+
+    expected = {
+        ".": {
+            "data": {
+                "subcontent": {"data.xml": None, "statistics": {"data.csv": None}}
+            },
+            "model": {"people.csv": None},
+            "structure.xml": None,
+        }
+    }
+    assert _simplify_tree(files) == expected
+
+
+def test_ls_tree_maxdepth(M, tmp_dir, scm, dvc):
+    tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
+    tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
+
+    files = ls_tree(os.fspath(tmp_dir), maxdepth=0)
+    assert _simplify_tree(files) == {".": None}
+
+    files = ls_tree(os.fspath(tmp_dir), maxdepth=1)
+    assert _simplify_tree(files) == {
+        ".": {
+            ".dvcignore": None,
+            ".gitignore": None,
+            "README.md": None,
+            "data": None,
+            "model": None,
+            "structure.xml": None,
+            "structure.xml.dvc": None,
+        }
+    }
+
+    files = ls_tree(os.fspath(tmp_dir), maxdepth=2)
+    assert _simplify_tree(files) == {
+        ".": {
+            ".dvcignore": None,
+            ".gitignore": None,
+            "README.md": None,
+            "data": {"subcontent": None},
+            "model": {
+                ".gitignore": None,
+                "people.csv": None,
+                "people.csv.dvc": None,
+                "script.py": None,
+                "train.py": None,
+            },
+            "structure.xml": None,
+            "structure.xml.dvc": None,
+        }
+    }
+
+    files = ls_tree(os.fspath(tmp_dir), maxdepth=3)
+    assert _simplify_tree(files) == {
+        ".": {
+            ".dvcignore": None,
+            ".gitignore": None,
+            "README.md": None,
+            "data": {
+                "subcontent": {
+                    ".gitignore": None,
+                    "data.xml": None,
+                    "data.xml.dvc": None,
+                    "statistics": None,
+                }
+            },
+            "model": {
+                ".gitignore": None,
+                "people.csv": None,
+                "people.csv.dvc": None,
+                "script.py": None,
+                "train.py": None,
+            },
+            "structure.xml": None,
+            "structure.xml.dvc": None,
+        }
+    }
+
+    files = ls_tree(os.fspath(tmp_dir), maxdepth=4)
+    assert _simplify_tree(files) == {
+        ".": {
+            ".dvcignore": None,
+            ".gitignore": None,
+            "README.md": None,
+            "data": {
+                "subcontent": {
+                    ".gitignore": None,
+                    "data.xml": None,
+                    "data.xml.dvc": None,
+                    "statistics": {
+                        ".gitignore": None,
+                        "data.csv": None,
+                        "data.csv.dvc": None,
+                    },
+                }
+            },
+            "model": {
+                ".gitignore": None,
+                "people.csv": None,
+                "people.csv.dvc": None,
+                "script.py": None,
+                "train.py": None,
+            },
+            "structure.xml": None,
+            "structure.xml.dvc": None,
+        }
+    }
+
+
+def test_fs_ls_tree():
+    fs = MemoryFileSystem(global_store=False)
+    fs.pipe({f: content.encode() for f, content in FS_STRUCTURE.items()})
+    root = fs.root_marker
+
+    files = _ls_tree(fs, "README.md")
+    assert _simplify_tree(files) == {"README.md": None}
+    files = _ls_tree(fs, root)
+    expected = {
+        root: {
+            ".gitignore": None,
+            "README.md": None,
+            "model": {
+                "script.py": None,
+                "train.py": None,
+            },
+        }
+    }
+    assert _simplify_tree(files) == expected
+
+    files = _ls_tree(fs, "model")
+    assert _simplify_tree(files) == {
+        "model": {
+            "script.py": None,
+            "train.py": None,
+        }
+    }
+
+
+def test_fs_ls_tree_maxdepth():
+    fs = MemoryFileSystem(global_store=False)
+    fs.pipe({f: content.encode() for f, content in FS_STRUCTURE.items()})
+
+    files = _ls_tree(fs, "/", maxdepth=0)
+    assert _simplify_tree(files) == {"/": None}
+
+    files = _ls_tree(fs, "/", maxdepth=1)
+    assert _simplify_tree(files) == {
+        "/": {
+            ".gitignore": None,
+            "README.md": None,
+            "model": None,
+        }
+    }
+
+    files = _ls_tree(fs, "/", maxdepth=2)
+    assert _simplify_tree(files) == {
+        "/": {
+            ".gitignore": None,
+            "README.md": None,
+            "model": {
+                "script.py": None,
+                "train.py": None,
+            },
+        }
+    }
+
+    files = _ls_tree(fs, "README.md", maxdepth=3)
+    assert _simplify_tree(files) == {"README.md": None}

tests/func/test_merge_driver.py

@@ -132,7 +132,8 @@ def test_merge_different_output_options(tmp_dir, dvc, caplog):
     )
 
     (tmp_dir / "their").write_text(
-        "outs:\n"
+        r"outs:"
+        "\n"
         "- md5: f987654321.dir\n"
         "  hash: md5\n"
         "  path: path\n"

tests/func/test_remote.py

@@ -115,6 +115,20 @@ def test_show_default(dvc, capsys):
     assert out == "foo\n"
 
 
+def test_list_shows_default(dvc, capsys):
+    default_remote = "foo"
+    other_remote = "bar"
+    bucket_url = "s3://bucket/name"
+    assert main(["remote", "add", default_remote, bucket_url]) == 0
+    assert main(["remote", "add", other_remote, bucket_url]) == 0
+    assert main(["remote", "default", default_remote]) == 0
+    assert main(["remote", "list"]) == 0
+    out, _ = capsys.readouterr()
+    out_lines = out.splitlines()
+    assert out_lines[0].split() == [default_remote, bucket_url, "(default)"]
+    assert out_lines[1].split() == [other_remote, bucket_url]
+
+
 def test_upper_case_remote(tmp_dir, dvc, local_cloud):
     remote_name = "UPPERCASEREMOTE"
 

tests/func/test_repo_index.py

@@ -114,13 +114,13 @@ def assert_index_equal(first, second, strict=True, ordered=True):
     assert len(first) == len(second), "Index have different no. of stages"
     assert set(first) == set(second), "Index does not have same stages"
     if ordered:
-        assert list(first) == list(
-            second
-        ), "Index does not have same sequence of stages"
+        assert list(first) == list(second), (
+            "Index does not have same sequence of stages"
+        )
     if strict:
-        assert set(map(id, first)) == set(
-            map(id, second)
-        ), "Index is not strictly equal"
+        assert set(map(id, first)) == set(map(id, second)), (
+            "Index is not strictly equal"
+        )
 
 
 def test_skip_graph_checks(dvc, mocker):

tests/func/test_state.py

@@ -65,7 +65,7 @@ def test_get_unused_links(tmp_dir, dvc):
     assert set(dvc.state.get_unused_links(links, dvc.fs)) == set()
     assert set(
         dvc.state.get_unused_links(
-            (links[:1] + [os.path.join(dvc.root_dir, "not-existing-file")]),
+            ([*links[:1], os.path.join(dvc.root_dir, "not-existing-file")]),
             dvc.fs,
         )
     ) == {"bar"}

tests/integration/plots/test_plots.py

@@ -553,11 +553,8 @@ def test_nested_x_defn_collection(tmp_dir, dvc, scm, capsys):
     )
     dvclive_metrics_dir.gen(
         {
-            "Error.tsv": "step\tError\n" "0\t0.11\n" "1\t0.22\n" "2\t0.44\n",
-            "Max_Leaf_Nodes.tsv": "step\tMax_Leaf_Nodes\n"
-            "0\t5\n"
-            "1\t50\n"
-            "2\t500\n",
+            "Error.tsv": "step\tError\n0\t0.11\n1\t0.22\n2\t0.44\n",
+            "Max_Leaf_Nodes.tsv": "step\tMax_Leaf_Nodes\n0\t5\n1\t50\n2\t500\n",
         }
     )
     (other_logger_dir / "multiple_metrics.json").dump(

tests/unit/command/ls/test_ls.py

@@ -1,7 +1,7 @@
 import json
 
 from dvc.cli import parse_args
-from dvc.commands.ls import CmdList
+from dvc.commands.ls import CmdList, show_tree
 
 
 def _test_cli(mocker, *args):
@@ -27,6 +27,7 @@ def test_list(mocker):
         config=None,
         remote=None,
         remote_config=None,
+        maxdepth=None,
     )
 
 
@@ -42,6 +43,7 @@ def test_list_recursive(mocker):
         config=None,
         remote=None,
         remote_config=None,
+        maxdepth=None,
     )
 
 
@@ -57,6 +59,7 @@ def test_list_git_ssh_rev(mocker):
         config=None,
         remote=None,
         remote_config=None,
+        maxdepth=None,
     )
 
 
@@ -73,6 +76,7 @@ def test_list_targets(mocker):
         config=None,
         remote=None,
         remote_config=None,
+        maxdepth=None,
     )
 
 
@@ -88,6 +92,7 @@ def test_list_outputs_only(mocker):
         config=None,
         remote=None,
         remote_config=None,
+        maxdepth=None,
     )
 
 
@@ -114,6 +119,65 @@ def test_list_config(mocker):
         config="myconfig",
         remote="myremote",
         remote_config={"k1": "v1", "k2": "v2"},
+        maxdepth=None,
+    )
+
+
+def test_list_level(mocker):
+    url = "local_dir"
+    m = _test_cli(mocker, url, None, "--level", "1")
+    m.assert_called_once_with(
+        url,
+        None,
+        recursive=False,
+        rev=None,
+        dvc_only=False,
+        config=None,
+        remote=None,
+        remote_config=None,
+        maxdepth=1,
+    )
+
+
+def test_list_tree(mocker):
+    url = "[email protected]:repo"
+    cli_args = parse_args(
+        [
+            "list",
+            url,
+            "local_dir",
+            "--rev",
+            "123",
+            "--tree",
+            "--show-hash",
+            "--size",
+            "--level",
+            "2",
+            "--dvc-only",
+            "--config",
+            "myconfig",
+            "--remote",
+            "myremote",
+            "--remote-config",
+            "k1=v1",
+            "k2=v2",
+        ]
+    )
+    assert cli_args.func == CmdList
+
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch("dvc.repo.ls.ls_tree")
+
+    assert cmd.run() == 0
+    m.assert_called_once_with(
+        url,
+        "local_dir",
+        rev="123",
+        dvc_only=True,
+        config="myconfig",
+        remote="myremote",
+        remote_config={"k1": "v1", "k2": "v2"},
+        maxdepth=2,
     )
 
 
@@ -165,6 +229,268 @@ def test_show_colors(mocker, capsys, monkeypatch):
     ]
 
 
+def test_show_size(mocker, capsys):
+    cli_args = parse_args(["list", "local_dir", "--size"])
+    assert cli_args.func == CmdList
+    cmd = cli_args.func(cli_args)
+
+    result = [
+        {
+            "isdir": False,
+            "isexec": 0,
+            "isout": False,
+            "path": ".dvcignore",
+            "size": 100,
+        },
+        {
+            "isdir": False,
+            "isexec": 0,
+            "isout": False,
+            "path": ".gitignore",
+            "size": 200,
+        },
+        {
+            "isdir": False,
+            "isexec": 0,
+            "isout": False,
+            "path": "README.md",
+            "size": 100_000,
+        },
+    ]
+    mocker.patch("dvc.repo.Repo.ls", return_value=result)
+
+    assert cmd.run() == 0
+    out, _ = capsys.readouterr()
+    entries = out.splitlines()
+
+    assert entries == ["  100  .dvcignore", "  200  .gitignore", "97.7k  README.md"]
+
+
+def test_show_hash(mocker, capsys):
+    cli_args = parse_args(["list", "local_dir", "--show-hash"])
+    assert cli_args.func == CmdList
+    cmd = cli_args.func(cli_args)
+
+    result = [
+        {
+            "isdir": False,
+            "isexec": 0,
+            "isout": False,
+            "path": ".dvcignore",
+            "md5": "123",
+        },
+        {
+            "isdir": False,
+            "isexec": 0,
+            "isout": False,
+            "path": ".gitignore",
+            "md5": "456",
+        },
+        {
+            "isdir": False,
+            "isexec": 0,
+            "isout": False,
+            "path": "README.md",
+            "md5": "789",
+        },
+    ]
+    mocker.patch("dvc.repo.Repo.ls", return_value=result)
+
+    assert cmd.run() == 0
+    out, _ = capsys.readouterr()
+    entries = out.splitlines()
+
+    assert entries == ["123  .dvcignore", "456  .gitignore", "789  README.md"]
+
+
+def test_show_size_and_hash(mocker, capsys):
+    cli_args = parse_args(["list", "local_dir", "--size", "--show-hash"])
+    assert cli_args.func == CmdList
+    cmd = cli_args.func(cli_args)
+
+    result = [
+        {
+            "isdir": False,
+            "isexec": 0,
+            "isout": False,
+            "path": ".dvcignore",
+            "size": 100,
+            "md5": "123",
+        },
+        {
+            "isdir": False,
+            "isexec": 0,
+            "isout": False,
+            "path": ".gitignore",
+            "size": 200,
+            "md5": "456",
+        },
+        {
+            "isdir": False,
+            "isexec": 0,
+            "isout": False,
+            "path": "README.md",
+            "size": 100_000,
+            "md5": "789",
+        },
+    ]
+    mocker.patch("dvc.repo.Repo.ls", return_value=result)
+
+    assert cmd.run() == 0
+    out, _ = capsys.readouterr()
+    entries = out.splitlines()
+
+    assert entries == [
+        "  100  123  .dvcignore",
+        "  200  456  .gitignore",
+        "97.7k  789  README.md",
+    ]
+
+
+def test_show_tree(capsys):
+    entries = {
+        "data": {
+            "isout": True,
+            "isdir": True,
+            "isexec": False,
+            "size": 192,
+            "path": "data",
+            "md5": "3fb071066d5d5b282f56a0169340346d.dir",
+            "contents": {
+                "dir": {
+                    "isout": False,
+                    "isdir": True,
+                    "isexec": False,
+                    "size": 96,
+                    "path": "data/dir",
+                    "md5": None,
+                    "contents": {
+                        "subdir": {
+                            "isout": True,
+                            "isdir": True,
+                            "isexec": False,
+                            "size": 96,
+                            "md5": None,
+                            "path": "data/dir/subdir",
+                            "contents": {
+                                "foobar": {
+                                    "isout": True,
+                                    "isdir": False,
+                                    "isexec": False,
+                                    "size": 4,
+                                    "md5": "d3b07384d113edec49eaa6238ad5ff00",
+                                }
+                            },
+                        },
+                        "foo": {
+                            "isout": False,
+                            "isdir": False,
+                            "isexec": False,
+                            "size": 4,
+                            "path": "data/dir/foo",
+                            "md5": None,
+                        },
+                    },
+                },
+                "bar": {
+                    "isout": True,
+                    "isdir": False,
+                    "isexec": False,
+                    "size": 4,
+                    "path": "data/bar",
+                    "md5": "c157a79031e1c40f85931829bc5fc552",
+                },
+                "large-file": {
+                    "isout": False,
+                    "isdir": False,
+                    "isexec": False,
+                    "size": 1073741824,
+                    "path": "data/large-file",
+                    "md5": None,
+                },
+                "dir2": {
+                    "isout": True,
+                    "isdir": True,
+                    "isexec": False,
+                    "size": 96,
+                    "md5": None,
+                    "path": "data/dir2",
+                    "contents": {
+                        "foo": {
+                            "isout": True,
+                            "isdir": False,
+                            "isexec": False,
+                            "size": 4,
+                            "path": "data/dir2/foo",
+                            "md5": "d3b07384d113edec49eaa6238ad5ff00",
+                        }
+                    },
+                },
+            },
+        }
+    }
+
+    show_tree(entries, with_color=False)
+    out, _ = capsys.readouterr()
+    expected = """\
+data
+├── dir
+│   ├── subdir
+│   │   └── foobar
+│   └── foo
+├── bar
+├── large-file
+└── dir2
+    └── foo
+"""
+    assert out == expected
+
+    show_tree(entries, with_color=False, with_size=True)
+    out, _ = capsys.readouterr()
+    expected = """\
+  192  data
+   96  ├── dir
+   96  │   ├── subdir
+    4  │   │   └── foobar
+    4  │   └── foo
+    4  ├── bar
+1.00G  ├── large-file
+   96  └── dir2
+    4      └── foo
+"""
+    assert out == expected
+
+    show_tree(entries, with_color=False, with_hash=True)
+    out, _ = capsys.readouterr()
+    expected = """\
+3fb071066d5d5b282f56a0169340346d.dir  data
+-                                     ├── dir
+-                                     │   ├── subdir
+d3b07384d113edec49eaa6238ad5ff00      │   │   └── foobar
+-                                     │   └── foo
+c157a79031e1c40f85931829bc5fc552      ├── bar
+-                                     ├── large-file
+-                                     └── dir2
+d3b07384d113edec49eaa6238ad5ff00          └── foo
+"""
+    assert out == expected
+
+    show_tree(entries, with_color=False, with_hash=True, with_size=True)
+    out, _ = capsys.readouterr()
+    expected = """\
+  192  3fb071066d5d5b282f56a0169340346d.dir  data
+   96  -                                     ├── dir
+   96  -                                     │   ├── subdir
+    4  d3b07384d113edec49eaa6238ad5ff00      │   │   └── foobar
+    4  -                                     │   └── foo
+    4  c157a79031e1c40f85931829bc5fc552      ├── bar
+1.00G  -                                     ├── large-file
+   96  -                                     └── dir2
+    4  d3b07384d113edec49eaa6238ad5ff00          └── foo
+"""
+    assert out == expected
+
+
 def test_list_alias():
     cli_args = parse_args(["ls", "local_dir"])
     assert cli_args.func == CmdList

tests/unit/command/test_dag.py

@@ -133,7 +133,7 @@ def test_show_dot(repo):
         "\"stage: 'b.dvc'\" -> \"stage: '2'\";",
         "\"stage: 'b.dvc'\" -> \"stage: '3'\";",
         "\"stage: 'b.dvc'\";",
-        "strict digraph  {",
+        "strict digraph {",
         "}",
     ]
     actual = sorted(line.rstrip() for line in _show_dot(repo.index.graph).splitlines())
@@ -153,7 +153,7 @@ def test_show_dot_properly_escapes():
     )
 
     expected = {
-        "strict digraph  {",
+        "strict digraph {",
         '"data\\raw\\2.dvc";',
         '"prepare";',
         '"4";',

tests/unit/command/test_dataset.py

@@ -14,9 +14,9 @@ from dvc.cli import main
             "Adding ds (url:/path @ main)\n",
         ),
         (
-            {"name": "mydataset", "url": "dvcx://dataset", "type": "dvcx"},
+            {"name": "mydataset", "url": "dc://dataset", "type": "dc"},
             {"version": 1, "created_at": datetime.now(tz=timezone.utc)},
-            "Adding mydataset (dvcx://dataset @ v1)\n",
+            "Adding mydataset (dc://dataset @ v1)\n",
         ),
         (
             {"name": "mydataset", "url": "s3://bucket/path", "type": "url"},
@@ -45,7 +45,7 @@ def test_add_already_exists(dvc, caplog, mocker):
     dataset = dvc.datasets._build_dataset("dvc.yaml", spec, None)
     mocker.patch("dvc.repo.datasets.Datasets.get", return_value=dataset)
 
-    assert main(["dataset", "add", "ds", "--dvcx", "dataset"]) == 255
+    assert main(["dataset", "add", "ds", "--dc", "dataset"]) == 255
     assert "ds already exists in dvc.yaml, use the --force to overwrite" in caplog.text
 
 
@@ -64,21 +64,21 @@ def test_add_already_exists(dvc, caplog, mocker):
             },
         ),
         (
-            {"name": "mydataset", "url": "dvcx://dataset", "type": "dvcx"},
+            {"name": "mydataset", "url": "dc://dataset", "type": "dc"},
             {"version": 1, "created_at": datetime.now(tz=timezone.utc)},
             {"version": 2},
             {
-                "missing": "Updating mydataset (dvcx://dataset @ v2)\n",
+                "missing": "Updating mydataset (dc://dataset @ v2)\n",
                 "unchanged": "Nothing to update\n",
                 "updated": "Updating mydataset (v1 -> v2)\n",
             },
         ),
         (
-            {"name": "mydataset", "url": "dvcx://dataset", "type": "dvcx"},
+            {"name": "mydataset", "url": "dc://dataset", "type": "dc"},
             {"version": 2, "created_at": datetime.now(tz=timezone.utc)},
             {"version": 1},
             {
-                "missing": "Updating mydataset (dvcx://dataset @ v1)\n",
+                "missing": "Updating mydataset (dc://dataset @ v1)\n",
                 "unchanged": "Nothing to update\n",
                 "updated": "Downgrading mydataset (v2 -> v1)\n",
             },
@@ -106,7 +106,8 @@ def test_add_already_exists(dvc, caplog, mocker):
                 "missing": "Updating mydataset (s3://bucket/path)\n",
                 "unchanged": "Nothing to update\n",
                 "updated": (
-                    "Updating mydataset (s3://bucket/path)\n"
+                    r"Updating mydataset (s3://bucket/path)"
+                    "\n"
                     "M\tbaz\n"
                     "A\tfoobar\n"
                     "D\tfoo\n"

tests/unit/command/test_experiments.py

@@ -384,6 +384,7 @@ def test_experiments_remove_flag(dvc, scm, mocker, capsys, caplog):
         num=2,
         queue=False,
         git_remote="myremote",
+        keep=False,
     )
 
 
@@ -410,6 +411,7 @@ def test_experiments_remove_special(dvc, scm, mocker, capsys, caplog):
         num=1,
         queue=False,
         git_remote="myremote",
+        keep=False,
     )
 
 

tests/unit/command/test_ls_url.py

@@ -1,6 +1,7 @@
 from dvc.cli import parse_args
 from dvc.commands.ls_url import CmdListUrl
 from dvc.config import Config
+from dvc.fs import LocalFileSystem
 
 
 def test_ls_url(mocker, M):
@@ -12,12 +13,16 @@ def test_ls_url(mocker, M):
     assert cmd.run() == 0
 
     m.assert_called_once_with(
-        "src", recursive=False, fs_config=None, config=M.instance_of(Config)
+        "src",
+        recursive=False,
+        maxdepth=None,
+        fs_config=None,
+        config=M.instance_of(Config),
     )
 
 
 def test_recursive(mocker, M):
-    cli_args = parse_args(["ls-url", "-R", "src"])
+    cli_args = parse_args(["ls-url", "-R", "-L", "2", "src"])
     assert cli_args.func == CmdListUrl
     cmd = cli_args.func(cli_args)
     m = mocker.patch("dvc.repo.Repo.ls_url", autospec=True)
@@ -25,5 +30,16 @@ def test_recursive(mocker, M):
     assert cmd.run() == 0
 
     m.assert_called_once_with(
-        "src", recursive=True, fs_config=None, config=M.instance_of(Config)
+        "src", recursive=True, maxdepth=2, fs_config=None, config=M.instance_of(Config)
     )
+
+
+def test_tree(mocker, M):
+    cli_args = parse_args(["ls-url", "--tree", "--level", "2", "src"])
+    assert cli_args.func == CmdListUrl
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch("dvc.repo.ls._ls_tree", autospec=True)
+
+    assert cmd.run() == 0
+
+    m.assert_called_once_with(M.instance_of(LocalFileSystem), "src", maxdepth=2)

tests/unit/fs/test_dvc_info.py

@@ -123,6 +123,6 @@ def test_info_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, dvcfs):
 
     for path in ["subrepo", "subrepo/foo", "subrepo/foobar"]:
         info = dvcfs.info(path)
-        assert info["repo"].root_dir == str(
-            subrepo
-        ), f"repo root didn't match for {path}"
+        assert info["repo"].root_dir == str(subrepo), (
+            f"repo root didn't match for {path}"
+        )

tests/unit/output/test_load.py

@@ -99,7 +99,7 @@ def test_load_from_pipeline_error_on_typ(dvc, typ):
         output.load_from_pipeline(Stage(dvc), ["file1"], typ)
 
 
-@pytest.mark.parametrize("key", [3, "list".split()])
+@pytest.mark.parametrize("key", [3, ["list"]])
 def test_load_from_pipeline_illegal_type(dvc, key):
     stage = Stage(dvc)
     with pytest.raises(ValueError, match=f"'{type(key).__name__}' not supported."):

tests/unit/test_config.py

@@ -130,6 +130,5 @@ def test_feature_section_supports_arbitrary_values(caplog):
     assert "random_key_1" not in data
     assert "random_key_2" not in data
     assert (
-        "'feature.random_key_1', 'feature.random_key_2' "
-        "config options are unsupported"
+        "'feature.random_key_1', 'feature.random_key_2' config options are unsupported"
     ) in caplog.text

tests/unit/test_context.py

@@ -60,7 +60,7 @@ def test_context_list():
         _ = context["lst"][3]
 
     context["lst"].insert(0, "baz")
-    assert context["lst"] == CtxList(["baz"] + lst[:2])
+    assert context["lst"] == CtxList(["baz", *lst[:2]])
 
 
 def test_context_setitem_getitem():

tests/unit/test_dirs.py

@@ -1,4 +1,8 @@
-from dvc.dirs import global_config_dir
+import sys
+
+import pytest
+
+from dvc.dirs import global_config_dir, site_cache_dir
 from dvc.env import DVC_GLOBAL_CONFIG_DIR
 
 
@@ -6,3 +10,8 @@ def test_global_config_dir_respects_env_var(monkeypatch):
     path = "/some/random/path"
     monkeypatch.setenv(DVC_GLOBAL_CONFIG_DIR, path)
     assert global_config_dir() == path
+
+
+@pytest.mark.skipif(sys.platform != "linux", reason="Only for Unix platforms")
+def test_site_cache_dir_on_unix():
+    assert site_cache_dir() == "/var/tmp/dvc"

tests/unit/test_ignore.py

@@ -168,8 +168,9 @@ def test_match_ignore_from_file(
 ):
     from dvc.fs import localfs
 
+    root = r"\\" if os.name == "nt" else "/"
     dvcignore_path = os.path.join(
-        os.path.sep, "full", "path", "to", "ignore", "file", ".dvcignore"
+        root, "full", "path", "to", "ignore", "file", ".dvcignore"
     )
     dvcignore_dirname = os.path.dirname(dvcignore_path)
 

tests/unit/test_logger.py

@@ -1,7 +1,5 @@
 import logging
-import time
 import traceback
-from datetime import datetime
 
 import colorama
 import pytest
@@ -22,11 +20,9 @@ colors = {
 
 @pytest.fixture
 def dt(mocker):
-    mocker.patch(
-        "time.time",
-        return_value=time.mktime(datetime(2020, 2, 2).timetuple()),  # noqa: DTZ001
-    )
-    return "2020-02-02 00:00:00,000"
+    dt_str = "2020-02-02 00:00:00,000"
+    mocker.patch.object(formatter, "formatTime", return_value=dt_str)
+    return dt_str
 
 
 class TestColorFormatter:
@@ -104,9 +100,7 @@ class TestColorFormatter:
                 logger.exception("")
 
             expected = (
-                "{red}{datetime}{nc} "
-                "{red}ERROR{nc}: description\n"
-                "{stack_trace}".format(
+                "{red}{datetime}{nc} {red}ERROR{nc}: description\n{stack_trace}".format(
                     stack_trace=stack_trace,
                     **colors,
                     datetime=dt,
@@ -144,9 +138,7 @@ class TestColorFormatter:
                 logger.exception("something", extra={"tb_only": True})
 
             expected = (
-                "{red}{datetime}{nc} "
-                "{red}ERROR{nc}: something\n"
-                "{stack_trace}".format(
+                "{red}{datetime}{nc} {red}ERROR{nc}: something\n{stack_trace}".format(
                     stack_trace=stack_trace,
                     **colors,
                     datetime=dt,

tests/unit/ui/test_pager.py

@@ -1,3 +1,5 @@
+from pydoc import plainpager
+
 import pytest
 
 from dvc.env import DVC_PAGER
@@ -70,8 +72,8 @@ def test_dvc_sets_some_options_on_less_if_less_env_defined(mocker, monkeypatch,
     assert find_pager() == "less --RAW-CONTROL-CHARS --chop-long-lines"
 
 
-def test_make_pager_when_no_pager_found(mocker, monkeypatch):
-    assert make_pager(None).__name__ == "plainpager"
+def test_make_pager_when_no_pager_found():
+    assert make_pager(None) is plainpager
 
 
 def test_pager(mocker, monkeypatch):