exp: Fix --rev args for ls pull and remove.

Closes #9471

(cherry picked from commit 3ddd4b87fc)

.dvc/.gitignore

@@ -1,3 +1,12 @@
+/state
+/lock
 /config.local
-/tmp
+/updater
+/updater.lock
+/state-journal
+/state-wal
 /cache
+/pkg
+/repos
+/tmp
+/experiments

.dvc/config

@@ -0,0 +1,4 @@
+['remote "dvc"']
+url = https://remote.dvc.org/dvc
+[core]
+remote = dvc

.flake8

@@ -0,0 +1,19 @@
+[flake8]
+ignore=
+    # Whitespace before ':'
+    E203
+    # Too many leading '#' for block comment
+    E266
+    # Line break occurred before a binary operator
+    W503
+    # unindexed parameters in the str.format, see:
+    # https://pypi.org/project/flake8-string-format/
+    P1
+    # use `!r` instead of surrounding by quotes
+    B028
+max_line_length = 88
+max-complexity = 10
+select = B,C,E,F,W,T4,B902,T,P,TYP
+show_source = true
+count = true
+min_python_version = 3.8.0

.github/ISSUE_TEMPLATE/bug_report.md

@@ -57,7 +57,7 @@ $ dvc doctor
 **Additional Information (if any):**
 
 <!--
-Please check https://github.com/treeverse/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
+Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
 
 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
 If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.

.github/ISSUE_TEMPLATE/epic_story.md

@@ -0,0 +1,60 @@
+---
+name: Epic/Story Issue Template
+about: Template for "top" level issues - Epics (>2 weeks) / Stories (<2 weeks)
+title: 'Epic: New Feature'
+labels: epic
+assignees:
+
+---
+
+## Summary / Background
+What do you want to achieve, why? business context
+...
+
+## Scope
+
+What will be impacted and what won't be?
+What needs implementation and what is invariant?
+e.g.
+- user should be able to run workflow X from UI
+- enable workflow Y, Z from CLI
+
+## Assumptions
+Product / UX assumptions as well as technical assumptions / limitations
+e.g.
+* Support only Python Runtime
+* Focus on DVC experiments only
+* Deployment environments don't change often and can be picked up from shared configuration
+
+## Open Questions
+e.g.
+- How should access control work for shared artifacts (workflow X)
+- Python runtime assumption - is it really valid? in light of <...>
+
+## Blockers / Dependencies
+List issues or other conditions / blockers
+
+## General Approach
+Invocation example:
+```shell
+$ mapper-run task.tar.gz --ray-cluster <ip>:<port>
+```
+
+## Steps
+
+### Must have (p1)
+- [ ] subissue2
+- [ ] step 2
+ - info
+ - info
+
+### Optional / followup (p2)
+- [ ] ⌛ step 3 wip
+- [ ] step 4
+
+## Timelines
+
+Put your estimations here. Update once certainty changes
+- end of week (Feb 3) for prototype with workflows X, Y
+- Feb 15 - MVP in production
+- Low priority followups can be done later

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,5 @@
 * [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
 
-* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/treeverse/dvc.org) and linked it here.
+* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
 
 Thank you for the contribution - we'll try to review it as soon as possible. 🙏

.github/mergify.yml

@@ -0,0 +1,9 @@
+pull_request_rules:
+  - name: backport patches to 2.x
+    conditions:
+      - base=main
+      - label=backport
+    actions:
+      backport:
+        branches:
+          - "2.x"

.github/workflows/benchmarks.yaml

@@ -5,40 +5,14 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
-env:
-  FORCE_COLOR: "1"
-  PY_COLORS: "1"
-
 jobs:
   bench:
     name: run benchmarks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-python@v7
+      - uses: actions/setup-python@v4
         with:
-            python-version: "3.14"
-
-      - uses: actions/checkout@v7
+            python-version: "3.11"
+      - uses: iterative/dvc-bench@main
         with:
-          ref: ${{ github.event.pull_request.base.sha }}
-          fetch-depth: 0
-
-      - uses: astral-sh/setup-uv@v7
-        with:
-          enable-cache: true
-          cache-dependency-glob: pyproject.toml
-
-      - run: uv pip install '.[tests]' --system
-      - name: run benchmarks on base branch
-        run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
-
-      - uses: actions/checkout@v7
-        with:
-          fetch-depth: 0
-          clean: false
-      - run: uv pip install '.[tests]' --system
-      - name: run benchmarks for PR
-        run: >
-          pytest --benchmark-compare --benchmark-compare-fail=min:5%
-          --benchmark-group-by name
-          dvc/testing/benchmarks/ -k 'test_init or test_help'
+            pytest_options: "-k 'test_init or test_help'"

.github/workflows/build.yaml

@@ -1,95 +0,0 @@
-name: Build and upload package
-on:
-  push:
-    branches:
-    - main
-  release:
-    types:
-    - published
-  workflow_dispatch:
-
-env:
-  FORCE_COLOR: 1
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    name: Build & Verify package
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v7
-      with:
-        fetch-depth: 0
-
-    - uses: actions/setup-python@v7
-      with:
-        python-version: "3.14"
-
-    - uses: astral-sh/setup-uv@v7
-
-    - name: Omit local version for Test PyPI upload
-      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-      run: echo SETUPTOOLS_SCM_OVERRIDES_FOR_DVC='{local_scheme="no-local-version"}' >> $GITHUB_ENV
-
-    - name: Build Python Package
-      run: |
-        echo 'PKG = "pip"'>dvc/_build.py
-        uv build
-
-    - name: Check dist
-      run: uv tool run twine check --strict dist/*
-
-    - uses: actions/upload-artifact@v7
-      with:
-        name: Packages
-        path: dist/
-        if-no-files-found: error
-
-  test-pypi-publish:
-    name: Publish dev package to test.pypi.org
-    runs-on: ubuntu-latest
-    needs: build
-    if: ${{ github.event.action == 'published' || (github.repository == 'treeverse/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
-
-    environment:
-      name: test-pypi
-      url: https://test.pypi.org/p/dvc/${{ github.event.release.tag_name }}
-
-    permissions:
-      id-token: write
-
-    steps:
-    - uses: actions/download-artifact@v8
-      with:
-        name: Packages
-        path: dist
-
-    - name: Upload package to Test PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        repository-url: https://test.pypi.org/legacy/
-        skip-existing: true
-
-  pypi-publish:
-    name: Publish released package to pypi.org
-    runs-on: ubuntu-latest
-    needs: build
-    if: github.event.action == 'published'
-
-    environment:
-      name: pypi
-      url: https://pypi.org/p/dvc/${{ github.event.release.tag_name }}
-
-    permissions:
-      id-token: write
-
-    steps:
-    - uses: actions/download-artifact@v8
-      with:
-        name: Packages
-        path: dist
-
-    - name: Upload package to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/codeql.yml

@@ -2,7 +2,7 @@ name: "CodeQL"
 
 on:
   push:
-    branches: ["main"]
+    branches: ["main", "2.x"]
   pull_request:
     branches: ["main"]
   schedule:
@@ -28,13 +28,12 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v7
+      uses: actions/checkout@v3
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v4.38.2
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
-        queries: security-extended
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v4.38.2
+      uses: github/codeql-action/analyze@v2

.github/workflows/packages.yaml

@@ -0,0 +1,111 @@
+name: Build packages
+on:
+  push:
+    branches:
+    - main
+  release:
+    types: [released, prereleased]
+  workflow_dispatch:
+
+permissions:  # added using https://github.com/step-security/secure-workflows
+  contents: read
+
+jobs:
+  binary:
+    permissions:
+      contents: write  # for actions/upload-release-asset to upload release asset
+    strategy:
+      matrix:
+        include:
+         - {pkg: "deb",  os: "ubuntu-20.04", asset: "dvc_${{ github.event.release.tag_name }}_amd64.deb"}
+         - {pkg: "rpm", os: "ubuntu-20.04", asset: "dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm"}
+         - {pkg: "osxpkg", os: "macos-11", asset: "dvc-${{ github.event.release.tag_name }}.pkg"}
+         - {pkg: "exe",  os: "windows-2019", asset: "dvc-${{ github.event.release.tag_name }}.exe"}
+
+    name: ${{ matrix.pkg }}
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+      with:
+          fetch-depth: 0
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+        cache: pip
+        cache-dependency-path: |
+          pyproject.toml
+          scripts/build-requirements.txt
+
+    - name: Set up Ruby 2.6
+      uses: ruby/setup-ruby@v1
+      if: matrix.pkg != 'exe'
+      with:
+        ruby-version: '2.6'
+
+    - name: Install fpm
+      if: matrix.pkg != 'exe'
+      run: gem install --no-document fpm
+
+    - name: Install deps
+      run: |
+        pip install --upgrade pip wheel setuptools
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+
+    - name: Pull images
+      run: dvc pull
+
+    - name: Build ${{ matrix.pkg }}
+      run: python scripts/build.py ${{ matrix.pkg }}
+
+    - name: Publish ${{ matrix.pkg }}
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/${{ matrix.pkg == 'exe' && 'innosetup' || 'fpm' }}/${{ matrix.asset }}
+        asset_name: ${{ matrix.asset }}
+        asset_content_type: binary/octet-stream
+
+  pip:
+    runs-on: ubuntu-20.04
+    permissions:
+      id-token: write
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.8
+
+    - name: Install
+      run: pip install --upgrade pip wheel
+
+    - name: Force version for Test PyPI uploads
+      if: ${{ !startsWith(github.ref, 'refs/tags') }}
+      run: |
+        pip install setuptools_scm
+        echo version=$(python -m setuptools_scm | awk -F+ '{print $1}' | tail -1) >> $GITHUB_ENV
+
+    - name: Build packages
+      run: ./scripts/build_package.sh
+      env:
+        SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.version }}
+
+    - name: Publish packages to PyPI
+      if: github.event_name == 'release'
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+    - name: Publish to Test PyPI
+      if: ${{ github.event_name == 'release' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository_url: https://test.pypi.org/legacy/
+        skip_existing: true

.github/workflows/plugin_tests.yaml

@@ -18,36 +18,35 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        pyv: ["3.14"]
+        os: [ubuntu-20.04]
+        pyv: ["3.10"]
         plugin: ["dvc-s3"]
 
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v3
       with:
         path: dvc
 
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v3
       with:
-        repository: treeverse/${{ matrix.plugin }}
+        repository: iterative/${{ matrix.plugin }}
         ref: main
         path: ${{ matrix.plugin }}
 
     - name: Set up Python
-      uses: actions/setup-python@v7
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.pyv }}
-
-    - uses: astral-sh/setup-uv@v7
-      with:
-        enable-cache: true
-        cache-dependency-glob: "**/pyproject.toml"
-        cache-suffix: ${{ matrix.pyv }}
+        cache: pip
+        cache-dependency-path: |
+          dvc/pyproject.toml
+          ${{ matrix.plugin }}/setup.cfg
 
     - name: Install plugin + DVC@PR
       run: |
-        uv pip install "./dvc[testing]" --system
-        uv pip install -e "./${{ matrix.plugin }}[tests]" --system
+        pip install --upgrade pip wheel
+        pip install "./dvc[testing]"
+        pip install -e "./${{ matrix.plugin }}[tests]"
 
     - name: Run plugin tests
       timeout-minutes: 15

.github/workflows/tests.yaml

@@ -2,7 +2,7 @@ name: Tests
 
 on:
   push:
-    branches: [main]
+    branches: [main, 2.x]
   pull_request:
   schedule:
     - cron: '5 1 * * *'  # every day at 01:05
@@ -16,146 +16,42 @@ permissions:
   contents: read
 
 jobs:
-  lint:
-    timeout-minutes: 10
-    runs-on: ${{ matrix.os }}
-    env:
-       DVC_NO_ANALYTICS: true
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-    steps:
-    - uses: actions/checkout@v7
-      with:
-        fetch-depth: 0
-    - name: Set up Python
-      uses: actions/setup-python@v7
-      with:
-        python-version: '3.14'
-    - uses: astral-sh/setup-uv@v7
-      with:
-        enable-cache: true
-        cache-dependency-glob: pyproject.toml
-    - run: uv pip install -e ".[dev]" --system
-      env:
-        UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
-
-    - name: Cache mypy
-      uses: actions/cache@v6
-      with:
-        path: .mypy_cache
-        key: mypy-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
-    - uses: pre-commit/[email protected]
-
   tests:
-    timeout-minutes: 20
+    timeout-minutes: 50
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
-        pyv: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
-        include:
-        - os: windows-latest
-          pyv: "3.9"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
-        - os: windows-latest
-          pyv: "3.9"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
-        - os: windows-latest
-          pyv: "3.9"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
-        - os: windows-latest
-          pyv: "3.9"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
-        - os: windows-latest
-          pyv: "3.10"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
-        - os: windows-latest
-          pyv: "3.10"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
-        - os: windows-latest
-          pyv: "3.10"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
-        - os: windows-latest
-          pyv: "3.10"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
-        - os: windows-latest
-          pyv: "3.11"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
-        - os: windows-latest
-          pyv: "3.11"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
-        - os: windows-latest
-          pyv: "3.11"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
-        - os: windows-latest
-          pyv: "3.11"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
-        - os: windows-latest
-          pyv: "3.12"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
-        - os: windows-latest
-          pyv: "3.12"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
-        - os: windows-latest
-          pyv: "3.12"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
-        - os: windows-latest
-          pyv: "3.12"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
-        - os: windows-latest
-          pyv: "3.13"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
-        - os: windows-latest
-          pyv: "3.13"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
-        - os: windows-latest
-          pyv: "3.13"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
-        - os: windows-latest
-          pyv: "3.13"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
-        - os: windows-latest
-          pyv: "3.14"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
-        - os: windows-latest
-          pyv: "3.14"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
-        - os: windows-latest
-          pyv: "3.14"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
-        - os: windows-latest
-          pyv: "3.14"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        os: [ubuntu-20.04, windows-latest, macos-latest]
+        pyv: ["3.8", "3.9", "3.10", "3.11"]
+
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - name: Set up Python
-      uses: actions/setup-python@v7
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.pyv }}
-    - uses: astral-sh/setup-uv@v7
-      with:
-        enable-cache: true
-        cache-dependency-glob: pyproject.toml
-        cache-suffix: ${{ matrix.pyv }}
-    - run: uv pip install -e ".[dev]" pytest-split --system
-      env:
-        UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
-
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: install
+      run: |
+        pip install --upgrade pip wheel
+        pip install -e ".[dev]"
+    - uses: pre-commit/[email protected]
     - name: run tests
       timeout-minutes: 40
       env:
-        COVERAGE_CORE: sysmon
         PYTHONUTF8: 1
       run: >
-        pytest ${{ matrix.pytestargs }} -n=logical --dist=worksteal --timeout=300 --durations=0
-        --cov --cov-report=xml --cov-report=term  --durations-path=./.github/.test_durations
+        pytest -n=logical --timeout=300 --durations=100
+        --cov --cov-report=xml --cov-report=term
     - name: upload coverage report
-      uses: codecov/codecov-action@v7
+      uses: codecov/codecov-action@v3
+      with:
+        file: ./coverage.xml
+        fail_ci_if_error: false
   check:
     if: always()
     needs: [tests]
@@ -170,7 +66,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Slack Notification
-      uses: rtCamp/action-slack-notify@v2.4.0
+      uses: rtCamp/action-slack-notify@v2.2.0
       env:
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_COLOR: ${{ job.status }}

.gitignore

@@ -43,7 +43,7 @@ scripts/ci/gcp-creds.json
 .gcp-creds.json
 
 *~
-/dvc/_version.py
+/dvc/_dvc_version.py
 
 .mypy_cache/
 .pytest_cache/

.pre-commit-config.yaml

@@ -2,11 +2,11 @@ default_language_version:
   python: python3
 
 ci:
-  skip: [mypy, dvc-pre-commit]
+  skip: [mypy, pylint, dvc-pre-commit]
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
+    rev: v4.4.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -17,21 +17,57 @@ repos:
         args: ['--assume-in-merge']
       - id: check-toml
       - id: check-yaml
+      - id: debug-statements
+        exclude: "dvc/_debug.py"
       - id: end-of-file-fixer
       - id: mixed-line-ending
       - id: sort-simple-yaml
       - id: trailing-whitespace
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.15.20'
+  - repo: https://github.com/lovesegfault/beautysh
+    rev: v6.2.1
     hooks:
-      - id: ruff-check
-        args: [--fix, --exit-non-zero-on-fix]
-      - id: ruff-format
+      - id: beautysh
+        args: ["-i", "2"]
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.3.1
+    hooks:
+      - id: pyupgrade
+        args:
+          - --py38-plus
+  - repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
+      - id: black
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: 'v0.0.261'
+    hooks:
+      - id: ruff
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.2
+    rev: v2.2.4
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-bugbear
+          - flake8-comprehensions
+          - flake8-debugger
+          - flake8-string-format
+          - flake8-typing-imports
+  - repo: https://github.com/PyCQA/bandit
+    rev: '1.7.5'
+    hooks:
+    - id: bandit
+      args: ["-c", "pyproject.toml", "--quiet", "--format=custom"]
+      files: ^dvc/
+      additional_dependencies: [".[toml]"]
   - repo: local
     hooks:
     - id: mypy
@@ -41,6 +77,13 @@ repos:
       language: system
       types: [python]
       require_serial: true
+    - id: pylint
+      name: pylint
+      entry: pylint
+      files: ^dvc/
+      language: system
+      types: [python]
+      require_serial: true
   - hooks:
       - args:
           - git-hook
@@ -50,7 +93,7 @@ repos:
         language: system
         name: DVC pre-commit
         stages:
-          - pre-commit
+          - commit
         verbose: true
         require_serial: true
       - args:
@@ -61,7 +104,7 @@ repos:
         language: system
         name: DVC pre-push
         stages:
-          - pre-push
+          - push
         require_serial: true
       - always_run: true
         args:

.pre-commit-hooks.yaml

@@ -8,7 +8,7 @@
   name: DVC pre-commit
   require_serial: true
   stages:
-  - pre-commit
+  - commit
   verbose: true
 - args:
   - git-hook
@@ -20,7 +20,7 @@
   name: DVC pre-push
   require_serial: true
   stages:
-  - pre-push
+  - push
 - always_run: true
   args:
   - git-hook
@@ -29,7 +29,7 @@
   id: dvc-post-checkout
   language: python
   language_version: python3
-  minimum_pre_commit_version: 3.2.0
+  minimum_pre_commit_version: 2.2.0
   name: DVC post-checkout
   require_serial: true
   stages:

.zenodo.json

@@ -4,5 +4,5 @@
     "data-science", "data-version-control", "machine-learning", "git",
     "developer-tools", "reproducibility", "collaboration", "ai", "python"],
   "contributors": [
-    {"name": "DVC team", "type": "Other", "affiliation": "Treeverse"}]
+    {"name": "DVC team", "type": "Other", "affiliation": "Iterative"}]
 }

CITATION.cff

@@ -1,25 +0,0 @@
-cff-version: 1.2.0
-title: 'DVC: Data Version Control - Git for Data & Models"'
-message: >-
-  If you use this software as part of a publication and wish
-  to cite it, please use the metadata from this file.
-type: software
-authors:
-  - name: The DVC team and contributors
-    website: 'https://github.com/treeverse'
-repository-code: 'https://github.com/treeverse/dvc'
-url: 'https://dvc.org'
-abstract: >-
-  DVC is a tool for data science that takes advantage of existing software engineering toolset.
-  It helps machine learning teams manage large datasets, make projects reproducible, and collaborate better.
-keywords:
-  - ai
-  - collaboration
-  - data-science
-  - data-version-control
-  - developer-tools
-  - git
-  - machine-learning
-  - python
-  - reproducibility
-license: Apache-2.0

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2025 Treeverse.
+   Copyright 2017-2021 Iterative, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.rst

@@ -90,8 +90,8 @@ Their results can be filtered and compared based on hyperparameters and metrics,
 
 .. _`VS Code Extension`:
 
-VS Code Extension
-=================
+Visual Studio Code Extension
+============================
 
 |VS Code|
 
@@ -167,7 +167,7 @@ To install the development version, run:
 
 .. code-block:: bash
 
-   pip install "dvc @ git+https://github.com/treeverse/dvc"
+   pip install git+git://github.com/iterative/dvc
 
 Package (Platform-specific)
 ---------------------------
@@ -175,7 +175,7 @@ Package (Platform-specific)
 |Packages|
 
 Self-contained packages for Linux, Windows, and Mac are available.
-The latest version of the packages can be found on the GitHub `releases page <https://github.com/treeverse/dvc/releases>`_.
+The latest version of the packages can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
 Ubuntu / Debian (deb)
 ^^^^^^^^^^^^^^^^^^^^^
@@ -198,6 +198,8 @@ Fedora / CentOS (rpm)
 Contributing
 ============
 
+|Maintainability|
+
 Contributions are welcome!
 Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more details.
 Thanks to all our contributors!
@@ -211,7 +213,7 @@ Community and Support
 * `Forum <https://discuss.dvc.org/>`_
 * `Discord Chat <https://dvc.org/chat>`_
 * `Email <mailto:[email protected]>`_
-* `Mailing List <https://dvc.org/community#subscribe>`_
+* `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
 Copyright
 =========
@@ -235,19 +237,23 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://dvc.org
    :alt: DVC logo
 
-.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/treeverse/vscode-dvc/main/extension/docs/overview.gif
+.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/iterative/vscode-dvc/main/extension/docs/overview.gif
    :alt: DVC Extension for VS Code
 
-.. |CI| image:: https://github.com/treeverse/dvc/actions/workflows/tests.yaml/badge.svg
-   :target: https://github.com/treeverse/dvc/actions/workflows/tests.yaml
+.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=main
+   :target: https://github.com/iterative/dvc/actions
    :alt: GHA Tests
 
+.. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
+   :target: https://codeclimate.com/github/iterative/dvc
+   :alt: Code Climate
+
 .. |Python Version| image:: https://img.shields.io/pypi/pyversions/dvc
    :target: https://pypi.org/project/dvc
    :alt: Python Version
 
-.. |Coverage| image:: https://codecov.io/gh/treeverse/dvc/branch/main/graph/badge.svg
-   :target: https://codecov.io/gh/treeverse/dvc
+.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/main/graph/badge.svg
+   :target: https://codecov.io/gh/iterative/dvc
    :alt: Codecov
 
 .. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
@@ -262,7 +268,7 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://formulae.brew.sh/formula/dvc
    :alt: Homebrew
 
-.. |Conda| image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
+.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/dvc.svg?label=conda&logo=conda-forge
    :target: https://anaconda.org/conda-forge/dvc
    :alt: Conda-forge
 
@@ -274,8 +280,8 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://pypi.org/project/dvc
    :alt: PyPI Downloads
 
-.. |Packages| image:: https://img.shields.io/badge/deb|pkg|rpm|exe-blue
-   :target: https://dvc.org/doc/install
+.. |Packages| image:: https://img.shields.io/github/v/release/iterative/dvc?label=deb|pkg|rpm|exe&logo=GitHub
+   :target: https://github.com/iterative/dvc/releases/latest
    :alt: deb|pkg|rpm|exe
 
 .. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.3677553-blue.svg
@@ -286,8 +292,8 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://dvc.org/img/flow.gif
    :alt: how_dvc_works
 
-.. |Contribs| image:: https://contrib.rocks/image?repo=treeverse/dvc
-   :target: https://github.com/treeverse/dvc/graphs/contributors
+.. |Contribs| image:: https://contrib.rocks/image?repo=iterative/dvc
+   :target: https://github.com/iterative/dvc/graphs/contributors
    :alt: Contributors
 
 .. |VS Code| image:: https://img.shields.io/visual-studio-marketplace/v/Iterative.dvc?color=blue&label=VSCode&logo=visualstudiocode&logoColor=blue

dvc/__init__.py

@@ -3,9 +3,7 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-
 import dvc.logger
-from dvc.build import PKG  # noqa: F401
 from dvc.version import __version__, version_tuple  # noqa: F401
 
 dvc.logger.setup()

dvc/__main__.py

@@ -1,5 +1,4 @@
 """Main entry point for DVC command line tool."""
-
 import sys
 
 from dvc.cli import main

dvc/__pyinstaller/__init__.py

@@ -1,5 +0,0 @@
-import os
-
-
-def get_hook_dirs():
-    return [os.path.dirname(__file__)]

dvc/__pyinstaller/hook-dvc.utils.flatten.py

@@ -1,4 +0,0 @@
-# ruff: noqa: N999
-from PyInstaller.utils.hooks import copy_metadata
-
-datas = copy_metadata("flatten-dict")

dvc/__pyinstaller/hook-pydrive2.py

@@ -1,4 +0,0 @@
-# ruff: noqa: N999
-from PyInstaller.utils.hooks import copy_metadata
-
-datas = copy_metadata("pydrive2")

dvc/_debug.py

@@ -14,7 +14,7 @@ def viztracer_profile(
     log_async: bool = False,
 ):
     try:
-        import viztracer  # ty: ignore[unresolved-import]
+        import viztracer  # pylint: disable=import-error
     except ImportError:
         print("Failed to run profiler, viztracer is not installed")  # noqa: T201
         yield
@@ -23,11 +23,10 @@ def viztracer_profile(
     tracer = viztracer.VizTracer(max_stack_depth=depth, log_async=log_async)
 
     tracer.start()
-    try:
-        yield
-    finally:
-        tracer.stop()
-        tracer.save(path() if callable(path) else path)
+    yield
+    tracer.stop()
+
+    tracer.save(path() if callable(path) else path)
 
 
 @contextmanager
@@ -37,7 +36,7 @@ def yappi_profile(
     separate_threads: Optional[bool] = False,
 ):
     try:
-        import yappi  # ty: ignore[unresolved-import]
+        import yappi  # pylint: disable=import-error
     except ImportError:
         print("Failed to run profiler, yappi is not installed")  # noqa: T201
         yield
@@ -46,40 +45,38 @@ def yappi_profile(
     yappi.set_clock_type("wall" if wall_clock else "cpu")
 
     yappi.start()
-    try:
-        yield
-    finally:
-        yappi.stop()
+    yield
+    yappi.stop()
 
-        threads = yappi.get_thread_stats()
-        stats = {}
-        if separate_threads:
-            for thread in threads:
-                ctx_id = thread.id
-                stats[ctx_id] = yappi.get_func_stats(ctx_id=ctx_id)
+    threads = yappi.get_thread_stats()
+    stats = {}
+    if separate_threads:
+        for thread in threads:
+            ctx_id = thread.id
+            stats[ctx_id] = yappi.get_func_stats(ctx_id=ctx_id)
+    else:
+        stats[None] = yappi.get_func_stats()
+
+    fpath = path() if callable(path) else path
+    for ctx_id, st in stats.items():
+        if fpath:
+            out = f"{fpath}-{ctx_id}" if ctx_id is not None else fpath
+            st.save(out, type="callgrind")
         else:
-            stats[None] = yappi.get_func_stats()
+            if ctx_id is not None:
+                print(f"\nThread {ctx_id}")  # noqa: T201
+            st.print_all()  # pylint:disable=no-member
+            if ctx_id is None:
+                threads.print_all()  # pylint:disable=no-member
 
-        fpath = path() if callable(path) else path
-        for ctx_id, st in stats.items():
-            if fpath:
-                out = f"{fpath}-{ctx_id}" if ctx_id is not None else fpath
-                st.save(out, type="callgrind")
-            else:
-                if ctx_id is not None:
-                    print(f"\nThread {ctx_id}")  # noqa: T201
-                st.print_all()
-                if ctx_id is None:
-                    threads.print_all()
-
-        yappi.clear_stats()
+    yappi.clear_stats()
 
 
 @contextmanager
 def instrument(html_output=False):
     """Run a statistical profiler"""
     try:
-        from pyinstrument import Profiler  # ty: ignore[unresolved-import]
+        from pyinstrument import Profiler  # pylint: disable=import-error
     except ImportError:
         print("Failed to run profiler, pyinstrument is not installed")  # noqa: T201
         yield
@@ -88,15 +85,13 @@ def instrument(html_output=False):
     profiler = Profiler()
 
     profiler.start()
-    try:
-        yield
-    finally:
-        profiler.stop()
+    yield
+    profiler.stop()
 
-        if html_output:
-            profiler.open_in_browser()
-        else:
-            print(profiler.output_text(unicode=True, color=True))  # noqa: T201
+    if html_output:
+        profiler.open_in_browser()
+        return
+    print(profiler.output_text(unicode=True, color=True))  # noqa: T201
 
 
 @contextmanager
@@ -107,23 +102,22 @@ def profile(dump_path: Optional[str] = None):
     prof = cProfile.Profile()
     prof.enable()
 
-    try:
-        yield
-    finally:
-        prof.disable()
-        if dump_path:
-            prof.dump_stats(dump_path)
-        else:
-            prof.print_stats(sort="cumtime")
+    yield
+
+    prof.disable()
+    if not dump_path:
+        prof.print_stats(sort="cumtime")
+        return
+    prof.dump_stats(dump_path)
 
 
 @contextmanager
 def debug():
     try:
         yield
-    except Exception:
+    except Exception:  # pylint: disable=broad-except
         try:
-            import ipdb as pdb  # noqa: T100  # ty: ignore[unresolved-import]
+            import ipdb as pdb  # noqa: T100, pylint: disable=import-error
         except ImportError:
             import pdb  # type: ignore[no-redef]  # noqa: T100
         pdb.post_mortem()
@@ -151,8 +145,7 @@ def show_stack():
         signal.signal(signal.SIGQUIT, _sigshow)
 
     try:
-        # only available on macOS
-        signal.signal(signal.SIGINFO, _sigshow)  # type: ignore[attr-defined]
+        signal.signal(signal.SIGINFO, _sigshow)  # only available on macOS
     except AttributeError:
         pass
     yield
@@ -162,7 +155,7 @@ def _get_path_func(tool: str, ext: str):
     fmt = f"{tool}.dvc-{{now:%Y%m%d}}_{{now:%H%M%S}}.{ext}"
 
     def func(now: Optional["datetime"] = None) -> str:
-        return fmt.format(now=now or datetime.now())  # noqa: DTZ005
+        return fmt.format(now=now or datetime.now())
 
     return func
 
@@ -205,84 +198,33 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs):
 def add_debugging_flags(parser):
     from argparse import SUPPRESS
 
-    # For detailed info see:
-    # https://github.com/treeverse/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
-    args, _ = parser.parse_known_args()
-    verbose = args.verbose
-
-    def debug_help(msg):
-        if verbose:
-            return msg
-        return SUPPRESS
-
-    parser = parser.add_argument_group("debug options")
-
-    parser.add_argument(
-        "--cprofile",
-        action="store_true",
-        default=False,
-        help=debug_help("Generate cprofile data for tools like snakeviz / tuna"),
-    )
-    parser.add_argument(
-        "--cprofile-dump", help=debug_help("Location to dump cprofile file")
-    )
-    parser.add_argument(
-        "--yappi",
-        action="store_true",
-        default=False,
-        help=debug_help(
-            "Generate a callgrind file for use with tools like "
-            "kcachegrind / qcachegrind"
-        ),
-    )
+    parser.add_argument("--cprofile", action="store_true", default=False, help=SUPPRESS)
+    parser.add_argument("--yappi", action="store_true", default=False, help=SUPPRESS)
     parser.add_argument(
         "--yappi-separate-threads",
         action="store_true",
         default=False,
-        help=debug_help("Generate one callgrind file per thread"),
+        help=SUPPRESS,
     )
     parser.add_argument(
-        "--viztracer",
-        action="store_true",
-        default=False,
-        help=debug_help("Generate a viztracer file for use with vizviewer"),
+        "--viztracer", action="store_true", default=False, help=SUPPRESS
+    )
+    parser.add_argument("--viztracer-depth", type=int, help=SUPPRESS)
+    parser.add_argument(
+        "--viztracer-async", action="store_true", default=False, help=SUPPRESS
+    )
+    parser.add_argument("--cprofile-dump", help=SUPPRESS)
+    parser.add_argument("--pdb", action="store_true", default=False, help=SUPPRESS)
+    parser.add_argument(
+        "--instrument", action="store_true", default=False, help=SUPPRESS
     )
     parser.add_argument(
-        "--viztracer-depth",
-        type=int,
-        help=debug_help("Set viztracer maximum stack depth"),
-    )
-    parser.add_argument(
-        "--viztracer-async",
-        action="store_true",
-        default=False,
-        help=debug_help("Treat async tasks as threads"),
-    )
-    parser.add_argument(
-        "--pdb",
-        action="store_true",
-        default=False,
-        help=debug_help("Drop into the pdb/ipdb debugger on any exception"),
-    )
-    parser.add_argument(
-        "--instrument",
-        action="store_true",
-        default=False,
-        help=debug_help("Use pyinstrument CLI profiler"),
-    )
-    parser.add_argument(
-        "--instrument-open",
-        action="store_true",
-        default=False,
-        help=debug_help("Use pyinstrument web profiler"),
+        "--instrument-open", action="store_true", default=False, help=SUPPRESS
     )
     parser.add_argument(
         "--show-stack",
         "--ss",
         action="store_true",
         default=False,
-        help=debug_help(
-            r"Use Ctrl+T on macOS or Ctrl+\ on Linux to print the stack "
-            "frame currently executing. Unavailable on Windows."
-        ),
+        help=SUPPRESS,
     )

dvc/analytics.py

@@ -1,15 +1,10 @@
 import json
+import logging
 import os
-from typing import TYPE_CHECKING, Optional
 
-from dvc.log import logger
+from .env import DVC_NO_ANALYTICS
 
-from .env import DVC_ANALYTICS_ENDPOINT, DVC_NO_ANALYTICS
-
-if TYPE_CHECKING:
-    from dvc.scm import Base
-
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 def collect_and_send_report(args=None, return_code=None):
@@ -39,8 +34,6 @@ def collect_and_send_report(args=None, return_code=None):
 
     with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
         json.dump(report, fobj)
-
-    logger.trace("Saving analytics report to %s", fobj.name)
     daemon(["analytics", fobj.name])
 
 
@@ -71,9 +64,8 @@ def send(path):
     thus, the need of removing such file afterwards.
     """
     import requests
-    from requests.exceptions import RequestException
 
-    url = os.environ.get(DVC_ANALYTICS_ENDPOINT, "https://analytics.dvc.org")
+    url = "https://analytics.dvc.org"
     headers = {"content-type": "application/json"}
 
     with open(path, encoding="utf-8") as fobj:
@@ -81,75 +73,28 @@ def send(path):
 
     report.update(_runtime_info())
 
-    logger.debug("uploading report to %s", url)
-    logger.trace("Sending %s to %s", report, url)
-
     try:
         requests.post(url, json=report, headers=headers, timeout=5)
-    except RequestException as e:
-        logger.trace("", exc_info=True)
-        logger.debug("failed to send analytics report %s", str(e))
+    except requests.exceptions.RequestException:
+        logger.debug("failed to send analytics report", exc_info=True)
 
-    logger.trace("removing report %s", path)
     os.remove(path)
 
 
-def _git_remote_url(scm: Optional["Base"]) -> Optional[str]:
-    from dvc.scm import Git
+def _scm_in_use():
+    from dvc.exceptions import NotDvcRepoError
+    from dvc.repo import Repo
+    from dvc.scm import NoSCM
 
-    if not isinstance(scm, Git):
-        return None
-
-    from dulwich.porcelain import get_remote_repo
-
-    dulwich_repo = scm.dulwich.repo
-    try:
-        _remote, url = get_remote_repo(dulwich_repo)
-    except IndexError:
-        # IndexError happens when the head is detached
-        _remote, url = get_remote_repo(dulwich_repo, b"origin")
-    # Dulwich returns (None, "origin") if no remote set
-    if (_remote, url) == (None, "origin"):
-        return None
-    return url
-
-
-def _scm_in_use(scm: Optional["Base"]) -> Optional[str]:
-    return type(scm).__name__ if scm else None
-
-
-def _parse_git_remote_path(remote_url: str) -> str:
-    from urllib.parse import urlparse
-
-    from scmrepo.urls import is_scp_style_url
-
-    parsed = urlparse(remote_url)
-    # Windows Path also gets parsed with a drive letter as scheme
-    # https://github.com/python/cpython/issues/86381
-    if parsed.scheme and parsed.scheme in ("http", "https", "git", "ssh"):
-        return parsed.path.strip("/")
-
-    if is_scp_style_url(remote_url):
-        # handle scp-style URL
-        parts = remote_url.split(":", 1)
-        if len(parts) == 2:
-            _, path = parts
-            return path.rstrip("/")
-    return remote_url
-
-
-def _git_remote_path_hash(scm: Optional["Base"]) -> Optional[str]:
-    """Return a hash of the git remote path."""
-    import hashlib
+    from .scm import SCM, SCMError
 
     try:
-        if remote_url := _git_remote_url(scm):
-            path = _parse_git_remote_path(remote_url)
-            h = hashlib.md5(path.encode("utf-8"), usedforsecurity=False)  # for FIPS
-            return h.hexdigest()
-    except Exception:
-        logger.debug("Failed to get git remote path", exc_info=True)
-    return None
+        scm = SCM(root_dir=Repo.find_root())
+        return type(scm).__name__
+    except SCMError:
+        return NoSCM.__name__
+    except NotDvcRepoError:
+        pass
 
 
 def _runtime_info():
@@ -159,8 +104,6 @@ def _runtime_info():
     from iterative_telemetry import _generate_ci_id, find_or_create_user_id
 
     from dvc import __version__
-    from dvc.info import _get_remotes
-    from dvc.repo import Repo
     from dvc.utils import is_binary
 
     ci_id = _generate_ci_id()
@@ -169,24 +112,13 @@ def _runtime_info():
     else:
         group_id, user_id = None, find_or_create_user_id()
 
-    scm = None
-    remotes = None
-    try:
-        repo = Repo()
-        scm = repo.scm
-        remotes = _get_remotes(repo.config)
-    except Exception as exc:  # noqa: BLE001
-        logger.debug("failed to open repo: %s", exc)
-
     return {
         "dvc_version": __version__,
         "is_binary": is_binary(),
-        "scm_class": _scm_in_use(scm),
+        "scm_class": _scm_in_use(),
         "system_info": _system_info(),
         "user_id": user_id,
         "group_id": group_id,
-        "remotes": remotes,
-        "git_remote_hash": _git_remote_path_hash(scm),
     }
 
 

dvc/annotations.py

@@ -1,5 +1,5 @@
 from dataclasses import asdict, dataclass, field, fields
-from typing import Any, ClassVar, Optional
+from typing import Any, ClassVar, Dict, List, Optional
 
 from funcy import compact
 from voluptuous import Required
@@ -13,11 +13,17 @@ class Annotation:
     PARAM_META: ClassVar[str] = "meta"
 
     desc: Optional[str] = None
-    type: Optional[str] = None
-    labels: list[str] = field(default_factory=list)
-    meta: dict[str, Any] = field(default_factory=dict)
+    type: Optional[str] = None  # noqa: A003
+    labels: List[str] = field(default_factory=list)
+    meta: Dict[str, Any] = field(default_factory=dict)
 
-    def to_dict(self) -> dict[str, str]:
+    def update(self, **kwargs) -> "Annotation":
+        for attr, value in kwargs.items():
+            if value and hasattr(self, attr):
+                setattr(self, attr, value)
+        return self
+
+    def to_dict(self) -> Dict[str, str]:
         return compact(asdict(self))
 
 
@@ -31,11 +37,11 @@ class Artifact:
 
     path: str
     desc: Optional[str] = None
-    type: Optional[str] = None
-    labels: list[str] = field(default_factory=list)
-    meta: dict[str, Any] = field(default_factory=dict)
+    type: Optional[str] = None  # noqa: A003
+    labels: List[str] = field(default_factory=list)
+    meta: Dict[str, Any] = field(default_factory=dict)
 
-    def to_dict(self) -> dict[str, str]:
+    def to_dict(self) -> Dict[str, str]:
         return compact(asdict(self))
 
 
@@ -46,7 +52,7 @@ ANNOTATION_SCHEMA = {
     Annotation.PARAM_LABELS: [str],
     Annotation.PARAM_META: object,
 }
-ARTIFACT_SCHEMA: dict[Any, Any] = {
+ARTIFACT_SCHEMA = {
     Required(Artifact.PARAM_PATH): str,
     **ANNOTATION_SCHEMA,  # type: ignore[arg-type]
 }

dvc/api/__init__.py

@@ -1,24 +1,22 @@
 from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem
 
-from .artifacts import artifacts_show
-from .data import get_url, open, read  # noqa: A004
-from .dataset import get as get_dataset
-from .experiments import exp_save, exp_show
+from .data import open  # pylint: disable=redefined-builtin
+from .data import get_url, read
+from .experiments import exp_save, exp_show, make_checkpoint
 from .scm import all_branches, all_commits, all_tags
 from .show import metrics_show, params_show
 
 __all__ = [
-    "DVCFileSystem",
     "all_branches",
     "all_commits",
     "all_tags",
-    "artifacts_show",
     "exp_save",
     "exp_show",
-    "get_dataset",
     "get_url",
-    "metrics_show",
+    "make_checkpoint",
     "open",
     "params_show",
+    "metrics_show",
     "read",
+    "DVCFileSystem",
 ]

dvc/api/artifacts.py

@@ -1,58 +0,0 @@
-import os
-from typing import Any, Optional
-
-from dvc.repo import Repo
-
-
-def artifacts_show(
-    name: str,
-    version: Optional[str] = None,
-    stage: Optional[str] = None,
-    repo: Optional[str] = None,
-) -> dict[str, str]:
-    """
-    Return path and Git revision for an artifact in a DVC project.
-
-    The resulting path and revision can be used in conjunction with other dvc.api
-    calls to open and read the artifact.
-
-    Args:
-        name (str): name of the artifact to open.
-        version (str, optional): version of the artifact to open. Defaults to
-            the latest version.
-        stage (str, optional): name of the model registry stage.
-        repo: (str, optional): path or URL for the DVC repo.
-
-    Returns:
-        Dictionary of the form:
-            {
-                "rev": ...,
-                "path": ...,
-            }
-
-    Raises:
-        dvc.exceptions.ArtifactNotFoundError: The specified artifact was not found in
-            the repo.
-    """
-    if version and stage:
-        raise ValueError("Artifact version and stage are mutually exclusive.")
-
-    from dvc.repo.artifacts import Artifacts
-    from dvc.utils import as_posix
-
-    repo_kwargs: dict[str, Any] = {
-        "subrepos": True,
-        "uninitialized": True,
-    }
-
-    dirname, _ = Artifacts.parse_path(name)
-    with Repo.open(repo, **repo_kwargs) as _repo:
-        rev = _repo.artifacts.get_rev(name, version=version, stage=stage)
-        with _repo.switch(rev):
-            root = _repo.fs.root_marker
-            _dirname = _repo.fs.join(root, dirname) if dirname else root
-            with Repo(_dirname, fs=_repo.fs, scm=_repo.scm) as r:
-                path = r.artifacts.get_path(name)
-                path = _repo.fs.join(_repo.fs.root_marker, as_posix(path))
-                parts = _repo.fs.relparts(path, _repo.root_dir)
-                return {"rev": rev, "path": os.path.join(*parts)}

dvc/api/data.py

@@ -1,6 +1,6 @@
 from contextlib import _GeneratorContextManager as GCM
 from contextlib import contextmanager
-from typing import Any, Optional
+from typing import Any, Dict, Optional
 
 from funcy import reraise
 
@@ -25,14 +25,7 @@ def _wrap_exceptions(repo, url):
         raise PathMissingError(exc.path, url) from exc
 
 
-def get_url(
-    path: str,
-    repo: Optional[str] = None,
-    rev: Optional[str] = None,
-    remote: Optional[str] = None,
-    config: Optional[dict[str, Any]] = None,
-    remote_config: Optional[dict[str, Any]] = None,
-):
+def get_url(path, repo=None, rev=None, remote=None):
     """
     Returns the URL to the storage location of a data file or directory tracked
     in a DVC repo. For Git repos, HEAD is used unless a rev argument is
@@ -42,72 +35,44 @@ def get_url(
 
     NOTE: This function does not check for the actual existence of the file or
     directory in the remote storage.
-
-    Args:
-        path (str): location and file name of the target, relative to the root
-            of `repo`.
-        repo (str, optional): location of the DVC project or Git Repo.
-            Defaults to the current DVC project (found by walking up from the
-            current working directory tree).
-            It can be a URL or a file system path.
-            Both HTTP and SSH protocols are supported for online Git repos
-            (e.g. [user@]server:project.git).
-        rev (str, optional): Any `Git revision`_ such as a branch or tag name,
-            a commit hash or a dvc experiment name.
-            Defaults to HEAD.
-            If `repo` is not a Git repo, this option is ignored.
-        remote (str, optional): Name of the `DVC remote`_ used to form the
-            returned URL string.
-            Defaults to the `default remote`_ of `repo`.
-            For local projects, the cache is tried before the default remote.
-        config(dict, optional): config to be passed to the DVC repository.
-            Defaults to None.
-        remote_config(dict, optional): remote config to be passed to the DVC
-            repository.
-            Defaults to None.
-
-    Returns:
-        str: URL to the file or directory.
     """
-    from dvc.config import NoRemoteError
-    from dvc_data.index import StorageKeyError
+    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
+        with _wrap_exceptions(_repo, path):
+            fs_path = _repo.dvcfs.from_os_path(path)
 
-    with Repo.open(
-        repo,
-        rev=rev,
-        subrepos=True,
-        uninitialized=True,
-        remote=remote,
-        config=config,
-        remote_config=remote_config,
-    ) as _repo:
-        index, entry = _repo.get_data_index_entry(path)
-        with reraise(
-            (StorageKeyError, ValueError),
-            NoRemoteError(f"no remote specified in {_repo}"),
-        ):
-            remote_fs, remote_path = index.storage_map.get_remote(entry)
-            return remote_fs.unstrip_protocol(remote_path)
+            with reraise(FileNotFoundError, PathMissingError(path, repo)):
+                info = _repo.dvcfs.info(fs_path)
+
+            dvc_info = info.get("dvc_info")
+            if not dvc_info:
+                raise OutputNotFoundError(path, repo)
+
+            dvc_repo = info["repo"]  # pylint: disable=unsubscriptable-object
+            md5 = dvc_info["md5"]
+
+            return dvc_repo.cloud.get_url_for(remote, checksum=md5)
 
 
 class _OpenContextManager(GCM):
-    def __init__(self, func, args, kwds):
+    def __init__(self, func, args, kwds):  # pylint: disable=super-init-not-called
         self.gen = func(*args, **kwds)
-        self.func, self.args, self.kwds = (func, args, kwds)  # type: ignore[assignment]
+        self.func, self.args, self.kwds = (  # type: ignore[assignment]
+            func,
+            args,
+            kwds,
+        )
 
     def __getattr__(self, name):
         raise AttributeError("dvc.api.open() should be used in a with statement.")
 
 
-def open(  # noqa: A001
+def open(  # noqa, pylint: disable=redefined-builtin
     path: str,
     repo: Optional[str] = None,
     rev: Optional[str] = None,
     remote: Optional[str] = None,
     mode: str = "r",
     encoding: Optional[str] = None,
-    config: Optional[dict[str, Any]] = None,
-    remote_config: Optional[dict[str, Any]] = None,
 ):
     """
     Opens a file tracked in a DVC project.
@@ -149,11 +114,6 @@ def open(  # noqa: A001
             Defaults to None.
             This should only be used in text mode.
             Mirrors the namesake parameter in builtin `open()`_.
-        config(dict, optional): config to be passed to the DVC repository.
-            Defaults to None.
-        remote_config(dict, optional): remote config to be passed to the DVC
-            repository.
-            Defaults to None.
 
     Returns:
         _OpenContextManager: A context manager that generatse a corresponding
@@ -249,29 +209,14 @@ def open(  # noqa: A001
         "rev": rev,
         "mode": mode,
         "encoding": encoding,
-        "config": config,
-        "remote_config": remote_config,
     }
     return _OpenContextManager(_open, args, kwargs)
 
 
-def _open(
-    path,
-    repo=None,
-    rev=None,
-    remote=None,
-    mode="r",
-    encoding=None,
-    config=None,
-    remote_config=None,
-):
-    repo_kwargs: dict[str, Any] = {
-        "subrepos": True,
-        "uninitialized": True,
-        "remote": remote,
-        "config": config,
-        "remote_config": remote_config,
-    }
+def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
+    repo_kwargs: Dict[str, Any] = {"subrepos": True, "uninitialized": True}
+    if remote:
+        repo_kwargs["config"] = {"core": {"remote": remote}}
 
     with Repo.open(repo, rev=rev, **repo_kwargs) as _repo:
         with _wrap_exceptions(_repo, path):
@@ -285,7 +230,7 @@ def _open(
             if TYPE_CHECKING:
                 from dvc.fs import FileSystem
 
-            fs: Union[FileSystem, DataFileSystem, DVCFileSystem]
+            fs: Union["FileSystem", DataFileSystem, DVCFileSystem]
             if os.path.isabs(path):
                 fs = DataFileSystem(index=_repo.index.data["local"])
                 fs_path = path
@@ -294,7 +239,11 @@ def _open(
                 fs_path = fs.from_os_path(path)
 
             try:
-                with fs.open(fs_path, mode=mode, encoding=encoding) as fobj:
+                with fs.open(
+                    fs_path,
+                    mode=mode,
+                    encoding=encoding,
+                ) as fobj:
                     yield fobj
             except FileNotFoundError as exc:
                 raise FileMissingError(path) from exc
@@ -302,29 +251,13 @@ def _open(
                 raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
 
 
-def read(
-    path,
-    repo=None,
-    rev=None,
-    remote=None,
-    mode="r",
-    encoding=None,
-    config=None,
-    remote_config=None,
-):
+def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
     """
     Returns the contents of a tracked file (by DVC or Git). For Git repos, HEAD
     is used unless a rev argument is supplied. The default remote is tried
     unless a remote argument is supplied.
     """
     with open(
-        path,
-        repo=repo,
-        rev=rev,
-        remote=remote,
-        mode=mode,
-        encoding=encoding,
-        config=config,
-        remote_config=remote_config,
+        path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
     ) as fd:
         return fd.read()

dvc/api/dataset.py

@@ -1,67 +0,0 @@
-from typing import Literal, TypedDict, Union
-
-
-class DatachainDataset(TypedDict):
-    type: Literal["dc"]
-    name: str
-    version: int
-
-
-class DVCDataset(TypedDict):
-    type: Literal["dvc"]
-    url: str
-    path: str
-    sha: str
-
-
-class URLDataset(TypedDict):
-    type: Literal["url"]
-    files: list[str]
-    path: str
-
-
-def get(name: str) -> Union[DatachainDataset, DVCDataset, URLDataset]:
-    from difflib import get_close_matches
-
-    from dvc.fs import get_cloud_fs
-    from dvc.repo import Repo, datasets
-
-    repo = Repo()
-    try:
-        dataset = repo.datasets[name]
-    except datasets.DatasetNotFoundError as e:
-        add_note = getattr(e, "add_note", lambda _: None)
-        if matches := get_close_matches(name, repo.datasets):
-            add_note(f"Did you mean: {matches[0]!r}?")
-        raise
-
-    if dataset._invalidated:
-        raise ValueError(f"dataset not in sync. Sync with 'dvc ds update {name}'.")
-    if not dataset.lock:
-        raise ValueError("missing lock information")
-    if dataset.type == "dvc":
-        return DVCDataset(
-            type="dvc",
-            url=dataset.lock.url,
-            path=dataset.lock.path,
-            sha=dataset.lock.rev_lock,
-        )
-    if dataset.type == "dc":
-        return DatachainDataset(
-            type="dc", name=dataset.name_version[0], version=dataset.lock.version
-        )
-    if dataset.type == "url":
-        fs_cls, _, path = get_cloud_fs(repo.config, url=dataset.lock.url)
-        assert fs_cls
-        join_version = getattr(fs_cls, "join_version", lambda path, _: path)
-        protocol = fs_cls.protocol
-        versioned_path = join_version(path, dataset.lock.meta.version_id)
-        versioned_path = f"{protocol}://{versioned_path}"
-        files = [
-            join_version(
-                fs_cls.join(versioned_path, file.relpath), file.meta.version_id
-            )
-            for file in dataset.lock.files
-        ]
-        return URLDataset(type="url", files=files, path=versioned_path)
-    raise AssertionError("unreachable")

dvc/api/experiments.py

@@ -1,15 +1,46 @@
-from typing import Optional, Union
+import builtins
+import os
+from time import sleep
+from typing import Dict, List, Optional, Union
 
 from rich.text import Text
 
+from dvc.env import DVC_CHECKPOINT, DVC_ROOT
 from dvc.repo import Repo
 from dvc.repo.experiments.show import tabulate
+from dvc.stage.monitor import CheckpointTask
+
+
+def make_checkpoint():
+    """
+    Signal DVC to create a checkpoint experiment.
+
+    If the current process is being run from DVC, this function will block
+    until DVC has finished creating the checkpoint. Otherwise, this function
+    will return immediately.
+    """
+    if os.getenv(DVC_CHECKPOINT) is None:
+        return
+
+    root_dir = os.getenv(DVC_ROOT, Repo.find_root())
+    signal_file = os.path.join(
+        root_dir, Repo.DVC_DIR, "tmp", CheckpointTask.SIGNAL_FILE
+    )
+
+    with builtins.open(signal_file, "w", encoding="utf-8") as fobj:
+        # NOTE: force flushing/writing empty file to disk, otherwise when
+        # run in certain contexts (pytest) file may not actually be written
+        fobj.write("")
+        fobj.flush()
+        os.fsync(fobj.fileno())
+    while os.path.exists(signal_file):
+        sleep(0.1)
 
 
 def exp_save(
     name: Optional[str] = None,
     force: bool = False,
-    include_untracked: Optional[list[str]] = None,
+    include_untracked: Optional[List[str]] = None,
 ):
     """
     Create a new DVC experiment using `exp save`.
@@ -53,7 +84,7 @@ def _postprocess(exp_rows):
                 except ValueError:
                     exp_row[k] = v_str
 
-            elif not exp_row[k]:
+            if not exp_row[k]:
                 exp_row[k] = None
 
     return exp_rows
@@ -61,12 +92,11 @@ def _postprocess(exp_rows):
 
 def exp_show(
     repo: Optional[str] = None,
-    revs: Optional[Union[str, list[str]]] = None,
+    revs: Optional[Union[str, List[str]]] = None,
     num: int = 1,
     param_deps: bool = False,
     force: bool = False,
-    config: Optional[dict] = None,
-) -> list[dict]:
+) -> List[Dict]:
     """Get DVC experiments tracked in `repo`.
 
     Without arguments, this function will retrieve all experiments derived from
@@ -100,15 +130,13 @@ def exp_show(
             When `force` is specified, DVC will reload all experiment data and
             ignore any previously cached results.
             Defaults to `False`.
-        config (dict, optional): config to be passed through to DVC project.
-            Defaults to `None`.
 
     Returns:
         List[Dict]: Each item in the list will contain a dictionary with
             the info for an individual experiment.
             See Examples below.
     """
-    with Repo.open(repo, config=config) as _repo:
+    with Repo.open(repo) as _repo:
         experiments = _repo.experiments.show(
             revs=revs,
             num=num,

dvc/api/scm.py

@@ -1,9 +1,9 @@
-from typing import Optional
+from typing import List, Optional
 
 from dvc.repo import Repo
 
 
-def all_branches(repo: Optional[str] = None) -> list[str]:
+def all_branches(repo: Optional[str] = None) -> List[str]:
     """Get all Git branches in a DVC repository.
 
     Args:
@@ -20,7 +20,7 @@ def all_branches(repo: Optional[str] = None) -> list[str]:
         return _repo.scm.list_branches()
 
 
-def all_commits(repo: Optional[str] = None) -> list[str]:
+def all_commits(repo: Optional[str] = None) -> List[str]:
     """Get all Git commits in a DVC repository.
 
     Args:
@@ -37,7 +37,7 @@ def all_commits(repo: Optional[str] = None) -> list[str]:
         return _repo.scm.list_all_commits()
 
 
-def all_tags(repo: Optional[str] = None) -> list[str]:
+def all_tags(repo: Optional[str] = None) -> List[str]:
     """Get all Git tags in a DVC repository.
 
     Args:

dvc/api/show.py

@@ -1,15 +1,18 @@
 import typing
 from collections import Counter
-from collections.abc import Iterable
-from typing import Optional, Union
+from typing import Dict, Iterable, Optional, Union
 
 from funcy import first
 
 from dvc.repo import Repo
 
 
+def _onerror_raise(exception: Exception, *args, **kwargs):
+    raise exception
+
+
 def _postprocess(results):
-    processed: dict[str, dict] = {}
+    processed: Dict[str, Dict] = {}
     for rev, rev_data in results.items():
         if not rev_data:
             continue
@@ -20,14 +23,16 @@ def _postprocess(results):
         for file_data in rev_data["data"].values():
             for k in file_data["data"]:
                 counts[k] += 1
+
         for file_name, file_data in rev_data["data"].items():
             to_merge = {
                 (k if counts[k] == 1 else f"{file_name}:{k}"): v
                 for k, v in file_data["data"].items()
             }
-            processed[rev] = processed[rev] | to_merge
+            processed[rev] = {**processed[rev], **to_merge}
 
-    processed.pop("workspace", None)
+    if "workspace" in processed:
+        del processed["workspace"]
 
     return processed
 
@@ -36,8 +41,7 @@ def metrics_show(
     *targets: str,
     repo: Optional[str] = None,
     rev: Optional[str] = None,
-    config: Optional[dict] = None,
-) -> dict:
+) -> Dict:
     """Get metrics tracked in `repo`.
 
     Without arguments, this function will retrieve all metrics from all tracked
@@ -65,8 +69,6 @@ def metrics_show(
             hash or a dvc experiment name.
             If `repo` is not a Git repo, this option is ignored.
             If `None`, the current working tree will be used.
-        config (dict, optional): config to be passed through to DVC project.
-            Defaults to `None`.
 
     Returns:
         Dict: See Examples below.
@@ -133,17 +135,13 @@ def metrics_show(
     .. _Git revision:
         https://git-scm.com/docs/revisions
     """
-    from dvc.repo.metrics.show import to_relpath
 
-    with Repo.open(repo, config=config) as _repo:
+    with Repo.open(repo) as _repo:
         metrics = _repo.metrics.show(
             targets=targets,
             revs=rev if rev is None else [rev],
-            on_error="raise",
+            onerror=_onerror_raise,
         )
-        metrics = {
-            k: to_relpath(_repo.fs, _repo.root_dir, v) for k, v in metrics.items()
-        }
 
     metrics = _postprocess(metrics)
 
@@ -159,8 +157,7 @@ def params_show(
     stages: Optional[Union[str, Iterable[str]]] = None,
     rev: Optional[str] = None,
     deps: bool = False,
-    config: Optional[dict] = None,
-) -> dict:
+) -> Dict:
     """Get parameters tracked in `repo`.
 
     Without arguments, this function will retrieve all params from all tracked
@@ -198,8 +195,6 @@ def params_show(
         deps (bool, optional): Whether to retrieve only parameters that are
             stage dependencies or not.
             Defaults to `False`.
-        config (dict, optional): config to be passed through to DVC project.
-            Defaults to `None`.
 
     Returns:
         Dict: See Examples below.
@@ -381,20 +376,17 @@ def params_show(
         https://git-scm.com/docs/revisions
 
     """
-    from dvc.repo.metrics.show import to_relpath
-
     if isinstance(stages, str):
         stages = [stages]
 
-    with Repo.open(repo, config=config) as _repo:
+    with Repo.open(repo) as _repo:
         params = _repo.params.show(
             revs=rev if rev is None else [rev],
             targets=targets,
-            deps_only=deps,
-            on_error="raise",
+            deps=deps,
+            onerror=_onerror_raise,
             stages=stages,
         )
-        params = {k: to_relpath(_repo.fs, _repo.root_dir, v) for k, v in params.items()}
 
     params = _postprocess(params)
 

dvc/build.py

@@ -1,9 +0,0 @@
-from typing import Optional
-
-try:
-    # file is created during dvc build
-    from . import _build  # type: ignore[attr-defined, import]
-
-    PKG: Optional[str] = _build.PKG  # type: ignore[assignment]
-except ImportError:
-    PKG = None  # type: ignore[assignment]

dvc/cachemgr.py

@@ -1,50 +1,40 @@
 import os
-from typing import TYPE_CHECKING, Optional
 
 from dvc.fs import GitFileSystem, Schemes
 from dvc_data.hashfile.db import get_odb
-from dvc_data.hashfile.hash import DEFAULT_ALGORITHM
-
-if TYPE_CHECKING:
-    from dvc.repo import Repo
-
-LEGACY_HASH_NAMES = {"md5-dos2unix", "params"}
 
 
-def _get_odb(
-    repo,
-    settings,
-    fs=None,
-    prefix: Optional[tuple[str, ...]] = None,
-    hash_name: Optional[str] = None,
-    **kwargs,
-):
+def _get_odb(repo, settings, fs=None):
     from dvc.fs import get_cloud_fs
 
     if not settings:
         return None
 
-    cls, config, fs_path = get_cloud_fs(repo.config, **settings)
+    cls, config, fs_path = get_cloud_fs(repo, **settings)
     fs = fs or cls(**config)
-    if prefix:
-        fs_path = fs.join(fs_path, *prefix)
-    if hash_name:
-        config["hash_name"] = hash_name
     return get_odb(fs, fs_path, state=repo.state, **config)
 
 
 class CacheManager:
     CACHE_DIR = "cache"
-    FILES_DIR = "files"
-    FS_DIR = "fs"
+    CLOUD_SCHEMES = [
+        Schemes.S3,
+        Schemes.GS,
+        Schemes.SSH,
+        Schemes.HDFS,
+        Schemes.WEBHDFS,
+    ]
 
     def __init__(self, repo):
         self._repo = repo
         self.config = config = repo.config["cache"]
         self._odb = {}
 
+        default = None
+        if repo and repo.local_dvc_dir:
+            default = os.path.join(repo.local_dvc_dir, self.CACHE_DIR)
+
         local = config.get("local")
-        default = self.default_local_cache_dir
 
         if local:
             settings = {"name": local}
@@ -63,90 +53,25 @@ class CacheManager:
         if not isinstance(repo.fs, GitFileSystem):
             kwargs["fs"] = repo.fs
 
-        odb = _get_odb(
-            repo,
-            settings,
-            prefix=(self.FILES_DIR, DEFAULT_ALGORITHM),
-            **kwargs,
-        )
+        odb = _get_odb(repo, settings, **kwargs)
         self._odb["repo"] = odb
         self._odb[Schemes.LOCAL] = odb
-        legacy_odb = _get_odb(repo, settings, hash_name="md5-dos2unix", **kwargs)
-        self._odb["legacy"] = legacy_odb
-
-    @property
-    def fs_cache(self):
-        """Filesystem-based cache.
-
-        Currently used as a temporary location to download files that we don't
-        yet have a regular oid (e.g. md5) for.
-        """
-        from dvc_data.index import FileStorage
-
-        return FileStorage(
-            key=(),
-            fs=self.local.fs,
-            path=self.local.fs.join(self.local_cache_dir, self.FS_DIR),
-        )
 
     def _init_odb(self, schemes):
         for scheme in schemes:
             remote = self.config.get(scheme)
             settings = {"name": remote} if remote else None
-            self._odb[scheme] = _get_odb(
-                self._repo,
-                settings,
-                prefix=(self.FILES_DIR, DEFAULT_ALGORITHM),
-            )
+            self._odb[scheme] = _get_odb(self._repo, settings)
 
     def __getattr__(self, name):
+        if name not in self._odb and name in self.CLOUD_SCHEMES:
+            self._init_odb([name])
+
         try:
             return self._odb[name]
         except KeyError as exc:
             raise AttributeError from exc
 
     def by_scheme(self):
+        self._init_odb(self.CLOUD_SCHEMES)
         yield from self._odb.items()
-
-    @property
-    def local_cache_dir(self) -> str:
-        """Return base local cache directory without any prefixes.
-
-        (i.e. `dvc cache dir`).
-        """
-        return self.legacy.path
-
-    @property
-    def default_local_cache_dir(self) -> Optional[str]:
-        repo = self._repo
-        if repo and repo.local_dvc_dir:
-            return os.path.join(repo.local_dvc_dir, self.CACHE_DIR)
-        return None
-
-
-def migrate_2_to_3(repo: "Repo", dry: bool = False):
-    """Migrate legacy 2.x objects to 3.x cache.
-
-    Legacy 'md5-dos2unix' objects will be re-hashed with 'md5', added to 3.x cache,
-    and then a link from the legacy 2.x location to the 3.x location will be created.
-    """
-    from dvc.fs.callbacks import TqdmCallback
-    from dvc.ui import ui
-    from dvc_data.hashfile.db.migrate import migrate, prepare
-
-    src = repo.cache.legacy
-    dest = repo.cache.local
-    if dry:
-        oids = list(src._list_oids())
-        ui.write(
-            f"{len(oids)} files will be re-hashed and migrated to the DVC 3.0 cache "
-            "location."
-        )
-        return
-
-    with TqdmCallback(desc="Computing DVC 3.0 hashes", unit="files") as cb:
-        migration = prepare(src, dest, callback=cb)
-
-    with TqdmCallback(desc="Migrating to DVC 3.0 cache", unit="files") as cb:
-        count = migrate(migration, callback=cb)
-    ui.write(f"Migrated {count} files to DVC 3.0 cache location.")

dvc/cli/__init__.py

@@ -2,17 +2,15 @@
 
 import logging
 import sys
-from typing import Optional
-
-from dvc.log import logger
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
 # [2] https://bugs.python.org/issue29288
+from typing import Optional
+
 "".encode("idna")
 
-
-logger = logger.getChild(__name__)
+logger = logging.getLogger("dvc")
 
 
 class DvcParserError(Exception):
@@ -77,7 +75,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
     from dvc.fs import AuthError, ConfigError, RemoteMissingDepsError
 
     if isinstance(exc, RemoteMissingDepsError):
-        from dvc import PKG
+        from dvc.utils.pkg import PKG
 
         proto = exc.protocol
         by_pkg = {
@@ -85,9 +83,9 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
             "conda": f"conda install -c conda-forge dvc-{proto}",
         }
 
-        if PKG in by_pkg:
+        cmd = by_pkg.get(PKG)
+        if cmd:
             link = format_link("https://dvc.org/doc/install")
-            cmd = by_pkg.get(PKG)
             hint = (
                 "To install dvc with those dependencies, run:\n"
                 "\n"
@@ -96,7 +94,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
                 f"See {link} for more info."
             )
         else:
-            link = format_link("https://github.com/treeverse/dvc/issues")
+            link = format_link("https://github.com/iterative/dvc/issues")
             hint = f"\nPlease report this bug to {link}. Thank you!"
 
         logger.exception(
@@ -157,8 +155,6 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
     Returns:
         int: command's return code.
     """
-    from contextlib import ExitStack
-
     from dvc._debug import debugtools
     from dvc.config import ConfigError
     from dvc.exceptions import DvcException, NotDvcRepoError
@@ -169,13 +165,14 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
     # the copied parent's standard file descriptors. If we make any logging
     # calls in this state it will cause an exception due to writing to a closed
     # file descriptor.
-    if not sys.stderr or sys.stderr.closed:
+    if sys.stderr.closed:  # pylint: disable=using-constant-test
         logging.disable()
-    elif not sys.stdout or sys.stdout.closed:
+    elif sys.stdout.closed:  # pylint: disable=using-constant-test
         logging.disable(logging.INFO)
 
     args = None
-    stack = ExitStack()
+
+    outer_log_level = logger.level
     try:
         args = parse_args(argv)
 
@@ -188,21 +185,22 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
             level = logging.TRACE  # type: ignore[attr-defined]
 
         if level is not None:
-            stack.enter_context(set_loggers_level(level))
+            set_loggers_level(level)
 
         if level and level <= logging.DEBUG:
             from platform import platform, python_implementation, python_version
 
-            from dvc import PKG, __version__
+            from dvc import __version__
+            from dvc.utils.pkg import PKG
 
-            pyv = f"{python_implementation()} {python_version()}"
+            pyv = " ".join([python_implementation(), python_version()])
             pkg = f" ({PKG})" if PKG else ""
             logger.debug("v%s%s, %s on %s", __version__, pkg, pyv, platform())
             logger.debug("command: %s", " ".join(argv or sys.argv))
 
-        logger.trace(args)
+        logger.trace(args)  # type: ignore[attr-defined]
 
-        if sys.stdout and not sys.stdout.closed and not args.quiet:
+        if not sys.stdout.closed and not args.quiet:
             from dvc.ui import ui
 
             ui.enable()
@@ -233,22 +231,19 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
         logger.exception("")
     except DvcParserError:
         ret = 254
-    except Exception as exc:  # noqa: BLE001
+    except Exception as exc:  # noqa, pylint: disable=broad-except
+        # pylint: disable=no-member
         ret = _log_exceptions(exc) or 255
 
     try:
-        import os
-
         from dvc import analytics
 
         if analytics.is_enabled():
             analytics.collect_and_send_report(args, ret)
 
-        logger.trace("Process %s exiting with %s", os.getpid(), ret)
-
         return ret
     finally:
-        stack.close()
+        logger.setLevel(outer_log_level)
 
         from dvc.repo.open_repo import clean_repos
 

dvc/cli/actions.py

@@ -0,0 +1,20 @@
+from argparse import _AppendAction
+
+
+class CommaSeparatedArgs(_AppendAction):  # pylint: disable=protected-access
+    def __call__(self, parser, namespace, values, option_string=None):  # noqa: ARG002
+        from funcy import ldistinct
+
+        items = getattr(namespace, self.dest) or []
+        items.extend(map(str.strip, values.split(",")))
+        setattr(namespace, self.dest, ldistinct(items))
+
+
+class KeyValueArgs(_AppendAction):
+    def __call__(self, parser, namespace, values, option_string=None):  # noqa: ARG002
+        items = getattr(namespace, self.dest) or {}
+        for value in filter(bool, values):
+            key, _, value = value.partition("=")
+            items[key.strip()] = value
+
+        setattr(namespace, self.dest, items)

dvc/cli/command.py

@@ -1,13 +1,12 @@
+import logging
 import os
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any
 
-from dvc.log import logger
-
 if TYPE_CHECKING:
     from dvc.config import Config
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdBase(ABC):
@@ -18,11 +17,8 @@ class CmdBase(ABC):
 
         os.chdir(args.cd)
 
-        self.repo: Repo = Repo(
-            uninitialized=self.UNINITIALIZED,
-            _wait_for_lock=args.wait_for_lock,
-        )
-        self.config: Config = self.repo.config
+        self.repo: "Repo" = Repo(uninitialized=self.UNINITIALIZED)
+        self.config: "Config" = self.repo.config
         self.args = args
 
     def do_run(self):
@@ -35,7 +31,7 @@ class CmdBase(ABC):
 
 
 class CmdBaseNoRepo(CmdBase):
-    def __init__(self, args):
+    def __init__(self, args):  # pylint: disable=super-init-not-called
         self.args = args
 
         os.chdir(args.cd)

dvc/cli/completion.py

@@ -22,15 +22,6 @@ _dvc_compgen_exps() {
     local _dvc_exps=($(dvc exp list -q --all-commits --names-only))
     compgen -W "${_dvc_exps[*]}" -- $1
 }
-
-_dvc_compgen_remotes() {
-    local _dvc_remotes=($(dvc remote list | cut -d' ' -f1))
-    compgen -W "${_dvc_remotes[*]}" -- $1
-}
-
-_dvc_compgen_config_vars() {
-    compgen -W "${_dvc_config_vars[*]}" -- $1
-}
 """
 
 ZSH_PREAMBLE = """
@@ -64,14 +55,6 @@ _dvc_compadd_stages_and_files() {
 _dvc_compadd_exps() {
     _describe 'experiments' "($(dvc exp list -q --all-commits --names-only))"
 }
-
-_dvc_compadd_remotes() {
-    _describe 'remotes' "($(dvc remote list | cut -d' ' -f1))"
-}
-
-_dvc_compadd_config_vars() {
-    _describe 'config_vars' _dvc_config_vars
-}
 """
 
 PREAMBLE = {
@@ -88,22 +71,3 @@ DVCFILES_AND_STAGE = {
     "zsh": "_dvc_compadd_stages_and_files",
 }
 EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
-REMOTE = {"bash": "_dvc_compgen_remotes", "zsh": "_dvc_compadd_remotes"}
-CONFIG_VARS = {"bash": "_dvc_compgen_config_vars", "zsh": "_dvc_compadd_config_vars"}
-
-
-def get_preamble() -> dict[str, str]:
-    from dvc.config_schema import config_vars_for_completion
-
-    ret: dict[str, str] = {}
-    config_vars = list(config_vars_for_completion())
-
-    nl = "\n\t".expandtabs(4)
-    config_vars_arr = f"""
-_dvc_config_vars=(
-    {nl.join(config_vars)}
-)
-"""
-    for shell, preamble in PREAMBLE.items():
-        ret[shell] = config_vars_arr + preamble
-    return ret

dvc/cli/formatter.py

@@ -1,14 +0,0 @@
-import argparse
-
-
-class HelpFormatter(argparse.HelpFormatter):
-    def _get_default_metavar_for_optional(self, action: argparse.Action) -> str:
-        return action.dest
-
-
-class RawTextHelpFormatter(HelpFormatter, argparse.RawTextHelpFormatter):
-    pass
-
-
-class RawDescriptionHelpFormatter(HelpFormatter, argparse.RawDescriptionHelpFormatter):
-    pass

dvc/cli/parser.py

@@ -1,13 +1,12 @@
 """Main parser for the dvc cli."""
-
 import argparse
+import logging
 import os
 from functools import lru_cache
 
 from dvc import __version__
 from dvc.commands import (
     add,
-    artifacts,
     cache,
     check_ignore,
     checkout,
@@ -18,10 +17,8 @@ from dvc.commands import (
     dag,
     data,
     data_sync,
-    dataset,
     destroy,
     diff,
-    du,
     experiments,
     freeze,
     gc,
@@ -29,12 +26,12 @@ from dvc.commands import (
     get_url,
     git_hook,
     imp,
-    imp_db,
     imp_url,
     init,
     install,
     ls,
     ls_url,
+    machine,
     metrics,
     move,
     params,
@@ -44,72 +41,68 @@ from dvc.commands import (
     remove,
     repro,
     root,
+    run,
     stage,
-    studio,
     unprotect,
     update,
     version,
 )
-from dvc.log import logger
 
-from . import DvcParserError, formatter
+from . import DvcParserError
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 COMMANDS = [
-    add,
-    artifacts,
-    cache,
-    check_ignore,
-    checkout,
-    commit,
-    completion,
-    config,
-    daemon,
-    dag,
-    data,
-    data_sync,
-    dataset,
-    destroy,
-    diff,
-    du,
-    experiments,
-    freeze,
-    gc,
+    init,
+    queue,
     get,
     get_url,
-    git_hook,
+    destroy,
+    add,
+    remove,
+    move,
+    unprotect,
+    run,
+    repro,
+    data_sync,
+    gc,
     imp,
-    imp_db,
     imp_url,
-    init,
+    config,
+    checkout,
+    remote,
+    cache,
+    metrics,
+    params,
     install,
+    root,
     ls,
     ls_url,
-    metrics,
-    move,
-    params,
-    plots,
-    queue,
-    remote,
-    remove,
-    repro,
-    root,
-    stage,
-    studio,
-    unprotect,
-    update,
+    freeze,
+    dag,
+    daemon,
+    commit,
+    completion,
+    diff,
     version,
+    update,
+    git_hook,
+    plots,
+    stage,
+    experiments,
+    check_ignore,
+    machine,
+    data,
 ]
 
 
 def _find_parser(parser, cmd_cls):
-    defaults = parser._defaults
+    defaults = parser._defaults  # pylint: disable=protected-access
     if not cmd_cls or cmd_cls == defaults.get("func"):
         parser.print_help()
         raise DvcParserError
 
-    actions = parser._actions
+    actions = parser._actions  # pylint: disable=protected-access
     for action in actions:
         if not isinstance(action.choices, dict):
             # NOTE: we are only interested in subparsers
@@ -121,7 +114,7 @@ def _find_parser(parser, cmd_cls):
 class DvcParser(argparse.ArgumentParser):
     """Custom parser class for dvc CLI."""
 
-    def error(self, message, cmd_cls=None):
+    def error(self, message, cmd_cls=None):  # pylint: disable=arguments-differ
         logger.error(message)
         _find_parser(self, cmd_cls)
 
@@ -146,6 +139,7 @@ def get_parent_parser():
     from dvc._debug import add_debugging_flags
 
     parent_parser = argparse.ArgumentParser(add_help=False)
+    add_debugging_flags(parent_parser)
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
         "-q", "--quiet", action="count", default=0, help="Be quiet."
@@ -153,7 +147,6 @@ def get_parent_parser():
     log_level_group.add_argument(
         "-v", "--verbose", action="count", default=0, help="Be verbose."
     )
-    add_debugging_flags(parent_parser)
 
     return parent_parser
 
@@ -168,7 +161,7 @@ def get_main_parser():
         prog="dvc",
         description=desc,
         parents=[parent_parser],
-        formatter_class=formatter.RawTextHelpFormatter,
+        formatter_class=argparse.RawTextHelpFormatter,
         add_help=False,
     )
 
@@ -200,23 +193,18 @@ def get_main_parser():
         type=str,
     )
 
-    parser.add_argument(
-        "--wait-for-lock",
-        action="store_true",
-        default=False,
-        help="Wait for the lock if it is already held by another process, instead of"
-        " failing immediately.",
-    )
-
     # Sub commands
     subparsers = parser.add_subparsers(
         title="Available Commands",
-        metavar="command",
+        metavar="COMMAND",
         dest="cmd",
-        help="Use `dvc command --help` for command-specific help.",
-        required=True,
+        help="Use `dvc COMMAND --help` for command-specific help.",
     )
 
+    from .utils import fix_subparsers
+
+    fix_subparsers(subparsers)
+
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 

dvc/cli/utils.py

@@ -1,29 +1,13 @@
-import argparse
+def fix_subparsers(subparsers):
+    """Workaround for bug in Python 3. See more info at:
+    https://bugs.python.org/issue16308
+    https://github.com/iterative/dvc/issues/769
 
-
-class DictAction(argparse.Action):
-    def __init__(self, *args, **kwargs):
-        kwargs.setdefault("metavar", "<name>=<value>")
-        super().__init__(*args, **kwargs)
-
-    def __call__(self, parser, args, values, option_string=None):  # noqa: ARG002
-        d = getattr(args, self.dest) or {}
-
-        if isinstance(values, list):
-            kvs = values
-        else:
-            kvs = [values]
-
-        for kv in kvs:
-            key, value = kv.split("=", 1)
-            if not value:
-                raise argparse.ArgumentError(
-                    self,
-                    f'Could not parse argument "{values}" as k1=v1 k2=v2 ... format',
-                )
-            d[key] = value
-
-        setattr(args, self.dest, d)
+    Args:
+        subparsers: subparsers to fix.
+    """
+    subparsers.required = True
+    subparsers.dest = "cmd"
 
 
 def append_doc_link(help_message, path):
@@ -32,7 +16,9 @@ def append_doc_link(help_message, path):
     if not path:
         return help_message
     doc_base = "https://man.dvc.org/"
-    return f"{help_message}\nDocumentation: {format_link(doc_base + path)}"
+    return "{message}\nDocumentation: {link}".format(
+        message=help_message, link=format_link(doc_base + path)
+    )
 
 
 def hide_subparsers_from_help(subparsers):
@@ -40,7 +26,7 @@ def hide_subparsers_from_help(subparsers):
     # from the 'positional arguments' choices list
     # see: https://bugs.python.org/issue22848
     # Need to set `add_help=False`, but avoid setting `help`
-    # (not even to `argparse.SUPPRESS`).
+    # (not even to `argparse.SUPPPRESS`).
     # NOTE: The argument is the parent subparser, not the subcommand parser.
     cmds = [cmd for cmd, parser in subparsers.choices.items() if parser.add_help]
     subparsers.metavar = "{{{}}}".format(",".join(cmds))

dvc/commands/add.py

@@ -1,57 +1,68 @@
-from dvc.cli import completion, formatter
+import argparse
+import logging
+
+from dvc.cli import completion
+from dvc.cli.actions import KeyValueArgs
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
+
+
+def _add_annotating_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help="User description of the data.",
+    )
+    parser.add_argument(
+        "--meta",
+        metavar="key=value",
+        nargs=1,
+        action=KeyValueArgs,
+        help="Custom metadata to add to the data.",
+    )
+    parser.add_argument(
+        "--label",
+        dest="labels",
+        type=str,
+        action="append",
+        metavar="<str>",
+        help="Label for the data.",
+    )
+    parser.add_argument(
+        "--type",
+        type=str,
+        metavar="<str>",
+        help="Type of the data.",
+    )
 
 
 class CmdAdd(CmdBase):
-    def validate_args(self) -> None:
-        from dvc.exceptions import InvalidArgumentError
-
-        args = self.args
-        invalid_opt = None
-
-        if args.to_remote or args.out:
-            message = "{option} can't be used with "
-            message += "--to-remote" if args.to_remote else "--out"
-            if len(args.targets) != 1:
-                invalid_opt = "multiple targets"
-            elif args.glob:
-                invalid_opt = "--glob option"
-            elif args.no_commit:
-                invalid_opt = "--no-commit option"
-        else:
-            message = "{option} can't be used without --to-remote"
-            if args.remote:
-                invalid_opt = "--remote"
-            elif args.remote_jobs:
-                invalid_opt = "--remote-jobs"
-
-        if invalid_opt is not None:
-            raise InvalidArgumentError(message.format(option=invalid_opt))
-
     def run(self):
-        from dvc.exceptions import DvcException, InvalidArgumentError
+        from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
         try:
-            self.validate_args()
-        except InvalidArgumentError:
-            logger.exception("")
-            return 1
+            if len(self.args.targets) > 1 and self.args.file:
+                raise RecursiveAddingWhileUsingFilename()
 
-        try:
             self.repo.add(
                 self.args.targets,
+                recursive=self.args.recursive,
                 no_commit=self.args.no_commit,
+                fname=self.args.file,
+                external=self.args.external,
                 glob=self.args.glob,
+                desc=self.args.desc,
                 out=self.args.out,
+                type=self.args.type,
+                labels=self.args.labels,
+                meta=self.args.meta,
                 remote=self.args.remote,
                 to_remote=self.args.to_remote,
-                remote_jobs=self.args.remote_jobs,
+                jobs=self.args.jobs,
                 force=self.args.force,
-                relink=self.args.relink,
             )
         except FileNotFoundError:
             logger.exception("")
@@ -70,7 +81,14 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(ADD_HELP, "add"),
         help=ADD_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Recursively add files under directory targets.",
     )
     parser.add_argument(
         "--no-commit",
@@ -78,12 +96,23 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Don't put files/directories into cache.",
     )
+    parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow targets that are outside of the DVC repository.",
+    )
     parser.add_argument(
         "--glob",
         action="store_true",
         default=False,
         help="Allows targets containing shell-style wildcards.",
     )
+    parser.add_argument(
+        "--file",
+        help="Specify name of the .dvc file this command will generate.",
+        metavar="<filename>",
+    )
     parser.add_argument(
         "-o",
         "--out",
@@ -101,9 +130,10 @@ def add_parser(subparsers, parent_parser):
         "--remote",
         help="Remote storage to download to",
         metavar="<name>",
-    ).complete = completion.REMOTE
+    )
     parser.add_argument(
-        "--remote-jobs",
+        "-j",
+        "--jobs",
         type=int,
         help=(
             "Only used along with '--to-remote'. "
@@ -120,13 +150,8 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Override local file or folder if exists.",
     )
-    parser.add_argument(
-        "--no-relink",
-        dest="relink",
-        action="store_false",
-        help="Don't recreate links from cache to workspace.",
-    )
-    parser.set_defaults(relink=True)
+
+    _add_annotating_args(parser)
     parser.add_argument(
         "targets", nargs="+", help="Input files/directories to add."
     ).complete = completion.FILE

dvc/commands/artifacts.py

@@ -1,164 +0,0 @@
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import DictAction, append_doc_link
-from dvc.exceptions import DvcException
-from dvc.log import logger
-
-logger = logger.getChild(__name__)
-
-
-class CmdArtifactsGet(CmdBaseNoRepo):
-    def run(self):
-        from dvc.repo.artifacts import Artifacts
-        from dvc.scm import CloneError
-        from dvc.ui import ui
-
-        if self.args.show_url:
-            return self._show_url()
-
-        try:
-            count, out = Artifacts.get(
-                self.args.url,
-                name=self.args.name,
-                version=self.args.rev,
-                stage=self.args.stage,
-                force=self.args.force,
-                config=self.args.config,
-                remote=self.args.remote,
-                remote_config=self.args.remote_config,
-                out=self.args.out,
-            )
-            ui.write(f"Downloaded {count} file(s) to '{out}'")
-            return 0
-        except CloneError:
-            logger.exception("failed to get '%s'", self.args.name)
-            return 1
-        except DvcException:
-            logger.exception(
-                "failed to get '%s' from '%s'", self.args.name, self.args.url
-            )
-            return 1
-
-    def _show_url(self):
-        from dvc.api import artifacts_show, get_url
-        from dvc.ui import ui
-
-        artifact = artifacts_show(
-            self.args.name,
-            version=self.args.rev,
-            stage=self.args.stage,
-            repo=self.args.url,
-        )
-
-        url = get_url(
-            artifact["path"],
-            repo=self.args.url,
-            rev=artifact["rev"],
-            remote=self.args.remote,
-            remote_config=self.args.remote_config,
-        )
-        ui.write(url, force=True)
-
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    ARTIFACTS_HELP = "DVC model registry artifact commands."
-
-    artifacts_parser = subparsers.add_parser(
-        "artifacts",
-        parents=[parent_parser],
-        description=append_doc_link(ARTIFACTS_HELP, "artifacts"),
-        help=ARTIFACTS_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
-    )
-    artifacts_subparsers = artifacts_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc artifacts CMD --help` to display command-specific help.",
-        required=True,
-    )
-
-    ARTIFACTS_GET_HELP = "Download an artifact from a DVC project."
-    get_parser = artifacts_subparsers.add_parser(
-        "get",
-        parents=[parent_parser],
-        description=append_doc_link(ARTIFACTS_GET_HELP, "artifacts/get"),
-        help=ARTIFACTS_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
-    )
-    get_parser.add_argument("url", help="Location of DVC repository to download from")
-    get_parser.add_argument(
-        "name", help="Name of artifact in the repository"
-    ).complete = completion.FILE
-    get_parser.add_argument(
-        "--rev",
-        nargs="?",
-        help="Artifact version",
-        metavar="<version>",
-    )
-    get_parser.add_argument(
-        "--stage",
-        nargs="?",
-        help="Artifact stage",
-        metavar="<stage>",
-    )
-    get_parser.add_argument(
-        "-o",
-        "--out",
-        nargs="?",
-        help="Destination path to download artifact to",
-        metavar="<path>",
-    ).complete = completion.DIR
-    get_parser.add_argument(
-        "--show-url",
-        action="store_true",
-        help=(
-            "Print the storage location (URL) the target data would be "
-            "downloaded from, and exit."
-        ),
-    )
-    get_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-        ),
-        metavar="<number>",
-    )
-    get_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Override local file or folder if exists.",
-    )
-    get_parser.add_argument(
-        "--config",
-        type=str,
-        help=(
-            "Path to a config file that will be merged with the config "
-            "in the target repository."
-        ),
-    )
-    get_parser.add_argument(
-        "--remote",
-        type=str,
-        help=(
-            "Remote name to set as a default in the target repository "
-            "(only applicable when downloading from DVC remote)."
-        ),
-    ).complete = completion.REMOTE
-    get_parser.add_argument(
-        "--remote-config",
-        type=str,
-        nargs="*",
-        action=DictAction,
-        help=(
-            "Remote config options to merge with a remote's config (default or one "
-            "specified by '--remote') in the target repository (only applicable "
-            "when downloading from DVC remote)."
-        ),
-    )
-    get_parser.set_defaults(func=CmdArtifactsGet)

dvc/commands/cache.py

@@ -1,9 +1,8 @@
 import argparse
 import os
 
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.cli import completion
+from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.commands.config import CmdConfig
 from dvc.ui import ui
 
@@ -37,17 +36,6 @@ class CmdCacheDir(CmdConfig):
         return 0
 
 
-class CmdCacheMigrate(CmdBase):
-    def run(self):
-        from dvc.cachemgr import migrate_2_to_3
-        from dvc.repo.commit import commit_2_to_3
-
-        migrate_2_to_3(self.repo, dry=self.args.dry)
-        if self.args.dvc_files:
-            commit_2_to_3(self.repo, dry=self.args.dry)
-        return 0
-
-
 def add_parser(subparsers, parent_parser):
     from dvc.commands.config import parent_config_parser
 
@@ -58,15 +46,16 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(CACHE_HELP, "cache"),
         help=CACHE_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     cache_subparsers = cache_parser.add_subparsers(
         dest="cmd",
         help="Use `dvc cache CMD --help` for command-specific help.",
-        required=True,
     )
 
+    fix_subparsers(cache_subparsers)
+
     parent_cache_config_parser = argparse.ArgumentParser(
         add_help=False, parents=[parent_config_parser]
     )
@@ -77,7 +66,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser, parent_cache_config_parser],
         description=append_doc_link(CACHE_HELP, "cache/dir"),
         help=CACHE_DIR_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     cache_dir_parser.add_argument(
         "-u",
@@ -97,29 +86,3 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
     ).complete = completion.DIR
     cache_dir_parser.set_defaults(func=CmdCacheDir)
-
-    CACHE_MIGRATE_HELP = "Migrate cached files to the DVC 3.0 cache location."
-    cache_migrate_parser = cache_subparsers.add_parser(
-        "migrate",
-        parents=[parent_parser],
-        description=append_doc_link(CACHE_HELP, "cache/migrate"),
-        help=CACHE_MIGRATE_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
-    )
-    cache_migrate_parser.add_argument(
-        "--dvc-files",
-        help=(
-            "Migrate entries in all existing DVC files in the repository "
-            "to the DVC 3.0 format."
-        ),
-        action="store_true",
-    )
-    cache_migrate_parser.add_argument(
-        "--dry",
-        help=(
-            "Only print actions which would be taken without actually migrating "
-            "any data."
-        ),
-        action="store_true",
-    )
-    cache_migrate_parser.set_defaults(func=CmdCacheMigrate)

dvc/commands/check_ignore.py

@@ -1,28 +1,22 @@
-from typing import TYPE_CHECKING
+import argparse
 
-from dvc.cli import completion, formatter
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
 
-if TYPE_CHECKING:
-    from dvc.ignore import CheckIgnoreResult
-
 
 class CmdCheckIgnore(CmdBase):
     def __init__(self, args):
         super().__init__(args)
         self.ignore_filter = self.repo.dvcignore
 
-    def _show_results(self, result: "CheckIgnoreResult"):
-        if not result.match and not self.args.details:
-            return None
+    def _show_results(self, result):
+        if not result.match and not self.args.non_matching:
+            return
 
         if self.args.details:
-            pattern_infos = result.pattern_infos
-            patterns = [str(pi) for pi in pattern_infos]
-            if not patterns and self.args.non_matching:
-                patterns = ["::"]
+            patterns = result.patterns
             if not self.args.all:
                 patterns = patterns[-1:]
 
@@ -30,11 +24,13 @@ class CmdCheckIgnore(CmdBase):
                 ui.write(pattern, result.file, sep="\t")
         else:
             ui.write(result.file)
-        return bool(result.pattern_infos)
 
     def _check_one_file(self, target):
         result = self.ignore_filter.check_ignore(target)
-        return bool(self._show_results(result))
+        self._show_results(result)
+        if result.match:
+            return 0
+        return 1
 
     def _interactive_mode(self):
         ret = 1
@@ -45,14 +41,14 @@ class CmdCheckIgnore(CmdBase):
                 break
             if not target:
                 break
-            if self._check_one_file(target):
+            if not self._check_one_file(target):
                 ret = 0
         return ret
 
     def _normal_mode(self):
         ret = 1
         for target in self.args.targets:
-            if self._check_one_file(target):
+            if not self._check_one_file(target):
                 ret = 0
         return ret
 
@@ -82,8 +78,6 @@ class CmdCheckIgnore(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    import argparse
-
     ADD_HELP = "Check whether files or directories are excluded due to `.dvcignore`."
 
     parser = subparsers.add_parser(
@@ -91,7 +85,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(ADD_HELP, "check-ignore"),
         help=ADD_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     parser.add_argument(
         "-d",
@@ -101,7 +95,14 @@ def add_parser(subparsers, parent_parser):
         help="Show the exclude patterns along with each target path.",
     )
     parser.add_argument(
-        "-a", "--all", action="store_true", default=False, help=argparse.SUPPRESS
+        "-a",
+        "--all",
+        action="store_true",
+        default=False,
+        help=(
+            "Include the target paths which don't match any pattern "
+            "in the `--details` list."
+        ),
     )
     parser.add_argument(
         "-n",

dvc/commands/checkout.py

@@ -1,4 +1,7 @@
-from dvc.cli import completion, formatter
+import argparse
+import operator
+
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.exceptions import CheckoutError
@@ -28,27 +31,23 @@ class CmdCheckout(CmdBase):
 
         stats, exc = None, None
         try:
-            result = self.repo.checkout(
+            stats = self.repo.checkout(
                 targets=self.args.targets,
                 with_deps=self.args.with_deps,
                 force=self.args.force,
                 relink=self.args.relink,
                 recursive=self.args.recursive,
-                allow_missing=self.args.allow_missing,
             )
         except CheckoutError as _exc:
             exc = _exc
-            result = exc.result
+            stats = exc.stats
 
         if self.args.summary:
             default_message = "No changes."
-            stats = result["stats"]
-            assert isinstance(stats, dict)
-            msg = get_summary(stats.items())
+            msg = get_summary(sorted(stats.items(), key=operator.itemgetter(0)))
             ui.write(msg or default_message)
         else:
-            result.pop("stats", {})
-            log_changes(result)
+            log_changes(stats)
 
         if exc:
             raise exc
@@ -67,7 +66,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(CHECKOUT_HELP, "checkout"),
         help=CHECKOUT_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     checkout_parser.add_argument(
         "--summary",
@@ -102,12 +101,6 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Recreate links or copies from cache to workspace.",
     )
-    checkout_parser.add_argument(
-        "--allow-missing",
-        action="store_true",
-        default=False,
-        help="Ignore errors if some of the files or directories are missing.",
-    )
     checkout_parser.add_argument(
         "targets",
         nargs="*",

dvc/commands/commit.py

@@ -1,9 +1,11 @@
-from dvc.cli import completion, formatter
+import argparse
+import logging
+
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdCommit(CmdBase):
@@ -20,7 +22,6 @@ class CmdCommit(CmdBase):
                     with_deps=self.args.with_deps,
                     recursive=self.args.recursive,
                     force=self.args.force,
-                    relink=self.args.relink,
                 )
             except DvcException:
                 logger.exception("failed to commit%s", (" " + target) if target else "")
@@ -39,17 +40,14 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(COMMIT_HELP, "commit"),
         help=COMMIT_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     commit_parser.add_argument(
         "-f",
         "--force",
         action="store_true",
         default=False,
-        help=(
-            "Commit data even if hash values for dependencies or "
-            "outputs did not change."
-        ),
+        help="Commit even if hash value for dependencies/outputs changed.",
     )
     commit_parser.add_argument(
         "-d",
@@ -65,13 +63,6 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Commit cache for subdirectories of the specified directory.",
     )
-    commit_parser.add_argument(
-        "--no-relink",
-        dest="relink",
-        action="store_false",
-        help="Don't recreate links from cache to workspace.",
-    )
-    commit_parser.set_defaults(relink=True)
     commit_parser.add_argument(
         "targets",
         nargs="*",

dvc/commands/completion.py

@@ -1,14 +1,12 @@
-from dvc.cli import formatter
+import argparse
+import logging
+
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.completion import get_preamble
+from dvc.cli.completion import PREAMBLE
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
-
-
-SUPPORTED_SHELLS = ["bash", "zsh"]
+logger = logging.getLogger(__name__)
 
 
 class CmdCompletion(CmdBaseNoRepo):
@@ -17,7 +15,7 @@ class CmdCompletion(CmdBaseNoRepo):
 
         shell = self.args.shell
         parser = self.args.parser
-        script = shtab.complete(parser, shell=shell, preamble=get_preamble())
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)  # nosec B604
         ui.write(script, force=True)
         return 0
 
@@ -30,13 +28,13 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
         help=COMPLETION_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     completion_parser.add_argument(
         "-s",
         "--shell",
         help="Shell syntax for completions.",
         default="bash",
-        choices=SUPPORTED_SHELLS,
+        choices=["bash", "zsh"],
     )
     completion_parser.set_defaults(func=CmdCompletion)

dvc/commands/config.py

@@ -1,17 +1,14 @@
 import argparse
+import logging
 import os
 
-from funcy import set_in
-
-from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBaseNoRepo
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
-NAME_REGEX = r"^(?P<top>(remote|db)\.)?(?P<section>[^\.]*)\.(?P<option>[^\.]*)$"
+NAME_REGEX = r"^(?P<remote>remote\.)?(?P<section>[^\.]*)\.(?P<option>[^\.]*)$"
 
 
 def _name_type(value):
@@ -20,12 +17,10 @@ def _name_type(value):
     match = re.match(NAME_REGEX, value)
     if not match:
         raise argparse.ArgumentTypeError(
-            "name argument should look like remote.name.option or "
-            "db.name.option or section.option"
+            "name argument should look like remote.name.option or section.option"
         )
-    top = match.group("top")
     return (
-        top.strip(".") if top else None,
+        bool(match.group("remote")),
         match.group("section").lower(),
         match.group("option").lower(),
     )
@@ -54,11 +49,12 @@ class CmdConfig(CmdBaseNoRepo):
             logger.error("name argument is required")
             return 1
 
-        remote_or_db, section, opt = self.args.name
+        remote, section, opt = self.args.name
 
         if self.args.value is None and not self.args.unset:
-            return self._get(remote_or_db, section, opt)
-        return self._set(remote_or_db, section, opt)
+            return self._get(remote, section, opt)
+
+        return self._set(remote, section, opt)
 
     def _list(self):
         if any((self.args.name, self.args.value, self.args.unset)):
@@ -79,18 +75,18 @@ class CmdConfig(CmdBaseNoRepo):
 
         return 0
 
-    def _get(self, remote_or_db, section, opt):
+    def _get(self, remote, section, opt):
         from dvc.config import ConfigError
 
         levels = self._get_appropriate_levels(self.args.level)[::-1]
 
         for level in levels:
             conf = self.config.read(level)
-            if remote_or_db:
-                conf = conf[remote_or_db]
+            if remote:
+                conf = conf["remote"]
 
             try:
-                self._check(conf, remote_or_db, section, opt)
+                self._check(conf, remote, section, opt)
             except ConfigError:
                 if self.args.level:
                     raise
@@ -103,15 +99,16 @@ class CmdConfig(CmdBaseNoRepo):
 
         return 0
 
-    def _set(self, remote_or_db, section, opt):
+    def _set(self, remote, section, opt):
         with self.config.edit(self.args.level) as conf:
-            if remote_or_db:
-                conf = conf[remote_or_db]
+            if remote:
+                conf = conf["remote"]
             if self.args.unset:
-                self._check(conf, remote_or_db, section, opt)
+                self._check(conf, remote, section, opt)
                 del conf[section][opt]
             else:
-                conf.update(set_in(conf, [section, opt], self.args.value))
+                self._check(conf, remote, section)
+                conf[section][opt] = self.args.value
 
         if self.args.name == "cache.type":
             logger.warning(
@@ -122,10 +119,10 @@ class CmdConfig(CmdBaseNoRepo):
 
         return 0
 
-    def _check(self, conf, remote_or_db, section, opt=None):
+    def _check(self, conf, remote, section, opt=None):
         from dvc.config import ConfigError
 
-        name = remote_or_db or "section"
+        name = "remote" if remote else "section"
         if section not in conf:
             raise ConfigError(f"{name} '{section}' doesn't exist")
 
@@ -210,7 +207,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_config_parser, parent_parser],
         description=append_doc_link(CONFIG_HELP, "config"),
         help=CONFIG_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     config_parser.add_argument(
         "-u",
@@ -224,7 +221,7 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         type=_name_type,
         help="Option name (section.option or remote.name.option).",
-    ).complete = completion.CONFIG_VARS
+    )
     config_parser.add_argument("value", nargs="?", help="Option value.")
     config_parser.add_argument(
         "-l",

dvc/commands/daemon.py

@@ -1,8 +1,6 @@
 from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.log import logger
-
-logger = logger.getChild(__name__)
+from dvc.cli.utils import fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):
@@ -23,8 +21,6 @@ class CmdDaemonUpdater(CmdDaemonBase):
         config = Config(dvc_dir, validate=False)
         hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
         updater = Updater(tmp_dir, hardlink_lock=hardlink_lock)
-
-        logger.info("Starting updater to fetch the latest version")
         updater.fetch(detach=False)
 
         return 0
@@ -34,7 +30,6 @@ class CmdDaemonAnalytics(CmdDaemonBase):
     def run(self):
         from dvc import analytics
 
-        logger.info("Sending analytics")
         analytics.send(self.args.target)
 
         return 0
@@ -52,9 +47,10 @@ def add_parser(subparsers, parent_parser):
     daemon_subparsers = daemon_parser.add_subparsers(
         dest="cmd",
         help="Use `dvc daemon CMD --help` for command-specific help.",
-        required=True,
     )
 
+    fix_subparsers(daemon_subparsers)
+
     DAEMON_UPDATER_HELP = "Fetch latest available version."
     daemon_updater_parser = daemon_subparsers.add_parser(
         "updater",

dvc/commands/dag.py

@@ -1,6 +1,6 @@
+import argparse
 from typing import TYPE_CHECKING
 
-from dvc.cli import formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.ui import ui
@@ -17,7 +17,7 @@ def _show_ascii(graph: "DiGraph"):
 
     ret = []
     for pipeline in pipelines:
-        ret.append(draw(pipeline.nodes, pipeline.edges))  # noqa: PERF401
+        ret.append(draw(pipeline.nodes, pipeline.edges))
 
     return "\n".join(ret)
 
@@ -86,7 +86,7 @@ def _collect_targets(repo, target, outs):
             targets.extend([str(out) for out in stage.outs])
             continue
 
-        for out in outs_trie.itervalues(prefix=repo.fs.parts(path)):
+        for out in outs_trie.itervalues(prefix=repo.fs.path.parts(path)):  # noqa: B301
             targets.extend(str(out))
 
     return targets
@@ -127,80 +127,19 @@ def _filter(graph, targets, full):
     return new_graph
 
 
-def _is_foreach_matrix_stage(node, join_string):
-    if node.endswith(".dvc"):
-        return False
-    return join_string in node
-
-
-def _collapse_foreach_matrix_get_nodes(graph):
-    from dvc.parsing import JOIN
-
-    new_nodes = set()
-    nodes_to_remove = set()
-    for _node in list(graph.nodes):
-        if not _is_foreach_matrix_stage(_node, JOIN):
-            continue
-        nodes_to_remove.add(_node)
-        new_nodes.add(_node.split(JOIN)[0])
-    return new_nodes, nodes_to_remove
-
-
-def _collapse_foreach_matrix_get_edges(graph):
-    from dvc.parsing import JOIN
-
-    new_edges = set()
-    edges_to_remove = set()
-    for _e1, _e2 in list(graph.edges):
-        _replace = False
-        _new_e1 = _e1
-        _new_e2 = _e2
-        if _is_foreach_matrix_stage(_e1, JOIN):
-            _new_e1 = _e1.split(JOIN)[0]
-            _replace = True
-        if _is_foreach_matrix_stage(_e2, JOIN):
-            _new_e2 = _e2.split(JOIN)[0]
-            _replace = True
-        if _replace:
-            edges_to_remove.add((_e1, _e2))
-            new_edges.add((_new_e1, _new_e2))
-    return new_edges, edges_to_remove
-
-
-def _collapse_foreach_matrix(graph):
-    new_nodes, nodes_to_remove = _collapse_foreach_matrix_get_nodes(graph)
-    new_edges, edges_to_remove = _collapse_foreach_matrix_get_edges(graph)
-    new_graph = graph.copy()
-    new_graph.remove_edges_from(edges_to_remove)
-    new_graph.add_nodes_from(new_nodes)
-    new_graph.add_edges_from(new_edges)
-    new_graph.remove_nodes_from(nodes_to_remove)
-    return new_graph
-
-
-def _build(repo, target=None, full=False, outs=False, collapse_foreach_matrix=False):
+def _build(repo, target=None, full=False, outs=False):
     targets = _collect_targets(repo, target, outs)
     graph = _transform(repo.index, outs)
-    filtered_graph = _filter(graph, targets, full)
-    if collapse_foreach_matrix:
-        filtered_graph = _collapse_foreach_matrix(filtered_graph)
-    return filtered_graph
+    return _filter(graph, targets, full)
 
 
 class CmdDAG(CmdBase):
     def run(self):
-        from dvc.exceptions import InvalidArgumentError
-
-        if self.args.outs and self.args.collapse_foreach_matrix:
-            raise InvalidArgumentError(
-                "`--outs` and `--collapse-foreach-matrix` are mutually exclusive"
-            )
         graph = _build(
             self.repo,
             target=self.args.target,
             full=self.args.full,
             outs=self.args.outs,
-            collapse_foreach_matrix=self.args.collapse_foreach_matrix,
         )
 
         if self.args.dot:
@@ -221,7 +160,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(DAG_HELP, "dag"),
         help=DAG_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     dag_parser.add_argument(
         "--dot",
@@ -237,19 +176,12 @@ def add_parser(subparsers, parent_parser):
     )
     dag_parser.add_argument(
         "--md",
+        "--show-md",
         action="store_true",
         default=False,
         dest="markdown",
         help="Print DAG with mermaid format wrapped in Markdown block.",
     )
-    dag_parser.add_argument(
-        "--collapse-foreach-matrix",
-        action="store_true",
-        default=False,
-        help=(
-            "Collapse stages from each foreach/matrix definition into a single node."
-        ),
-    )
     dag_parser.add_argument(
         "--full",
         action="store_true",

dvc/commands/data.py

@@ -1,31 +1,33 @@
-from typing import TYPE_CHECKING, ClassVar
+import argparse
+import logging
+from operator import itemgetter
+from typing import TYPE_CHECKING, Any, Dict, Iterable, Set
 
 from funcy import chunks, compact, log_durations
 
-from dvc.cli import completion, formatter
+from dvc.cli import completion
+from dvc.cli.actions import CommaSeparatedArgs
 from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.log import logger
+from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_help
 from dvc.ui import ui
 from dvc.utils import colorize
 
 if TYPE_CHECKING:
-    from dvc.repo.data import GitInfo
     from dvc.repo.data import Status as DataStatus
 
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdDataStatus(CmdBase):
-    COLORS: ClassVar[dict[str, str]] = {
+    COLORS = {
         "not_in_remote": "red",
         "not_in_cache": "red",
         "committed": "green",
         "uncommitted": "yellow",
         "untracked": "cyan",
     }
-    LABELS: ClassVar[dict[str, str]] = {
+    LABELS = {
         "not_in_remote": "Not in remote",
         "not_in_cache": "Not in cache",
         "committed": "DVC committed changes",
@@ -33,7 +35,7 @@ class CmdDataStatus(CmdBase):
         "untracked": "Untracked files",
         "unchanged": "DVC unchanged files",
     }
-    HINTS: ClassVar[dict[str, tuple[str, ...]]] = {
+    HINTS = {
         "not_in_remote": ('use "dvc push <file>..." to upload files',),
         "not_in_cache": ('use "dvc fetch <file>..." to download files',),
         "committed": ("git commit the corresponding dvc files to update the repo",),
@@ -56,15 +58,20 @@ class CmdDataStatus(CmdBase):
     def _process_status(status: "DataStatus"):
         """Flatten stage status, and filter empty stage status contents."""
         for stage, stage_status in status.items():
-            if not stage_status or (
-                isinstance(stage_status, dict) and not any(stage_status.values())
-            ):
+            items = stage_status
+            if isinstance(stage_status, dict):
+                items = {
+                    file: state
+                    for state, files in stage_status.items()
+                    for file in files
+                }
+            if not items:
                 continue
-            yield stage, stage_status
+            yield stage, items
 
     @classmethod
     def _show_status(cls, status: "DataStatus") -> int:  # noqa: C901
-        git_info: GitInfo = status.pop("git")  # type: ignore[misc]
+        git_info = status.pop("git")  # type: ignore[misc]
         result = dict(cls._process_status(status))
         if not result:
             no_changes = "No changes"
@@ -87,14 +94,7 @@ class CmdDataStatus(CmdBase):
 
             if isinstance(stage_status, dict):
                 items = [
-                    f"{state}: "
-                    + (
-                        " -> ".join(change.values())
-                        if isinstance(change, dict)
-                        else change
-                    )
-                    for state, changes in stage_status.items()
-                    for change in changes
+                    ": ".join([state, file]) for file, state in stage_status.items()
                 ]
             else:
                 items = stage_status
@@ -111,12 +111,12 @@ class CmdDataStatus(CmdBase):
         return 0
 
     def run(self) -> int:
-        with log_durations(logger.trace, "in data_status"):
+        with log_durations(
+            logger.trace, "in data_status"  # type: ignore[attr-defined]
+        ):
             status = self.repo.data_status(
-                targets=self.args.targets,
                 granular=self.args.granular,
                 untracked_files=self.args.untracked_files,
-                remote=self.args.remote,
                 not_in_remote=self.args.not_in_remote,
                 remote_refresh=self.args.remote_refresh,
             )
@@ -132,20 +132,74 @@ class CmdDataStatus(CmdBase):
         return self._show_status(status)
 
 
+class CmdDataLs(CmdBase):
+    @staticmethod
+    def _show_table(
+        d: Iterable[Dict[str, Any]],
+        filter_types: Set[str],
+        filter_labels: Set[str],
+        markdown: bool = False,
+    ) -> None:
+        from rich.style import Style
+
+        from dvc.compare import TabularData
+
+        td = TabularData(
+            columns=["Path", "Type", "Labels", "Description"], fill_value="-"
+        )
+        for entry in sorted(d, key=itemgetter("path")):
+            typ = entry.get("type", "")
+            desc = entry.get("desc", "-")
+            labels = entry.get("labels", [])
+
+            if filter_types and typ not in filter_types:
+                continue
+            if filter_labels and filter_labels.isdisjoint(labels):
+                continue
+
+            rich_label = ui.rich_text()
+            for index, label in enumerate(labels):
+                if index:
+                    rich_label.append(",")
+                style = Style(bold=label in filter_labels, color="green")
+                rich_label.append(label, style=style)
+
+            if markdown and desc:
+                desc = desc.partition("\n")[0]
+
+            path = ui.rich_text(entry["path"], style="cyan")
+            type_style = Style(bold=typ in filter_types, color="yellow")
+            typ = ui.rich_text(entry.get("type", ""), style=type_style)
+            td.append([path, typ or "-", rich_label or "-", desc])
+
+        td.render(markdown=markdown, rich_table=True)
+
+    def run(self):
+        from dvc.repo.data import ls
+
+        filter_labels = set(self.args.labels)
+        filter_types = set(self.args.type)
+        d = ls(self.repo, targets=self.args.targets, recursive=self.args.recursive)
+        self._show_table(
+            d,
+            filter_labels=filter_labels,
+            filter_types=filter_types,
+            markdown=self.args.markdown,
+        )
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
-    data_help = "Commands related to data management."
     data_parser = subparsers.add_parser(
         "data",
         parents=[parent_parser],
-        description=append_doc_link(data_help, "data/status"),
-        help=data_help,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     data_subparsers = data_parser.add_subparsers(
         dest="cmd",
         help="Use `dvc data CMD --help` to display command-specific help.",
-        required=True,
     )
+    fix_subparsers(data_subparsers)
 
     DATA_STATUS_HELP = (
         "Show changes between the last git commit, the dvcfiles and the workspace."
@@ -154,23 +208,22 @@ def add_parser(subparsers, parent_parser):
         "status",
         parents=[parent_parser],
         description=append_doc_link(DATA_STATUS_HELP, "data/status"),
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
         help=DATA_STATUS_HELP,
     )
-    data_status_parser.add_argument(
-        "targets",
-        nargs="*",
-        help=(
-            "Limit command scope to these tracked files/directories, "
-            ".dvc files and stage names."
-        ),
-    ).complete = completion.FILE  # type: ignore[attr-defined]
     data_status_parser.add_argument(
         "--json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
+    data_status_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        dest="json",
+        help=argparse.SUPPRESS,
+    )
     data_status_parser.add_argument(
         "--granular",
         action="store_true",
@@ -191,12 +244,6 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help="Show untracked files.",
     )
-    data_status_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to check (only applicable with --not-in-remote).",
-        metavar="<name>",
-    ).complete = completion.REMOTE
     data_status_parser.add_argument(
         "--not-in-remote",
         action="store_true",
@@ -210,3 +257,51 @@ def add_parser(subparsers, parent_parser):
         help="Use cached remote index (don't check remote).",
     )
     data_status_parser.set_defaults(func=CmdDataStatus)
+
+    DATA_LS_HELP = "List data tracked by DVC with its metadata."
+    data_ls_parser = data_subparsers.add_parser(
+        "ls",
+        aliases=["list"],
+        parents=[parent_parser],
+        description=append_doc_link(DATA_LS_HELP, "data/ls"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        add_help=False,
+    )
+    data_ls_parser.add_argument(
+        "--md",
+        "--show-md",
+        dest="markdown",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    data_ls_parser.add_argument(
+        "--type",
+        action=CommaSeparatedArgs,
+        default=[],
+        help="Comma-separated list of type to filter.",
+    )
+    data_ls_parser.add_argument(
+        "--labels",
+        action=CommaSeparatedArgs,
+        default=[],
+        help="Comma-separated list of labels to filter.",
+    )
+    data_ls_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Recursively list from the specified directories.",
+    )
+    data_ls_parser.add_argument(
+        "targets",
+        default=None,
+        nargs="*",
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVCFILES_AND_STAGE
+    data_ls_parser.set_defaults(func=CmdDataLs)
+    hide_subparsers_from_help(data_subparsers)

dvc/commands/data_sync.py

@@ -1,11 +1,11 @@
 import argparse
+import logging
 
-from dvc.cli import completion, formatter
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
@@ -14,26 +14,21 @@ class CmdDataBase(CmdBase):
         from dvc.utils.humanize import get_summary
 
         default_msg = "Everything is up to date."
-
-        if not self.args.remote and not self.repo.config["core"].get("remote"):
-            ui.warn("No remote provided and no default remote set.")
-
         ui.write(get_summary(stats.items()) or default_msg)
 
 
 class CmdDataPull(CmdDataBase):
-    def log_summary(self, result):
+    def log_summary(self, stats):
         from dvc.commands.checkout import log_changes
 
-        stats = result.pop("stats", {})
-        log_changes(result)
+        log_changes(stats)
         super().log_summary(stats)
 
     def run(self):
         from dvc.exceptions import CheckoutError, DvcException
 
         try:
-            result = self.repo.pull(
+            stats = self.repo.pull(
                 targets=self.args.targets,
                 jobs=self.args.jobs,
                 remote=self.args.remote,
@@ -47,10 +42,10 @@ class CmdDataPull(CmdDataBase):
                 glob=self.args.glob,
                 allow_missing=self.args.allow_missing,
             )
-            self.log_summary(result)
+            self.log_summary(stats)
         except (CheckoutError, DvcException) as exc:
-            if result := getattr(exc, "result", {}):
-                self.log_summary(result)
+            if stats := getattr(exc, "stats", {}):
+                self.log_summary(stats)
             logger.exception("failed to pull data from the cloud")
             return 1
 
@@ -96,8 +91,6 @@ class CmdDataFetch(CmdDataBase):
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
-                max_size=self.args.max_size,
-                types=self.args.types,
             )
             self.log_summary({"fetched": processed_files_count})
         except DvcException:
@@ -146,11 +139,11 @@ def add_parser(subparsers, _parent_parser):
         parents=[shared_parent_parser()],
         description=append_doc_link(PULL_HELP, "pull"),
         help=PULL_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
         "-r", "--remote", help="Remote storage to pull from", metavar="<name>"
-    ).complete = completion.REMOTE
+    )
     pull_parser.add_argument(
         "-a",
         "--all-branches",
@@ -195,7 +188,7 @@ def add_parser(subparsers, _parent_parser):
     )
     pull_parser.add_argument(
         "--run-cache",
-        action=argparse.BooleanOptionalAction,
+        action="store_true",
         default=False,
         help="Fetch run history for all stages.",
     )
@@ -221,11 +214,11 @@ def add_parser(subparsers, _parent_parser):
         parents=[shared_parent_parser()],
         description=append_doc_link(PUSH_HELP, "push"),
         help=PUSH_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
         "-r", "--remote", help="Remote storage to push to", metavar="<name>"
-    ).complete = completion.REMOTE
+    )
     push_parser.add_argument(
         "-a",
         "--all-branches",
@@ -263,7 +256,7 @@ def add_parser(subparsers, _parent_parser):
     )
     push_parser.add_argument(
         "--run-cache",
-        action=argparse.BooleanOptionalAction,
+        action="store_true",
         default=False,
         help="Push run history for all stages.",
     )
@@ -283,11 +276,11 @@ def add_parser(subparsers, _parent_parser):
         parents=[shared_parent_parser()],
         description=append_doc_link(FETCH_HELP, "fetch"),
         help=FETCH_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
         "-r", "--remote", help="Remote storage to fetch from", metavar="<name>"
-    ).complete = completion.REMOTE
+    )
     fetch_parser.add_argument(
         "-a",
         "--all-branches",
@@ -325,26 +318,10 @@ def add_parser(subparsers, _parent_parser):
     )
     fetch_parser.add_argument(
         "--run-cache",
-        action=argparse.BooleanOptionalAction,
+        action="store_true",
         default=False,
         help="Fetch run history for all stages.",
     )
-    fetch_parser.add_argument(
-        "--max-size",
-        type=int,
-        help="Fetch data files/directories that are each below specified size (bytes).",
-    )
-    fetch_parser.add_argument(
-        "--type",
-        dest="types",
-        action="append",
-        default=[],
-        help=(
-            "Only fetch data files/directories that are of a particular "
-            "type (metrics, plots)."
-        ),
-        choices=["metrics", "plots"],
-    )
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
@@ -356,7 +333,7 @@ def add_parser(subparsers, _parent_parser):
         description=append_doc_link(STATUS_HELP, "status"),
         help=STATUS_HELP,
         conflict_handler="resolve",
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     status_parser.add_argument(
         "-q",
@@ -380,7 +357,7 @@ def add_parser(subparsers, _parent_parser):
         "--remote",
         help="Remote storage to compare local cache to",
         metavar="<name>",
-    ).complete = completion.REMOTE
+    )
     status_parser.add_argument(
         "-a",
         "--all-branches",
@@ -426,15 +403,10 @@ def add_parser(subparsers, _parent_parser):
     )
     status_parser.add_argument(
         "--json",
+        "--show-json",
         action="store_true",
         default=False,
         help="Show status in JSON format.",
     )
-    status_parser.add_argument(
-        "--no-updates",
-        dest="check_updates",
-        action="store_false",
-        help="Ignore updates to imported data.",
-    )
 
     status_parser.set_defaults(func=CmdDataStatus)

dvc/commands/dataset.py

@@ -1,226 +0,0 @@
-from typing import TYPE_CHECKING, Optional
-
-from dvc.cli import formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.exceptions import DvcException
-from dvc.log import logger
-
-if TYPE_CHECKING:
-    from rich.text import Text
-
-    from dvc.repo.datasets import Dataset, FileInfo
-
-logger = logger.getChild(__name__)
-
-
-def diff_files(old: list["FileInfo"], new: list["FileInfo"]) -> dict[str, list[str]]:
-    old_files = {d.relpath: d for d in old}
-    new_files = {d.relpath: d for d in new}
-    rest = old_files.keys() & new_files.keys()
-    return {
-        "added": list(new_files.keys() - old_files.keys()),
-        "deleted": list(old_files.keys() - new_files.keys()),
-        "modified": [p for p in rest if new_files[p] != old_files[p]],
-    }
-
-
-class CmdDatasetAdd(CmdBase):
-    @classmethod
-    def display(cls, name: str, dataset: "Dataset", action: str = "Adding"):
-        from dvc.ui import ui
-
-        assert dataset.lock
-
-        url = dataset.spec.url
-        ver: str = ""
-        if dataset.type == "dc":
-            ver = f"v{dataset.lock.version}"
-        if dataset.type == "dvc":
-            if dataset.lock.path:
-                url = f"{url}:/{dataset.lock.path.lstrip('/')}"
-            if rev := dataset.lock.rev:
-                ver = rev
-
-        ver_part: Optional[Text] = None
-        if ver:
-            ver_part = ui.rich_text.assemble(" @ ", (ver, "repr.number"))
-        text = ui.rich_text.assemble("(", (url, "repr.url"), ver_part or "", ")")
-        ui.write(action, ui.rich_text(name, "cyan"), text, styled=True)
-
-    def run(self):
-        if not self.args.dvc and self.args.rev:
-            raise DvcException("--rev can't be used without --dvc")
-        if not self.args.dvc and self.args.path:
-            raise DvcException("--path can't be used without --dvc")
-
-        d = vars(self.args)
-        for key in ["dvc", "dc", "url"]:
-            if url := d.pop(key, None):
-                d.update({"type": key, "url": url})
-                break
-
-        existing = self.repo.datasets.get(self.args.name)
-        with self.repo.scm_context:
-            if not self.args.force and existing:
-                path = self.repo.fs.relpath(existing.manifest_path)
-                raise DvcException(
-                    f"{self.args.name} already exists in {path}, "
-                    "use the --force to overwrite"
-                )
-            dataset = self.repo.datasets.add(**d)
-            self.display(self.args.name, dataset)
-            return 0
-
-
-class CmdDatasetUpdate(CmdBase):
-    def display(self, name: str, dataset: "Dataset", new: "Dataset"):
-        from dvc.commands.checkout import log_changes
-        from dvc.ui import ui
-
-        action = "Updating"
-        if not dataset.lock:
-            return CmdDatasetAdd.display(name, new, action)
-        if dataset == new:
-            ui.write("[yellow]Nothing to update[/]", styled=True)
-            return None
-
-        assert new.lock
-
-        v: Optional[tuple[str, str]] = None
-        if dataset.type == "dc":
-            assert new.type == "dc"
-            if new.lock.version < dataset.lock.version:
-                action = "Downgrading"
-
-            v = (f"v{dataset.lock.version}", f"v{new.lock.version}")
-        if dataset.type == "dvc":
-            assert new.type == "dvc"
-            v = (f"{dataset.lock.rev_lock[:9]}", f"{new.lock.rev_lock[:9]}")
-
-        if v:
-            part = ui.rich_text.assemble(
-                (v[0], "repr.number"),
-                " -> ",
-                (v[1], "repr.number"),
-            )
-        else:
-            part = ui.rich_text(dataset.spec.url, "repr.url")
-        changes = ui.rich_text.assemble("(", part, ")")
-        ui.write(action, ui.rich_text(name, "cyan"), changes, styled=True)
-        if dataset.type == "url":
-            assert new.type == "url"
-            stats = diff_files(dataset.lock.files, new.lock.files)
-            log_changes(stats)
-        return None
-
-    def run(self):
-        from difflib import get_close_matches
-
-        from dvc.repo.datasets import DatasetNotFoundError
-        from dvc.ui import ui
-
-        version = None
-        if self.args.rev:
-            try:
-                version = int(self.args.rev.lstrip("v"))
-            except ValueError:
-                version = self.args.rev
-
-        d = vars(self.args) | {"version": version}
-        with self.repo.scm_context:
-            try:
-                dataset, new = self.repo.datasets.update(**d)
-            except DatasetNotFoundError:
-                logger.exception("")
-                if matches := get_close_matches(self.args.name, self.repo.datasets):
-                    ui.write(
-                        "did you mean?",
-                        ui.rich_text(matches[0], "cyan"),
-                        stderr=True,
-                        styled=True,
-                    )
-                return 1
-            self.display(self.args.name, dataset, new)
-            return 0
-
-
-def add_parser(subparsers, parent_parser):
-    ds_parser = subparsers.add_parser(
-        "dataset",
-        aliases=["ds"],
-        parents=[parent_parser],
-        formatter_class=formatter.RawDescriptionHelpFormatter,
-    )
-    ds_subparsers = ds_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc dataset CMD --help` to display command-specific help.",
-        required=True,
-    )
-
-    dataset_add_help = "Add a dataset."
-    ds_add_parser = ds_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(dataset_add_help, "dataset/add"),
-        formatter_class=formatter.RawTextHelpFormatter,
-        help=dataset_add_help,
-    )
-
-    url_exclusive_group = ds_add_parser.add_mutually_exclusive_group(required=True)
-    url_exclusive_group.add_argument(
-        "--dc", metavar="name", help="Name of the DataChain dataset to track"
-    )
-    url_exclusive_group.add_argument(
-        "--dvc",
-        help="Path or URL to a Git/DVC repository to track",
-        metavar="url",
-    )
-    url_exclusive_group.add_argument(
-        "--url",
-        help="""\
-URL of a cloud-versioned remote to track. Supported URLs:
-
-s3://bucket/key/path
-gs://bucket/path/to/file/or/dir
-azure://mycontainer/path
-remote://remote_name/path/to/file/or/dir (see `dvc remote`)
-""",
-    )
-    ds_add_parser.add_argument("name", help="Name of the dataset to add")
-    ds_add_parser.add_argument(
-        "--rev",
-        help="Git revision, e.g. SHA, branch, tag (only applicable with --dvc)",
-        metavar="<commit>",
-    )
-    ds_add_parser.add_argument(
-        "--path",
-        help="Path to a file or a directory within a git repository "
-        "(only applicable with --dvc)",
-    )
-    ds_add_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Overwrite existing dataset",
-    )
-    ds_add_parser.set_defaults(func=CmdDatasetAdd)
-
-    dataset_update_help = "Update a dataset."
-    ds_update_parser = ds_subparsers.add_parser(
-        "update",
-        parents=[parent_parser],
-        description=append_doc_link(dataset_update_help, "dataset/update"),
-        formatter_class=formatter.RawDescriptionHelpFormatter,
-        help=dataset_update_help,
-    )
-    ds_update_parser.add_argument("name", help="Name of the dataset to update")
-    ds_update_parser.add_argument(
-        "--rev",
-        "--version",
-        nargs="?",
-        help="DataChain dataset version or Git revision (e.g. SHA, branch, tag)",
-        metavar="<version>",
-    )
-    ds_update_parser.set_defaults(func=CmdDatasetUpdate)

dvc/commands/destroy.py

@@ -1,9 +1,10 @@
-from dvc.cli import formatter
+import argparse
+import logging
+
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdDestroy(CmdBase):
@@ -20,7 +21,7 @@ class CmdDestroy(CmdBase):
             )
 
             if not self.args.force and not ui.confirm(statement):
-                raise DvcException(  # noqa: TRY301
+                raise DvcException(
                     "cannot destroy without a confirmation from the user."
                     " Use `-f` to force."
                 )
@@ -40,7 +41,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(DESTROY_HELP, "destroy"),
         help=DESTROY_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     destroy_parser.add_argument(
         "-f",

dvc/commands/diff.py

@@ -1,12 +1,13 @@
+import argparse
+import logging
 import os
 
-from dvc.cli import completion, formatter
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 def _digest(checksum):
@@ -162,7 +163,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(DIFF_DESCRIPTION, "diff"),
         help=DIFF_DESCRIPTION,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     diff_parser.add_argument(
         "--targets",
@@ -183,6 +184,7 @@ def add_parser(subparsers, parent_parser):
     )
     diff_parser.add_argument(
         "--json",
+        "--show-json",
         help="Format the output into a JSON",
         action="store_true",
         default=False,
@@ -195,6 +197,7 @@ def add_parser(subparsers, parent_parser):
     )
     diff_parser.add_argument(
         "--md",
+        "--show-md",
         help="Show tabulated output in the Markdown format (GFM).",
         action="store_true",
         dest="markdown",

dvc/commands/du.py

@@ -1,79 +0,0 @@
-import logging
-
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import DictAction, append_doc_link
-from dvc.ui import ui
-
-logger = logging.getLogger(__name__)
-
-
-class CmdDU(CmdBaseNoRepo):
-    def run(self):
-        from dvc.repo import Repo
-        from dvc.utils.humanize import naturalsize
-
-        entries = Repo.du(
-            self.args.url,
-            self.args.path,
-            rev=self.args.rev,
-            summarize=self.args.summarize,
-            config=self.args.config,
-            remote=self.args.remote,
-            remote_config=self.args.remote_config,
-        )
-        ui.table([(naturalsize(size), path) for path, size in entries])
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    DU_HELP = "Show disk usage."
-    du_parser = subparsers.add_parser(
-        "du",
-        parents=[parent_parser],
-        description=append_doc_link(DU_HELP, "du"),
-        help=DU_HELP,
-        formatter_class=formatter.RawTextHelpFormatter,
-    )
-    du_parser.add_argument("url", help="Location of DVC repository")
-    du_parser.add_argument(
-        "--rev",
-        nargs="?",
-        help="Git revision (e.g. SHA, branch, tag)",
-        metavar="<commit>",
-    )
-    du_parser.add_argument(
-        "-s",
-        "--summarize",
-        action="store_true",
-        help="Show total disk usage.",
-    )
-    du_parser.add_argument(
-        "--config",
-        type=str,
-        help=(
-            "Path to a config file that will be merged with the config "
-            "in the target repository."
-        ),
-    )
-    du_parser.add_argument(
-        "--remote",
-        type=str,
-        help="Remote name to set as a default in the target repository.",
-    ).complete = completion.REMOTE
-    du_parser.add_argument(
-        "--remote-config",
-        type=str,
-        nargs="*",
-        action=DictAction,
-        help=(
-            "Remote config options to merge with a remote's config (default or one "
-            "specified by '--remote') in the target repository."
-        ),
-    )
-    du_parser.add_argument(
-        "path",
-        nargs="?",
-        help="Path to directory within the repository",
-    ).complete = completion.DIR
-    du_parser.set_defaults(func=CmdDU)

dvc/commands/experiments/__init__.py

@@ -1,17 +1,19 @@
-from dvc.cli import formatter
-from dvc.cli.utils import append_doc_link, hide_subparsers_from_help
+import argparse
+
+from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_help
 from dvc.commands.experiments import (
     apply,
     branch,
     clean,
     diff,
     exec_run,
+    gc,
+    init,
     ls,
     pull,
     push,
     queue_worker,
     remove,
-    rename,
     run,
     save,
     show,
@@ -23,12 +25,13 @@ SUB_COMMANDS = [
     clean,
     diff,
     exec_run,
+    gc,
+    init,
     ls,
     pull,
     push,
     queue_worker,
     remove,
-    rename,
     run,
     save,
     show,
@@ -43,30 +46,21 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         aliases=["exp"],
         description=append_doc_link(EXPERIMENTS_HELP, "exp"),
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
         help=EXPERIMENTS_HELP,
     )
 
     experiments_subparsers = experiments_parser.add_subparsers(
         dest="cmd",
         help="Use `dvc experiments CMD --help` to display command-specific help.",
-        required=True,
     )
 
+    fix_subparsers(experiments_subparsers)
     for cmd in SUB_COMMANDS:
         cmd.add_parser(experiments_subparsers, parent_parser)
     hide_subparsers_from_help(experiments_subparsers)
 
 
-def add_keep_selection_flag(experiments_subcmd_parser):
-    experiments_subcmd_parser.add_argument(
-        "--keep",
-        action="store_true",
-        default=False,
-        help="Keep the selected experiments instead of removing them.",
-    )
-
-
 def add_rev_selection_flags(
     experiments_subcmd_parser, command: str, default: bool = True
 ):

dvc/commands/experiments/apply.py

@@ -1,10 +1,12 @@
-from dvc.cli import completion, formatter
+import argparse
+import logging
+
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsApply(CmdBase):
@@ -28,7 +30,7 @@ def add_parser(experiments_subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_APPLY_HELP, "exp/apply"),
         help=EXPERIMENTS_APPLY_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_apply_parser.add_argument(
         "--no-force",

dvc/commands/experiments/branch.py

@@ -1,9 +1,10 @@
-from dvc.cli import formatter
+import argparse
+import logging
+
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsBranch(CmdBase):
@@ -20,7 +21,7 @@ def add_parser(experiments_subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_BRANCH_HELP, "exp/branch"),
         help=EXPERIMENTS_BRANCH_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_branch_parser.add_argument(
         "experiment", help="Experiment to be promoted."

dvc/commands/experiments/clean.py

@@ -1,9 +1,10 @@
-from dvc.cli import formatter
+import argparse
+import logging
+
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsClean(CmdBase):
@@ -19,6 +20,6 @@ def add_parser(experiments_subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_CLEAN_HELP, "exp/clean"),
         help=EXPERIMENTS_CLEAN_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_clean_parser.set_defaults(func=CmdExperimentsClean)

dvc/commands/experiments/diff.py

@@ -1,12 +1,14 @@
-from dvc.cli import completion, formatter
+import argparse
+import logging
+
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.commands.metrics import DEFAULT_PRECISION
 from dvc.exceptions import DvcException
-from dvc.log import logger
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsDiff(CmdBase):
@@ -57,7 +59,7 @@ def add_parser(experiments_subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_DIFF_HELP, "exp/diff"),
         help=EXPERIMENTS_DIFF_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_diff_parser.add_argument(
         "a_rev", nargs="?", help="Old experiment to compare (defaults to HEAD)"
@@ -81,12 +83,14 @@ def add_parser(experiments_subparsers, parent_parser):
     )
     experiments_diff_parser.add_argument(
         "--json",
+        "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
     experiments_diff_parser.add_argument(
         "--md",
+        "--show-md",
         action="store_true",
         default=False,
         dest="markdown",

dvc/commands/experiments/exec_run.py

@@ -1,7 +1,8 @@
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.log import logger
+import logging
 
-logger = logger.getChild(__name__)
+from dvc.cli.command import CmdBaseNoRepo
+
+logger = logging.getLogger(__name__)
 
 
 class CmdExecutorRun(CmdBaseNoRepo):
@@ -48,7 +49,7 @@ def add_parser(experiments_subparsers, parent_parser):
         ),
     )
     exec_run_parser.add_argument(
-        "-m",
+        "-M",
         "--message",
         type=str,
         default=None,

dvc/commands/experiments/gc.py

@@ -0,0 +1,146 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+def _raise_error_if_all_disabled(**kwargs):
+    if not any(kwargs.values()):
+        raise InvalidArgumentError(
+            "Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags`,"
+            " `--all-commits` or `--date` needs to be set."
+        )
+
+
+class CmdExperimentsGC(CmdBase):
+    def run(self):
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            commit_date=self.args.commit_date,
+            workspace=self.args.workspace,
+        )
+
+        msg = "This will remove all experiments except those derived from "
+
+        msg += "the workspace"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        else:
+            if self.args.all_branches and self.args.all_tags:
+                msg += " and all git branches and tags"
+            elif self.args.all_branches:
+                msg += " and all git branches"
+            elif self.args.all_tags:
+                msg += " and all git tags"
+
+            if self.args.commit_date:
+                msg += f" and all git commits before date {self.args.commit_date}"
+        msg += " of the current repo."
+
+        if self.args.queued:
+            msg += " Run queued experiments will be preserved."
+        else:
+            msg += " Run queued experiments will be removed."
+
+        logger.warning(msg)
+
+        msg = "Are you sure you want to proceed?"
+        if not self.args.force and not ui.confirm(msg):
+            return 1
+
+        removed = self.repo.experiments.gc(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+            commit_date=self.args.commit_date,
+            queued=self.args.queued,
+        )
+
+        if removed:
+            ui.write(
+                f"Removed {removed} experiments.",
+                "To remove unused cache files",
+                "use 'dvc gc'.",
+            )
+        else:
+            ui.write("No experiments to remove.")
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
+    EXPERIMENTS_GC_DESCRIPTION = (
+        "Removes all experiments which are not derived"
+        " from the specified Git revisions."
+    )
+    experiments_gc_parser = experiments_subparsers.add_parser(
+        "gc",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_GC_DESCRIPTION, "exp/gc"),
+        help=EXPERIMENTS_GC_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the current workspace.",
+    )
+    experiments_gc_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the tips of all Git branches.",
+    )
+    experiments_gc_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git tags.",
+    )
+    experiments_gc_parser.add_argument(
+        "-A",
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git commits.",
+    )
+    experiments_gc_parser.add_argument(
+        "--date",
+        type=str,
+        dest="commit_date",
+        metavar="<YYYY-MM-DD>",
+        default=None,
+        help=(
+            "Keep experiments from the commits after (inclusive) a certain "
+            "date. Date must match the extended ISO 8601 format (YYYY-MM-DD)."
+        ),
+    )
+    experiments_gc_parser.add_argument(
+        "--queued",
+        action="store_true",
+        default=False,
+        help=(
+            "Keep queued experiments (experiments run queue will be cleared "
+            "by default)."
+        ),
+    )
+    experiments_gc_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force garbage collection - automatically agree to all prompts.",
+    )
+    experiments_gc_parser.set_defaults(func=CmdExperimentsGC)

dvc/commands/experiments/init.py

@@ -0,0 +1,219 @@
+import argparse
+import logging
+from typing import TYPE_CHECKING, List
+
+from funcy import compact
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+if TYPE_CHECKING:
+    from dvc.dependency import Dependency
+    from dvc.stage import PipelineStage
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsInit(CmdBase):
+    DEFAULT_NAME = "train"
+    CODE = "src"
+    DATA = "data"
+    MODELS = "models"
+    DEFAULT_METRICS = "metrics.json"
+    DEFAULT_PARAMS = "params.yaml"
+    PLOTS = "plots"
+    DEFAULTS = {
+        "code": CODE,
+        "data": DATA,
+        "models": MODELS,
+        "metrics": DEFAULT_METRICS,
+        "params": DEFAULT_PARAMS,
+        "plots": PLOTS,
+    }
+
+    def run(self):
+        from dvc.commands.stage import parse_cmd
+
+        cmd = parse_cmd(self.args.command)
+        if not self.args.interactive and not cmd:
+            raise InvalidArgumentError("command is not specified")
+
+        from dvc.repo.experiments.init import init
+
+        defaults = {}
+        if not self.args.explicit:
+            config = self.repo.config["exp"]
+            defaults.update({**self.DEFAULTS, **config})
+
+        cli_args = compact(
+            {
+                "cmd": cmd,
+                "code": self.args.code,
+                "data": self.args.data,
+                "models": self.args.models,
+                "metrics": self.args.metrics,
+                "params": self.args.params,
+                "plots": self.args.plots,
+                "live": self.args.live,
+            }
+        )
+
+        initialized_stage, initialized_deps, initialized_out_dirs = init(
+            self.repo,
+            name=self.args.name,
+            type=self.args.type,
+            defaults=defaults,
+            overrides=cli_args,
+            interactive=self.args.interactive,
+            force=self.args.force,
+        )
+        self._post_init_display(
+            initialized_stage, initialized_deps, initialized_out_dirs
+        )
+        if self.args.run:
+            self.repo.experiments.run(targets=[initialized_stage.addressing])
+        return 0
+
+    def _post_init_display(
+        self,
+        stage: "PipelineStage",
+        new_deps: List["Dependency"],
+        new_out_dirs: List[str],
+    ) -> None:
+        from dvc.utils import humanize
+
+        path_fmt = "[green]{}[/green]".format
+        if new_deps:
+            deps_paths = humanize.join(map(path_fmt, new_deps))
+            ui.write(f"Creating dependencies: {deps_paths}", styled=True)
+
+        if new_out_dirs:
+            out_dirs_paths = humanize.join(map(path_fmt, new_out_dirs))
+            ui.write(f"Creating output directories: {out_dirs_paths}", styled=True)
+
+        ui.write(
+            f"Creating [b]{self.args.name}[/b] stage in [green]dvc.yaml[/]",
+            styled=True,
+        )
+        if stage.outs or not self.args.run:
+            # separate the above status-like messages with help/tips section
+            ui.write(styled=True)
+
+        if stage.outs:
+            outs_paths = humanize.join(map(path_fmt, stage.outs))
+            tips = f"Ensure your experiment command creates {outs_paths}."
+            ui.write(tips, styled=True)
+
+        if not self.args.run:
+            ui.write(
+                'You can now run your experiment using [b]"dvc exp run"[/].',
+                styled=True,
+            )
+        else:
+            # separate between `exp.run` output and `dvc exp init` output
+            ui.write(styled=True)
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    EXPERIMENTS_INIT_HELP = "Quickly setup any project to use experiments."
+    experiments_init_parser = experiments_subparsers.add_parser(
+        "init",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_INIT_HELP, "exp/init"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        help=EXPERIMENTS_INIT_HELP,
+    )
+    experiments_init_parser.add_argument(
+        "command",
+        nargs=argparse.REMAINDER,
+        help="Command to execute.",
+        metavar="command",
+    )
+    experiments_init_parser.add_argument(
+        "--run",
+        action="store_true",
+        help="Run the experiment after initializing it",
+    )
+    experiments_init_parser.add_argument(
+        "--interactive",
+        "-i",
+        action="store_true",
+        help="Prompt for values that are not provided",
+    )
+    experiments_init_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite existing stage",
+    )
+    experiments_init_parser.add_argument(
+        "--explicit",
+        action="store_true",
+        default=False,
+        help="Only use the path values explicitly provided",
+    )
+    experiments_init_parser.add_argument(
+        "--name",
+        "-n",
+        help="Name of the stage to create (default: %(default)s)",
+        default=CmdExperimentsInit.DEFAULT_NAME,
+    )
+    experiments_init_parser.add_argument(
+        "--code",
+        help=(
+            "Path to the source file or directory "
+            "which your experiments depend"
+            f" (default: {CmdExperimentsInit.CODE})"
+        ),
+    )
+    experiments_init_parser.add_argument(
+        "--data",
+        help=(
+            "Path to the data file or directory "
+            "which your experiments depend"
+            f" (default: {CmdExperimentsInit.DATA})"
+        ),
+    )
+    experiments_init_parser.add_argument(
+        "--models",
+        help=(
+            "Path to the model file or directory for your experiments"
+            f" (default: {CmdExperimentsInit.MODELS})"
+        ),
+    )
+    experiments_init_parser.add_argument(
+        "--params",
+        help=(
+            "Path to the parameters file for your experiments"
+            f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})"
+        ),
+    )
+    experiments_init_parser.add_argument(
+        "--metrics",
+        help=(
+            "Path to the metrics file for your experiments"
+            f" (default: {CmdExperimentsInit.DEFAULT_METRICS})"
+        ),
+    )
+    experiments_init_parser.add_argument(
+        "--plots",
+        help=(
+            "Path to the plots file or directory for your experiments"
+            f" (default: {CmdExperimentsInit.PLOTS})"
+        ),
+    )
+    experiments_init_parser.add_argument(
+        "--live",
+        help="Path to log dvclive outputs for your experiments",
+    )
+    experiments_init_parser.add_argument(
+        "--type",
+        choices=["default", "checkpoint"],
+        default="default",
+        help="Select type of stage to create (default: %(default)s)",
+    )
+    experiments_init_parser.set_defaults(func=CmdExperimentsInit)

dvc/commands/experiments/ls.py

@@ -1,53 +1,29 @@
-from dvc.cli import formatter
+import argparse
+import logging
+
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.exceptions import InvalidArgumentError
-from dvc.log import logger
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsList(CmdBase):
     def run(self):
         name_only = self.args.name_only
-        sha_only = self.args.sha_only
-        git_remote = self.args.git_remote
-        if sha_only and git_remote:
-            raise InvalidArgumentError("--sha-only not supported with git_remote.")
         exps = self.repo.experiments.ls(
             all_commits=self.args.all_commits,
             rev=self.args.rev,
             num=self.args.num,
-            git_remote=git_remote,
+            git_remote=self.args.git_remote,
         )
 
-        from dvc.repo.experiments.utils import describe
-        from dvc.scm import Git
-
-        if name_only or sha_only:
-            names = {}
-        else:
-            assert isinstance(self.repo.scm, Git)
-            names = describe(
-                self.repo.scm,
-                (baseline for baseline in exps),
-                logger=logger,
-            )
-
         for baseline in exps:
-            if not (name_only or sha_only):
-                name = names.get(baseline) or baseline[:7]
-                ui.write(f"{name}:")
-            for exp_name, rev in exps[baseline]:
-                if name_only:
-                    ui.write(exp_name)
-                elif sha_only:
-                    ui.write(rev)
-                elif rev:
-                    ui.write(f"\t{rev[:7]} [{exp_name}]")
-                else:
-                    ui.write(f"\t{exp_name}")
+            if not name_only:
+                ui.write(f"{baseline}:")
+            for exp_name in exps[baseline]:
+                indent = "" if name_only else "\t"
+                ui.write(f"{indent}{exp_name}")
 
         return 0
 
@@ -58,25 +34,17 @@ def add_parser(experiments_subparsers, parent_parser):
     EXPERIMENTS_LIST_HELP = "List local and remote experiments."
     experiments_list_parser = experiments_subparsers.add_parser(
         "list",
-        aliases=["ls"],
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_LIST_HELP, "exp/list"),
         help=EXPERIMENTS_LIST_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     add_rev_selection_flags(experiments_list_parser, "List")
-    display_group = experiments_list_parser.add_mutually_exclusive_group()
-    display_group.add_argument(
+    experiments_list_parser.add_argument(
         "--name-only",
         "--names-only",
         action="store_true",
-        help="Only output experiment names (without SHAs or parent commits).",
-    )
-    display_group.add_argument(
-        "--sha-only",
-        "--shas-only",
-        action="store_true",
-        help="Only output experiment commit SHAs (without names or parent commits).",
+        help="Only output experiment names (without parent commits).",
     )
     experiments_list_parser.add_argument(
         "git_remote",

dvc/commands/experiments/pull.py

@@ -1,16 +1,25 @@
 import argparse
+import logging
 
-from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
+from dvc.exceptions import InvalidArgumentError
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsPull(CmdBase):
+    def raise_error_if_all_disabled(self):
+        if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
+            raise InvalidArgumentError(
+                "Either provide an `experiment` argument, or use the "
+                "`--rev` or `--all-commits` flag."
+            )
+
     def run(self):
+        self.raise_error_if_all_disabled()
+
         pulled_exps = self.repo.experiments.pull(
             self.args.git_remote,
             self.args.experiment,
@@ -50,9 +59,9 @@ def add_parser(experiments_subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_PULL_HELP, "exp/pull"),
         help=EXPERIMENTS_PULL_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    add_rev_selection_flags(experiments_pull_parser, "Pull", True)
+    add_rev_selection_flags(experiments_pull_parser, "Pull", False)
     experiments_pull_parser.add_argument(
         "-f",
         "--force",
@@ -71,7 +80,7 @@ def add_parser(experiments_subparsers, parent_parser):
         dest="dvc_remote",
         metavar="<name>",
         help="Name of the DVC remote to use when pulling cached outputs.",
-    ).complete = completion.REMOTE
+    )
     experiments_pull_parser.add_argument(
         "-j",
         "--jobs",
@@ -83,7 +92,7 @@ def add_parser(experiments_subparsers, parent_parser):
     )
     experiments_pull_parser.add_argument(
         "--run-cache",
-        action=argparse.BooleanOptionalAction,
+        action="store_true",
         default=False,
         help="Pull run history for all stages.",
     )

dvc/commands/experiments/push.py

@@ -1,18 +1,26 @@
 import argparse
-from typing import Any
+import logging
+from typing import Any, Dict
 
-from dvc.cli import completion, formatter
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
+from dvc.exceptions import InvalidArgumentError
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsPush(CmdBase):
+    def raise_error_if_all_disabled(self):
+        if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
+            raise InvalidArgumentError(
+                "Either provide an `experiment` argument, or use the "
+                "`--rev` or `--all-commits` flag."
+            )
+
     @staticmethod
-    def log_result(result: dict[str, Any], remote: str):
+    def log_result(result: Dict[str, Any], remote: str):
         from dvc.utils import humanize
 
         def join_exps(exps):
@@ -51,6 +59,8 @@ class CmdExperimentsPush(CmdBase):
     def run(self):
         from dvc.repo.experiments.push import UploadError
 
+        self.raise_error_if_all_disabled()
+
         try:
             result = self.repo.experiments.push(
                 self.args.git_remote,
@@ -90,9 +100,9 @@ def add_parser(experiments_subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_PUSH_HELP, "exp/push"),
         help=EXPERIMENTS_PUSH_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    add_rev_selection_flags(experiments_push_parser, "Push", True)
+    add_rev_selection_flags(experiments_push_parser, "Push", False)
     experiments_push_parser.add_argument(
         "-f",
         "--force",
@@ -111,7 +121,7 @@ def add_parser(experiments_subparsers, parent_parser):
         dest="dvc_remote",
         metavar="<name>",
         help="Name of the DVC remote to use when pushing cached outputs.",
-    ).complete = completion.REMOTE
+    )
     experiments_push_parser.add_argument(
         "-j",
         "--jobs",
@@ -121,7 +131,7 @@ def add_parser(experiments_subparsers, parent_parser):
     )
     experiments_push_parser.add_argument(
         "--run-cache",
-        action=argparse.BooleanOptionalAction,
+        action="store_true",
         default=False,
         help="Push run history for all stages.",
     )

dvc/commands/experiments/queue_worker.py

@@ -1,7 +1,8 @@
-from dvc.cli.command import CmdBase
-from dvc.log import logger
+import logging
 
-logger = logger.getChild(__name__)
+from dvc.cli.command import CmdBase
+
+logger = logging.getLogger(__name__)
 
 
 class CmdQueueWorker(CmdBase):

dvc/commands/experiments/remove.py

@@ -1,11 +1,12 @@
-from dvc.cli import formatter
+import argparse
+import logging
+
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.exceptions import InvalidArgumentError
-from dvc.log import logger
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsRemove(CmdBase):
@@ -34,7 +35,6 @@ class CmdExperimentsRemove(CmdBase):
             num=self.args.num,
             queue=self.args.queue,
             git_remote=self.args.git_remote,
-            keep=self.args.keep,
         )
         if removed:
             ui.write(f"Removed experiments: {humanize.join(map(repr, removed))}")
@@ -45,20 +45,18 @@ class CmdExperimentsRemove(CmdBase):
 
 
 def add_parser(experiments_subparsers, parent_parser):
-    from . import add_keep_selection_flag, add_rev_selection_flags
+    from . import add_rev_selection_flags
 
     EXPERIMENTS_REMOVE_HELP = "Remove experiments."
     experiments_remove_parser = experiments_subparsers.add_parser(
         "remove",
-        aliases=["rm"],
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_REMOVE_HELP, "exp/remove"),
         help=EXPERIMENTS_REMOVE_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remove_group = experiments_remove_parser.add_mutually_exclusive_group()
     add_rev_selection_flags(experiments_remove_parser, "Remove", False)
-    add_keep_selection_flag(experiments_remove_parser)
     remove_group.add_argument(
         "--queue", action="store_true", help="Remove all queued experiments."
     )

dvc/commands/experiments/rename.py

@@ -1,67 +0,0 @@
-from dvc.cli import formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.exceptions import InvalidArgumentError
-from dvc.log import logger
-from dvc.ui import ui
-
-logger = logger.getChild(__name__)
-
-
-class CmdExperimentsRename(CmdBase):
-    def run(self):
-        from dvc.utils import humanize
-
-        if not (self.args.experiment and self.args.name):
-            raise InvalidArgumentError(
-                "An experiment to rename and a new experiment name are required."
-            )
-        renamed = self.repo.experiments.rename(
-            exp_name=self.args.experiment,
-            new_name=self.args.name,
-            git_remote=self.args.git_remote,
-            force=self.args.force,
-        )
-        if renamed:
-            ui.write(f"Renamed experiments: {humanize.join(map(repr, renamed))}")
-        else:
-            ui.write("No experiments to rename.")
-
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-    EXPERIMENTS_RENAME_HELP = "Rename experiments."
-    experiments_rename_parser = experiments_subparsers.add_parser(
-        "rename",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_RENAME_HELP, "exp/rename"),
-        help=EXPERIMENTS_RENAME_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
-    )
-    rename_group = experiments_rename_parser.add_mutually_exclusive_group()
-    rename_group.add_argument(
-        "-g",
-        "--git-remote",
-        metavar="<git_remote>",
-        help="Name or URL of the Git remote to rename the experiment from",
-    )
-    experiments_rename_parser.add_argument(
-        "experiment",
-        help="Experiment to rename.",
-        nargs="?",
-        metavar="<experiment>",
-    )
-    experiments_rename_parser.add_argument(
-        "name",
-        help="New name for the experiment.",
-        metavar="<name>",
-    )
-    experiments_rename_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Replace experiment if it already exists.",
-    )
-    experiments_rename_parser.set_defaults(func=CmdExperimentsRename)

dvc/commands/experiments/run.py

@@ -1,42 +1,80 @@
 import argparse
+import logging
 
-from dvc.cli import formatter
+from dvc.cli import completion
 from dvc.cli.utils import append_doc_link
 from dvc.commands.repro import CmdRepro
 from dvc.commands.repro import add_arguments as add_repro_arguments
-from dvc.log import logger
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsRun(CmdRepro):
     def run(self):
-        self.repo.experiments.run(
+        from dvc.compare import show_metrics
+
+        if self.args.checkpoint_resume:
+            if self.args.reset:
+                raise InvalidArgumentError("--reset and --rev are mutually exclusive.")
+            if not (self.args.queue or self.args.tmp_dir):
+                raise InvalidArgumentError(
+                    "--rev can only be used in conjunction with --queue or --temp."
+                )
+
+        if self.args.reset:
+            ui.write("Any existing checkpoints will be reset and re-run.")
+
+        results = self.repo.experiments.run(
             name=self.args.name,
             queue=self.args.queue,
             run_all=self.args.run_all,
             jobs=self.args.jobs,
             params=self.args.set_param,
+            checkpoint_resume=self.args.checkpoint_resume,
+            reset=self.args.reset,
             tmp_dir=self.args.tmp_dir,
+            machine=self.args.machine,
             copy_paths=self.args.copy_paths,
             message=self.args.message,
-            no_hydra=self.args.no_hydra,
             **self._common_kwargs,
         )
 
+        if self.args.metrics and results:
+            metrics = self.repo.metrics.show(revs=list(results))
+            metrics.pop("workspace", None)
+            show_metrics(metrics)
+
         return 0
 
 
 def add_parser(experiments_subparsers, parent_parser):
-    EXPERIMENTS_RUN_HELP = "Run an experiment."
+    EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
     experiments_run_parser = experiments_subparsers.add_parser(
         "run",
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_RUN_HELP, "exp/run"),
         help=EXPERIMENTS_RUN_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     _add_run_common(experiments_run_parser)
+    experiments_run_parser.add_argument(
+        "-r",
+        "--rev",
+        type=str,
+        dest="checkpoint_resume",
+        help=(
+            "Continue the specified checkpoint experiment. Can only be used "
+            "in conjunction with --queue or --temp."
+        ),
+        metavar="<experiment_rev>",
+    ).complete = completion.EXPERIMENT
+    experiments_run_parser.add_argument(
+        "--reset",
+        action="store_true",
+        help="Reset existing checkpoints and restart the experiment.",
+    )
     experiments_run_parser.set_defaults(func=CmdExperimentsRun)
 
 
@@ -91,6 +129,15 @@ def _add_run_common(parser):
             "your workspace."
         ),
     )
+    parser.add_argument(
+        "--machine",
+        default=None,
+        help=argparse.SUPPRESS,
+        # help=(
+        #     "Run this experiment on the specified 'dvc machine' instance."
+        # )
+        # metavar="<name>",
+    )
     parser.add_argument(
         "-C",
         "--copy-paths",
@@ -102,20 +149,9 @@ def _add_run_common(parser):
         ),
     )
     parser.add_argument(
-        "-m",
+        "-M",
         "--message",
         type=str,
         default=None,
         help="Custom commit message to use when committing the experiment.",
     )
-    parser.add_argument(
-        "--no-hydra",
-        action="store_true",
-        default=False,
-        help=(
-            "Disables automatically updating `params.yaml` with Hydra configuration. "
-            " You can still use `--set-param` to update individual params if needed."
-            " Default is False."
-        ),
-    )
-    parser.add_argument("-M", dest="message", help=argparse.SUPPRESS)  # obsolete

dvc/commands/experiments/save.py

@@ -1,22 +1,19 @@
 import argparse
+import logging
 
-from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
-from dvc.log import logger
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsSave(CmdBase):
     def run(self):
         try:
             ref = self.repo.experiments.save(
-                targets=self.args.targets,
                 name=self.args.name,
-                recursive=self.args.recursive,
                 force=self.args.force,
                 include_untracked=self.args.include_untracked,
                 message=self.args.message,
@@ -30,6 +27,10 @@ class CmdExperimentsSave(CmdBase):
         else:
             name = self.repo.experiments.get_exact_name([ref])[ref]
             ui.write(f"Experiment has been saved as: {name}")
+            ui.write(
+                "\nTo promote an experiment to a Git branch run:\n\n"
+                "\tdvc exp branch <exp> <branch>\n"
+            )
 
         return 0
 
@@ -41,19 +42,7 @@ def add_parser(experiments_subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_SAVE_HELP, "exp/save"),
         help=EXPERIMENTS_SAVE_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
-    )
-    save_parser.add_argument(
-        "targets",
-        nargs="*",
-        help=("Limit DVC caching to these .dvc files and stage names."),
-    ).complete = completion.DVCFILES_AND_STAGE
-    save_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="Cache subdirectories of the specified directory.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     save_parser.add_argument(
         "-f",
@@ -64,6 +53,7 @@ def add_parser(experiments_subparsers, parent_parser):
     )
     save_parser.add_argument(
         "--json",
+        "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
@@ -87,11 +77,10 @@ def add_parser(experiments_subparsers, parent_parser):
         metavar="<path>",
     )
     save_parser.add_argument(
-        "-m",
+        "-M",
         "--message",
         type=str,
         default=None,
         help="Custom commit message to use when committing the experiment.",
     )
-    save_parser.add_argument("-M", dest="message", help=argparse.SUPPRESS)  # obsolete
     save_parser.set_defaults(func=CmdExperimentsSave)

dvc/commands/experiments/show.py

@@ -1,17 +1,17 @@
 import argparse
+import logging
+import os
 import re
-from collections.abc import Collection, Iterable
 from datetime import date, datetime
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Dict, Iterable
 
 from funcy import lmap
 
-from dvc.cli import formatter
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.commands.metrics import DEFAULT_PRECISION
 from dvc.exceptions import DvcException
-from dvc.log import logger
 from dvc.ui import ui
 from dvc.utils.serialize import encode_exception
 
@@ -23,10 +23,13 @@ FILL_VALUE = "-"
 FILL_VALUE_ERRORED = "!"
 
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 experiment_types = {
+    "checkpoint_tip": "│ ╓",
+    "checkpoint_commit": "│ ╟",
+    "checkpoint_base": "├─╨",
     "branch_commit": "├──",
     "branch_base": "└──",
     "baseline": "",
@@ -56,19 +59,21 @@ def baseline_styler(typ):
     return {"style": "bold"} if typ == "baseline" else {}
 
 
-def show_experiments(
+def show_experiments(  # noqa: C901, PLR0912
     td: "TabularData",
-    headers: dict[str, Iterable[str]],
-    keep: Collection[str] = (),
-    drop: Collection[str] = (),
+    headers: Dict[str, Iterable[str]],
+    keep=None,
+    drop=None,
     pager=True,
     csv=False,
     markdown=False,
+    pcp=False,
     **kwargs,
 ):
     if keep:
-        keep_re = re.compile("|".join(keep))
-        td.protect(*(col for col in td.keys() if keep_re.match(col)))  # noqa: SIM118
+        for col in td.keys():  # noqa: SIM118
+            if re.match(keep, col):
+                td.protect(col)
 
     for col in ("State", "Executor"):
         if td.is_empty(col):
@@ -105,24 +110,51 @@ def show_experiments(
         }
     )
 
-    if kwargs.get("only_changed", False):
+    if kwargs.get("only_changed", False) or pcp:
         td.drop_duplicates("cols", ignore_empty=False)
 
     cols_to_drop = set()
-    if drop:
-        drop_re = re.compile("|".join(drop))
-        cols_to_drop = {col for col in td.keys() if drop_re.match(col)}  # noqa: SIM118
+    if drop is not None:
+        cols_to_drop = {col for col in td.keys() if re.match(drop, col)}  # noqa: SIM118
+    if pcp:
+        cols_to_drop.add("Created")
     td.drop(*cols_to_drop)
 
-    td.render(
-        pager=pager,
-        borders="horizontals",
-        rich_table=True,
-        header_styles=styles,
-        row_styles=row_styles,
-        csv=csv,
-        markdown=markdown,
-    )
+    if pcp:
+        subset = {x for x in td.keys() if x != "Experiment"}  # noqa: SIM118
+        td.dropna(
+            "rows",
+            how="all",
+            subset=subset,
+        )
+        td.drop_duplicates("rows", subset=subset)
+        if "Experiment" in td:
+            td.column("Experiment")[:] = [
+                # remove tree characters
+                str(x).encode("ascii", "ignore").strip().decode()
+                for x in td.column("Experiment")
+            ]
+        out = kwargs.get("out") or "dvc_plots"
+        output_file = os.path.join(out, "index.html")
+        ui.write(
+            td.to_parallel_coordinates(
+                output_path=os.path.abspath(output_file),
+                color_by=kwargs.get("sort_by") or "Experiment",
+            )
+        )
+        if kwargs.get("open"):
+            return ui.open_browser(output_file)
+
+    else:
+        td.render(
+            pager=pager,
+            borders="horizontals",
+            rich_table=True,
+            header_styles=styles,
+            row_styles=row_styles,
+            csv=csv,
+            markdown=markdown,
+        )
 
 
 def _normalize_headers(names, count):
@@ -150,7 +182,6 @@ class CmdExperimentsShow(CmdBase):
                 all_commits=self.args.all_commits,
                 hide_queued=self.args.hide_queued,
                 hide_failed=self.args.hide_failed,
-                hide_workspace=self.args.hide_workspace,
                 revs=self.args.rev,
                 num=self.args.num,
                 sha_only=self.args.sha,
@@ -190,6 +221,9 @@ class CmdExperimentsShow(CmdBase):
                 csv=self.args.csv,
                 markdown=self.args.markdown,
                 only_changed=self.args.only_changed,
+                pcp=self.args.pcp,
+                out=self.args.out,
+                open=self.args.open,
             )
         return 0
 
@@ -203,7 +237,7 @@ def add_parser(experiments_subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_SHOW_HELP, "exp/show"),
         help=EXPERIMENTS_SHOW_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     add_rev_selection_flags(experiments_show_parser, "Show")
     experiments_show_parser.add_argument(
@@ -237,15 +271,11 @@ def add_parser(experiments_subparsers, parent_parser):
     )
     experiments_show_parser.add_argument(
         "--drop",
-        action="append",
-        default=[],
         help="Remove the columns matching the specified regex pattern.",
         metavar="<regex_pattern>",
     )
     experiments_show_parser.add_argument(
         "--keep",
-        action="append",
-        default=[],
         help="Preserve the columns matching the specified regex pattern.",
         metavar="<regex_pattern>",
     )
@@ -284,26 +314,23 @@ def add_parser(experiments_subparsers, parent_parser):
         default=False,
         help="Hide queued experiments in the table.",
     )
-    experiments_show_parser.add_argument(
-        "--hide-workspace",
-        action="store_true",
-        default=False,
-        help="Hide workspace row in the table.",
-    )
     experiments_show_parser.add_argument(
         "--json",
+        "--show-json",
         action="store_true",
         default=False,
         help="Print output in JSON format instead of a human-readable table.",
     )
     experiments_show_parser.add_argument(
         "--csv",
+        "--show-csv",
         action="store_true",
         default=False,
         help="Print output in csv format instead of a human-readable table.",
     )
     experiments_show_parser.add_argument(
         "--md",
+        "--show-md",
         action="store_true",
         default=False,
         dest="markdown",
@@ -318,6 +345,27 @@ def add_parser(experiments_subparsers, parent_parser):
         ),
         metavar="<n>",
     )
+    experiments_show_parser.add_argument(
+        "--parallel-coordinates-plot",
+        "--pcp",
+        dest="pcp",
+        action="store_true",
+        default=False,
+        help="Generate a Parallel Coordinates Plot from the tabulated output.",
+    )
+    experiments_show_parser.add_argument(
+        "-o",
+        "--out",
+        default=None,
+        help="Destination folder to save the Parallel Coordinates Plot to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    experiments_show_parser.add_argument(
+        "--open",
+        action="store_true",
+        default=False,
+        help="Open the Parallel Coordinates Plot directly in the browser.",
+    )
     experiments_show_parser.add_argument(
         "--no-fetch",
         dest="fetch_running",

dvc/commands/freeze.py

@@ -1,10 +1,12 @@
-from dvc.cli import completion, formatter
+import argparse
+import logging
+
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdFreezeBase(CmdBase):
@@ -36,7 +38,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(FREEZE_HELP, "freeze"),
         help=FREEZE_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
         "targets", nargs="+", help="Stages or .dvc files to freeze"
@@ -49,7 +51,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(UNFREEZE_HELP, "unfreeze"),
         help=UNFREEZE_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
         "targets", nargs="+", help="Stages or .dvc files to unfreeze"

dvc/commands/gc.py

@@ -1,12 +1,12 @@
+import argparse
+import logging
 import os
 
-from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdGC(CmdBase):
@@ -26,10 +26,6 @@ class CmdGC(CmdBase):
             not_in_remote=self.args.not_in_remote,
         )
 
-        # Don't prompt during dry run
-        if self.args.dry:
-            self.args.force = True
-
         if self.args.rev:
             self.args.num = self.args.num or 1
 
@@ -40,16 +36,9 @@ class CmdGC(CmdBase):
             msg += " and all git commits"
         else:
             if self.args.all_branches and self.args.all_tags:
-                if self.args.num:
-                    msg += f" and last {self.args.num} commits from all git branches"
-                    msg += " and all git tags"
-                else:
-                    msg += " and all git branches and tags"
+                msg += " and all git branches and tags"
             elif self.args.all_branches:
-                if self.args.num:
-                    msg += f" and last {self.args.num} commits from all git branches"
-                else:
-                    msg += " and all git branches"
+                msg += " and all git branches"
             elif self.args.all_tags:
                 msg += " and all git tags"
             if self.args.commit_date:
@@ -61,13 +50,13 @@ class CmdGC(CmdBase):
             msg += " and all experiments"
 
         if self.args.not_in_remote:
-            msg += " that are not present in the DVC remote"
+            msg += " that are present in the DVC remote"
 
         if self.args.repos:
             msg += " of the current and the following repos:"
 
             for repo_path in self.args.repos:
-                msg += f"\n  - {os.path.abspath(repo_path)}"
+                msg += "\n  - %s" % os.path.abspath(repo_path)
         else:
             msg += " of the current repo."
 
@@ -92,8 +81,6 @@ class CmdGC(CmdBase):
             rev=self.args.rev,
             num=self.args.num,
             not_in_remote=self.args.not_in_remote,
-            dry=self.args.dry,
-            skip_failed=self.args.skip_failed,
         )
         return 0
 
@@ -109,7 +96,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(GC_DESCRIPTION, "gc"),
         help=GC_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     gc_parser.add_argument(
         "-w",
@@ -133,8 +120,8 @@ def add_parser(subparsers, parent_parser):
         metavar="<num>",
         help=(
             "Keep data files used in the last `num` commits "
-            "starting from each selected revision root. "
-            "Can be used with `--rev` and `--all-branches`. "
+            "starting from the `--rev` <commit>. "
+            "Only used if `--rev` is also provided. "
             "Defaults to `1`."
         ),
     )
@@ -163,12 +150,12 @@ def add_parser(subparsers, parent_parser):
         "--date",
         type=str,
         dest="commit_date",
-        metavar="<yyyy-mm-dd>",
+        metavar="<YYYY-MM-DD>",
         default=None,
         help=(
             "Keep cached data referenced in the commits after ( inclusive )"
             " a certain time. Date must match the extended ISO 8601 format "
-            "(yyyy-mm-dd)."
+            "(YYYY-MM-DD)."
         ),
     )
     gc_parser.add_argument(
@@ -195,12 +182,6 @@ def add_parser(subparsers, parent_parser):
         "--remote",
         help="Remote storage to collect garbage in",
         metavar="<name>",
-    ).complete = completion.REMOTE
-    gc_parser.add_argument(
-        "--skip-failed",
-        action="store_true",
-        default=False,
-        help="Skip revisions that fail when collected.",
     )
     gc_parser.add_argument(
         "-f",
@@ -232,10 +213,4 @@ def add_parser(subparsers, parent_parser):
         ),
         metavar="<paths>",
     )
-    gc_parser.add_argument(
-        "--dry",
-        action="store_true",
-        default=False,
-        help=("Only print what would get removed without actually removing."),
-    )
     gc_parser.set_defaults(func=CmdGC)

dvc/commands/get.py

@@ -1,10 +1,12 @@
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import DictAction, append_doc_link
-from dvc.exceptions import DvcException
-from dvc.log import logger
+import argparse
+import logging
 
-logger = logger.getChild(__name__)
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 class CmdGet(CmdBaseNoRepo):
@@ -12,13 +14,7 @@ class CmdGet(CmdBaseNoRepo):
         from dvc.api import get_url
         from dvc.ui import ui
 
-        url = get_url(
-            self.args.path,
-            repo=self.args.url,
-            rev=self.args.rev,
-            remote=self.args.remote,
-            remote_config=self.args.remote_config,
-        )
+        url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
         ui.write(url, force=True)
 
         return 0
@@ -41,9 +37,6 @@ class CmdGet(CmdBaseNoRepo):
                 rev=self.args.rev,
                 jobs=self.args.jobs,
                 force=self.args.force,
-                config=self.args.config,
-                remote=self.args.remote,
-                remote_config=self.args.remote_config,
             )
             return 0
         except CloneError:
@@ -63,7 +56,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(GET_HELP, "get"),
         help=GET_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     get_parser.add_argument(
         "url", help="Location of DVC or Git repository to download from"
@@ -109,27 +102,4 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Override local file or folder if exists.",
     )
-    get_parser.add_argument(
-        "--config",
-        type=str,
-        help=(
-            "Path to a config file that will be merged with the config "
-            "in the target repository."
-        ),
-    )
-    get_parser.add_argument(
-        "--remote",
-        type=str,
-        help="Remote name to set as a default in the target repository.",
-    ).complete = completion.REMOTE
-    get_parser.add_argument(
-        "--remote-config",
-        type=str,
-        nargs="*",
-        action=DictAction,
-        help=(
-            "Remote config options to merge with a remote's config (default or one "
-            "specified by '--remote') in the target repository."
-        ),
-    )
     get_parser.set_defaults(func=CmdGet)

dvc/commands/get_url.py

@@ -1,15 +1,16 @@
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import DictAction, append_doc_link
-from dvc.exceptions import DvcException
-from dvc.log import logger
+import argparse
+import logging
 
-logger = logger.getChild(__name__)
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 class CmdGetUrl(CmdBaseNoRepo):
     def run(self):
-        from dvc.config import Config
         from dvc.repo import Repo
 
         try:
@@ -18,8 +19,6 @@ class CmdGetUrl(CmdBaseNoRepo):
                 out=self.args.out,
                 jobs=self.args.jobs,
                 force=self.args.force,
-                fs_config=self.args.fs_config,
-                config=Config.from_cwd(),
             )
             return 0
         except DvcException:
@@ -34,7 +33,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(GET_HELP, "get-url"),
         help=GET_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     get_parser.add_argument(
         "url", help="See `dvc import-url -h` for full list of supported URLs."
@@ -59,11 +58,4 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Override local file or folder if exists.",
     )
-    get_parser.add_argument(
-        "--fs-config",
-        type=str,
-        nargs="*",
-        action=DictAction,
-        help="Config options for the target url.",
-    )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/commands/git_hook.py

@@ -1,11 +1,11 @@
+import logging
 import os
 
-from dvc.cli import formatter
 from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import fix_subparsers
 from dvc.exceptions import NotDvcRepoError
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdHookBase(CmdBaseNoRepo):
@@ -94,9 +94,10 @@ def add_parser(subparsers, parent_parser):
     git_hook_subparsers = git_hook_parser.add_subparsers(
         dest="cmd",
         help="Use `dvc daemon CMD --help` for command-specific help.",
-        required=True,
     )
 
+    fix_subparsers(git_hook_subparsers)
+
     PRE_COMMIT_HELP = "Run pre-commit GIT hook."
     pre_commit_parser = git_hook_subparsers.add_parser(
         "pre-commit",
@@ -139,7 +140,6 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=MERGE_DRIVER_HELP,
         help=MERGE_DRIVER_HELP,
-        formatter_class=formatter.HelpFormatter,
     )
     merge_driver_parser.add_argument(
         "--ancestor",

dvc/commands/imp.py

@@ -1,10 +1,12 @@
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import DictAction, append_doc_link
-from dvc.exceptions import DvcException
-from dvc.log import logger
+import argparse
+import logging
 
-logger = logger.getChild(__name__)
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 class CmdImport(CmdBase):
@@ -16,14 +18,15 @@ class CmdImport(CmdBase):
                 self.args.url,
                 self.args.path,
                 out=self.args.out,
+                fname=self.args.file,
                 rev=self.args.rev,
                 no_exec=self.args.no_exec,
                 no_download=self.args.no_download,
+                desc=self.args.desc,
+                type=self.args.type,
+                labels=self.args.labels,
+                meta=self.args.meta,
                 jobs=self.args.jobs,
-                config=self.args.config,
-                remote=self.args.remote,
-                remote_config=self.args.remote_config,
-                force=self.args.force,
             )
         except CloneError:
             logger.exception("failed to import '%s'", self.args.path)
@@ -39,6 +42,8 @@ class CmdImport(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
+    from .add import _add_annotating_args
+
     IMPORT_HELP = (
         "Download file or directory tracked by DVC or by Git "
         "into the workspace, and track it."
@@ -49,7 +54,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(IMPORT_HELP, "import"),
         help=IMPORT_HELP,
-        formatter_class=formatter.RawTextHelpFormatter,
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
         "url", help="Location of DVC or Git repository to download from"
@@ -64,19 +69,17 @@ def add_parser(subparsers, parent_parser):
         help="Destination path to download files to",
         metavar="<path>",
     ).complete = completion.DIR
-    import_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Override destination file or folder if exists.",
-    )
     import_parser.add_argument(
         "--rev",
         nargs="?",
         help="Git revision (e.g. SHA, branch, tag)",
         metavar="<commit>",
     )
+    import_parser.add_argument(
+        "--file",
+        help="Specify name of the .dvc file this command will generate.",
+        metavar="<filename>",
+    )
     no_download_exec_group = import_parser.add_mutually_exclusive_group()
     no_download_exec_group.add_argument(
         "--no-exec",
@@ -103,27 +106,6 @@ def add_parser(subparsers, parent_parser):
         ),
         metavar="<number>",
     )
-    import_parser.add_argument(
-        "--config",
-        type=str,
-        help=(
-            "Path to a config file that will be merged with the config "
-            "in the target repository."
-        ),
-    )
-    import_parser.add_argument(
-        "--remote",
-        type=str,
-        help="Remote name to set as a default in the target repository.",
-    ).complete = completion.REMOTE
-    import_parser.add_argument(
-        "--remote-config",
-        type=str,
-        nargs="*",
-        action=DictAction,
-        help=(
-            "Remote config options to merge with a remote's config (default or one "
-            "specified by '--remote') in the target repository."
-        ),
-    )
+
+    _add_annotating_args(import_parser)
     import_parser.set_defaults(func=CmdImport)

dvc/commands/imp_db.py

@@ -1,100 +0,0 @@
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase, CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
-from dvc.log import logger
-from dvc.ui import ui
-
-logger = logger.getChild(__name__)
-
-
-class CmdTestDb(CmdBaseNoRepo):
-    def run(self):
-        from dvc.config import Config
-        from dvc.database import client
-        from dvc.exceptions import DvcException
-
-        connection = self.args.conn
-        db_config = Config.from_cwd().get("db", {})
-        if connection not in db_config:
-            raise DvcException(f"connection {connection} not found in config")
-
-        config = db_config.get(connection, {})
-        if self.args.url:
-            config["url"] = self.args.url
-        if self.args.username:
-            config["username"] = self.args.username
-        if self.args.password:
-            config["password"] = self.args.password
-        with client(config) as db:
-            ui.write(f"Testing with {db}", styled=True)
-            db.test_connection()
-        ui.write("Connection successful", styled=True)
-
-
-class CmdImportDb(CmdBase):
-    def run(self):
-        self.repo.imp_db(
-            sql=self.args.sql,
-            table=self.args.table,
-            output_format=self.args.output_format,
-            out=self.args.out,
-            force=self.args.force,
-            connection=self.args.conn,
-        )
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    IMPORT_HELP = "Snapshot a table or a SQL query result to a CSV/JSON format"
-    import_parser = subparsers.add_parser(
-        "import-db",
-        parents=[parent_parser],
-        description=append_doc_link(IMPORT_HELP, "import-db"),
-        help=IMPORT_HELP,
-        formatter_class=formatter.RawTextHelpFormatter,
-    )
-    group = import_parser.add_mutually_exclusive_group(required=True)
-    group.add_argument("--sql", help="SQL query to snapshot")
-    group.add_argument("--table", help="Table to snapshot")
-    import_parser.add_argument(
-        "--output-format",
-        default="csv",
-        const="csv",
-        nargs="?",
-        choices=["csv", "json"],
-        help="Export format",
-    )
-    import_parser.add_argument(
-        "-o",
-        "--out",
-        nargs="?",
-        help="Destination path to download files to",
-        metavar="<path>",
-    ).complete = completion.FILE
-    import_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Override destination file or folder if exists.",
-    )
-    import_parser.add_argument(
-        "--conn",
-        required=True,
-        help="Database connection to use, needs to be set in config",
-    )
-
-    import_parser.set_defaults(func=CmdImportDb)
-
-    TEST_DB_HELP = "Test the database connection"
-    test_db_parser = subparsers.add_parser(
-        "test-db",
-        parents=[parent_parser],
-        description=append_doc_link(TEST_DB_HELP, "test-db"),
-        add_help=False,
-    )
-    test_db_parser.add_argument("--conn", required=True)
-    test_db_parser.add_argument("--url")
-    test_db_parser.add_argument("--password")
-    test_db_parser.add_argument("--username")
-    test_db_parser.set_defaults(func=CmdTestDb)

dvc/commands/imp_url.py

@@ -1,10 +1,12 @@
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import DictAction, append_doc_link
-from dvc.exceptions import DvcException
-from dvc.log import logger
+import argparse
+import logging
 
-logger = logger.getChild(__name__)
+from dvc.cli import completion
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 class CmdImportUrl(CmdBase):
@@ -13,14 +15,18 @@ class CmdImportUrl(CmdBase):
             self.repo.imp_url(
                 self.args.url,
                 out=self.args.out,
+                fname=self.args.file,
                 no_exec=self.args.no_exec,
                 no_download=self.args.no_download,
                 remote=self.args.remote,
                 to_remote=self.args.to_remote,
+                desc=self.args.desc,
+                type=self.args.type,
+                labels=self.args.labels,
+                meta=self.args.meta,
                 jobs=self.args.jobs,
                 force=self.args.force,
                 version_aware=self.args.version_aware,
-                fs_config=self.args.fs_config,
             )
         except DvcException:
             logger.exception(
@@ -35,6 +41,8 @@ class CmdImportUrl(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
+    from .add import _add_annotating_args
+
     IMPORT_HELP = "Download or copy file from URL and take it under DVC control."
 
     import_parser = subparsers.add_parser(
@@ -42,7 +50,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(IMPORT_HELP, "import-url"),
         help=IMPORT_HELP,
-        formatter_class=formatter.RawTextHelpFormatter,
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
         "url",
@@ -62,6 +70,11 @@ def add_parser(subparsers, parent_parser):
     import_parser.add_argument(
         "out", nargs="?", help="Destination path to put files to."
     ).complete = completion.DIR
+    import_parser.add_argument(
+        "--file",
+        help="Specify name of the .dvc file this command will generate.",
+        metavar="<filename>",
+    ).complete = completion.DIR
     import_parser.add_argument(
         "--to-remote",
         action="store_true",
@@ -73,7 +86,7 @@ def add_parser(subparsers, parent_parser):
         "--remote",
         help="Remote storage to download to",
         metavar="<name>",
-    ).complete = completion.REMOTE
+    )
     no_download_exec_group = import_parser.add_mutually_exclusive_group()
     no_download_exec_group.add_argument(
         "--no-exec",
@@ -113,11 +126,5 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Import using cloud versioning. Implied if the URL contains a version ID.",
     )
-    import_parser.add_argument(
-        "--fs-config",
-        type=str,
-        nargs="*",
-        action=DictAction,
-        help="Config options for the target url.",
-    )
+    _add_annotating_args(import_parser)
     import_parser.set_defaults(func=CmdImportUrl)

dvc/commands/init.py

@@ -1,14 +1,15 @@
+import argparse
+import logging
+
 import colorama
 
 from dvc import analytics
-from dvc.cli import formatter
 from dvc.cli.command import CmdBaseNoRepo
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 from dvc.utils import boxify
 from dvc.utils import format_link as fmt_link
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 def _welcome_message():
@@ -29,7 +30,7 @@ def _welcome_message():
         "{yellow}------------{nc}\n"
         f"- Check out the documentation: {fmt_link('https://dvc.org/doc')}\n"
         f"- Get help and share ideas: {fmt_link('https://dvc.org/chat')}\n"
-        f"- Star us on GitHub: {fmt_link('https://github.com/treeverse/dvc')}"
+        f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
     ).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
 
     ui.write(msg)
@@ -42,7 +43,7 @@ class CmdInit(CmdBaseNoRepo):
 
         try:
             with Repo.init(
-                self.args.directory,
+                ".",
                 no_scm=self.args.no_scm,
                 force=self.args.force,
                 subdir=self.args.subdir,
@@ -57,11 +58,10 @@ class CmdInit(CmdBaseNoRepo):
 
 def add_parser(subparsers, parent_parser):
     """Setup parser for `dvc init`."""
-    INIT_HELP = "Initialize DVC repository."
+    INIT_HELP = "Initialize DVC in the current directory."
     INIT_DESCRIPTION = (
-        "Initialize DVC repository in the given directory (defaults to the current "
-        "working directory).\n"
-        "Expects directory to be a Git repository unless --no-scm option is specified."
+        "Initialize DVC in the current directory. Expects directory\n"
+        "to be a Git repository unless --no-scm option is specified."
     )
 
     init_parser = subparsers.add_parser(
@@ -69,15 +69,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(INIT_DESCRIPTION, "init"),
         help=INIT_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
-    )
-    init_parser.add_argument(
-        "directory",
-        nargs="?",
-        default=".",
-        help=(
-            "Directory to initialize DVC in. Defaults to the current working directory."
-        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     init_parser.add_argument(
         "--no-scm",

dvc/commands/install.py

@@ -1,10 +1,11 @@
-from dvc.cli import formatter
+import argparse
+import logging
+
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdInstall(CmdBase):
@@ -24,7 +25,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(INSTALL_HELP, "install"),
         help=INSTALL_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     install_parser.add_argument(
         "--use-pre-commit-tool",

dvc/commands/ls/__init__.py

@@ -1,183 +1,46 @@
-from typing import Callable
+import argparse
+import logging
 
-from dvc.cli import completion, formatter
+from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import DictAction, append_doc_link
+from dvc.cli.utils import append_doc_link
 from dvc.commands.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
-from dvc.log import logger
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
-def _get_formatter(with_color: bool = False) -> Callable[[dict], str]:
-    def fmt(entry: dict) -> str:
-        return entry["path"]
-
+def _prettify(entries, with_color=False):
     if with_color:
         ls_colors = LsColors()
-        return ls_colors.format
+        fmt = ls_colors.format
+    else:
 
-    return fmt
+        def fmt(entry):
+            return entry["path"]
 
-
-def _format_entry(entry, name, with_size=True, with_hash=False):
-    from dvc.utils.humanize import naturalsize
-
-    ret = []
-    if with_size:
-        size = entry.get("size")
-        if size is None or (size <= 0 and entry.get("isdir")):
-            size = ""
-        else:
-            size = naturalsize(size)
-        ret.append(size)
-    if with_hash:
-        md5 = entry.get("md5", "")
-        ret.append(md5)
-    ret.append(name)
-    return ret
-
-
-def show_entries(entries, with_color=False, with_size=False, with_hash=False):
-    fmt = _get_formatter(with_color)
-    if with_size or with_hash:
-        colalign = ("right",) if with_size else None
-        ui.table(
-            [
-                _format_entry(
-                    entry,
-                    fmt(entry),
-                    with_size=with_size,
-                    with_hash=with_hash,
-                )
-                for entry in entries
-            ],
-            colalign=colalign,
-        )
-        return
-
-    # NOTE: this is faster than ui.table for very large number of entries
-    ui.write("\n".join(fmt(entry) for entry in entries))
-
-
-class TreePart:
-    Edge = "├── "
-    Line = "│   "
-    Corner = "└── "
-    Blank = "    "
-
-
-def _build_tree_structure(
-    entries, with_color=False, with_size=False, with_hash=False, _depth=0, _prefix=""
-):
-    rows = []
-    fmt = _get_formatter(with_color)
-
-    num_entries = len(entries)
-    for i, (name, entry) in enumerate(entries.items()):
-        entry["path"] = name
-        is_last = i >= num_entries - 1
-        tree_part = ""
-        if _depth > 0:
-            tree_part = TreePart.Corner if is_last else TreePart.Edge
-
-        row = _format_entry(
-            entry,
-            _prefix + tree_part + fmt(entry),
-            with_size=with_size,
-            with_hash=with_hash,
-        )
-        rows.append(row)
-
-        if contents := entry.get("contents"):
-            new_prefix = _prefix
-            if _depth > 0:
-                new_prefix += TreePart.Blank if is_last else TreePart.Line
-            new_rows = _build_tree_structure(
-                contents,
-                with_color=with_color,
-                with_size=with_size,
-                with_hash=with_hash,
-                _depth=_depth + 1,
-                _prefix=new_prefix,
-            )
-            rows.extend(new_rows)
-
-    return rows
-
-
-def show_tree(entries, with_color=False, with_size=False, with_hash=False):
-    rows = _build_tree_structure(
-        entries,
-        with_color=with_color,
-        with_size=with_size,
-        with_hash=with_hash,
-    )
-
-    colalign = ("right",) if with_size else None
-    ui.table(rows, colalign=colalign, preserve_whitespace=True)
+    return [fmt(entry) for entry in entries]
 
 
 class CmdList(CmdBaseNoRepo):
-    def _show_tree(self):
-        from dvc.repo.ls import ls_tree
-
-        entries = ls_tree(
-            self.args.url,
-            self.args.path,
-            rev=self.args.rev,
-            dvc_only=self.args.dvc_only,
-            config=self.args.config,
-            remote=self.args.remote,
-            remote_config=self.args.remote_config,
-            maxdepth=self.args.level,
-        )
-        show_tree(
-            entries,
-            with_color=True,
-            with_size=self.args.size,
-            with_hash=self.args.show_hash,
-        )
-        return 0
-
-    def _show_list(self):
+    def run(self):
         from dvc.repo import Repo
 
-        entries = Repo.ls(
-            self.args.url,
-            self.args.path,
-            rev=self.args.rev,
-            recursive=self.args.recursive,
-            dvc_only=self.args.dvc_only,
-            config=self.args.config,
-            remote=self.args.remote,
-            remote_config=self.args.remote_config,
-            maxdepth=self.args.level,
-        )
-        if self.args.json:
-            ui.write_json(entries)
-        elif entries:
-            show_entries(
-                entries,
-                with_color=True,
-                with_size=self.args.size,
-                with_hash=self.args.show_hash,
-            )
-        return 0
-
-    def run(self):
-        if self.args.tree and self.args.json:
-            raise DvcException("Cannot use --tree and --json options together.")
-
         try:
-            if self.args.tree:
-                return self._show_tree()
-            return self._show_list()
-        except FileNotFoundError:
-            logger.exception("")
-            return 1
+            entries = Repo.ls(
+                self.args.url,
+                self.args.path,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+                dvc_only=self.args.dvc_only,
+            )
+            if self.args.json:
+                ui.write_json(entries)
+            elif entries:
+                entries = _prettify(entries, with_color=True)
+                ui.write("\n".join(entries))
+            return 0
         except DvcException:
             logger.exception("failed to list '%s'", self.args.url)
             return 1
@@ -194,7 +57,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(LIST_HELP, "list"),
         help=LIST_HELP,
-        formatter_class=formatter.RawTextHelpFormatter,
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     list_parser.add_argument("url", help="Location of DVC repository to list")
     list_parser.add_argument(
@@ -203,24 +66,12 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Recursively list files.",
     )
-    list_parser.add_argument(
-        "-T",
-        "--tree",
-        action="store_true",
-        help="Recurse into directories as a tree.",
-    )
-    list_parser.add_argument(
-        "-L",
-        "--level",
-        metavar="depth",
-        type=int,
-        help="Limit the depth of recursion.",
-    )
     list_parser.add_argument(
         "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
     list_parser.add_argument(
         "--json",
+        "--show-json",
         action="store_true",
         help="Show output in JSON format.",
     )
@@ -230,36 +81,6 @@ def add_parser(subparsers, parent_parser):
         help="Git revision (e.g. SHA, branch, tag)",
         metavar="<commit>",
     )
-    list_parser.add_argument(
-        "--config",
-        type=str,
-        help=(
-            "Path to a config file that will be merged with the config "
-            "in the target repository."
-        ),
-    )
-    list_parser.add_argument(
-        "--remote",
-        type=str,
-        help="Remote name to set as a default in the target repository.",
-    ).complete = completion.REMOTE
-    list_parser.add_argument(
-        "--remote-config",
-        type=str,
-        nargs="*",
-        action=DictAction,
-        help=(
-            "Remote config options to merge with a remote's config (default or one "
-            "specified by '--remote') in the target repository."
-        ),
-    )
-    list_parser.add_argument("--size", action="store_true", help="Show sizes.")
-    list_parser.add_argument(
-        "--show-hash",
-        help="Display hash value for each item.",
-        action="store_true",
-        default=False,
-    )
     list_parser.add_argument(
         "path",
         nargs="?",

dvc/commands/ls/ls_colors.py

@@ -20,7 +20,7 @@ class LsColors:
             else:
                 self._codes[code] = color
 
-    def format(self, entry):
+    def format(self, entry):  # noqa: A003
         text = entry["path"]
 
         if entry.get("isout", False) and "out" in self._codes:
@@ -32,9 +32,7 @@ class LsColors:
         if entry.get("isexec", False):
             return self._format(text, code="ex")
 
-        stem, ext = os.path.splitext(text)
-        if not ext and stem.startswith("."):
-            ext = stem
+        _, ext = os.path.splitext(text)
         return self._format(text, ext=ext)
 
     def _format(self, text, code=None, ext=None):

dvc/commands/ls_url.py

@@ -1,47 +1,25 @@
-from dvc.cli import formatter
+import argparse
+import logging
+
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import DictAction, append_doc_link
-from dvc.log import logger
+from dvc.cli.utils import append_doc_link
+from dvc.ui import ui
 
-from .ls import show_entries, show_tree
+from .ls import _prettify
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdListUrl(CmdBaseNoRepo):
-    def _show_tree(self, config):
-        from dvc.fs import parse_external_url
-        from dvc.repo.ls import _ls_tree
-
-        fs, fs_path = parse_external_url(
-            self.args.url, fs_config=self.args.fs_config, config=config
-        )
-        entries = _ls_tree(fs, fs_path, maxdepth=self.args.level)
-        show_tree(entries, with_color=True, with_size=self.args.size)
-        return 0
-
-    def _show_list(self, config):
+    def run(self):
         from dvc.repo import Repo
 
-        entries = Repo.ls_url(
-            self.args.url,
-            recursive=self.args.recursive,
-            maxdepth=self.args.level,
-            fs_config=self.args.fs_config,
-            config=config,
-        )
+        entries = Repo.ls_url(self.args.url, recursive=self.args.recursive)
         if entries:
-            show_entries(entries, with_color=True, with_size=self.args.size)
+            entries = _prettify(entries, with_color=True)
+            ui.write("\n".join(entries))
         return 0
 
-    def run(self):
-        from dvc.config import Config
-
-        config = Config.from_cwd()
-        if self.args.tree:
-            return self._show_tree(config=config)
-        return self._show_list(config=config)
-
 
 def add_parser(subparsers, parent_parser):
     LS_HELP = "List directory contents from URL."
@@ -51,33 +29,15 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(LS_HELP, "list-url"),
         help=LS_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     lsurl_parser.add_argument(
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     lsurl_parser.add_argument(
-        "-R", "--recursive", action="store_true", help="Recursively list files."
-    )
-    lsurl_parser.add_argument(
-        "-T",
-        "--tree",
+        "-R",
+        "--recursive",
         action="store_true",
-        help="Recurse into directories as a tree.",
-    )
-    lsurl_parser.add_argument(
-        "-L",
-        "--level",
-        metavar="depth",
-        type=int,
-        help="Limit the depth of recursion.",
-    )
-    lsurl_parser.add_argument("--size", action="store_true", help="Show sizes.")
-    lsurl_parser.add_argument(
-        "--fs-config",
-        type=str,
-        nargs="*",
-        action=DictAction,
-        help="Config options for the target url.",
+        help="Recursively list files.",
     )
     lsurl_parser.set_defaults(func=CmdListUrl)

dvc/commands/machine.py

@@ -0,0 +1,490 @@
+import argparse
+from typing import Dict, List
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.commands.config import CmdConfig
+from dvc.compare import TabularData
+from dvc.config import ConfigError
+from dvc.exceptions import DvcException
+from dvc.ui import ui
+from dvc.utils import format_link
+
+
+class MachineDisabledError(ConfigError):
+    def __init__(self):
+        super().__init__("Machine feature is disabled")
+
+
+class CmdMachineConfig(CmdConfig):
+    def __init__(self, args):
+        super().__init__(args)
+        if not self.config["feature"].get("machine", False):
+            raise MachineDisabledError
+
+        if getattr(self.args, "name", None):
+            self.args.name = self.args.name.lower()
+
+    def _check_exists(self, conf):
+        if self.args.name not in conf["machine"]:
+            raise ConfigError(f"machine '{self.args.name}' doesn't exist.")
+
+
+class CmdMachineAdd(CmdMachineConfig):
+    def run(self):
+        from dvc.machine import validate_name
+
+        validate_name(self.args.name)
+
+        if self.args.default:
+            ui.write(f"Setting '{self.args.name}' as a default machine.")
+
+        with self.config.edit(self.args.level) as conf:
+            if self.args.name in conf["machine"] and not self.args.force:
+                raise ConfigError(
+                    "machine '{}' already exists. Use `-f|--force` to "
+                    "overwrite it.".format(self.args.name)
+                )
+
+            conf["machine"][self.args.name] = {"cloud": self.args.cloud}
+            if self.args.default:
+                conf["core"]["machine"] = self.args.name
+
+        return 0
+
+
+class CmdMachineRemove(CmdMachineConfig):
+    def run(self):
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            del conf["machine"][self.args.name]
+
+        up_to_level = self.args.level or "repo"
+        # Remove core.machine refs to this machine in any shadowing configs
+        for level in reversed(self.config.LEVELS):
+            with self.config.edit(level) as conf:
+                if conf["core"].get("machine") == self.args.name:
+                    del conf["core"]["machine"]
+
+            if level == up_to_level:
+                break
+
+        return 0
+
+
+class CmdMachineList(CmdMachineConfig):
+    TABLE_COLUMNS = [
+        "name",
+        "cloud",
+        "region",
+        "image",
+        "spot",
+        "spot_price",
+        "instance_hdd_size",
+        "instance_type",
+        "ssh_private",
+        "startup_script",
+    ]
+
+    PRIVATE_COLUMNS = ["ssh_private", "startup_script"]
+
+    def _hide_private(self, conf):
+        for machine in conf:
+            for column in self.PRIVATE_COLUMNS:
+                if column in conf[machine]:
+                    conf[machine][column] = "***"
+
+    def _show_origin(self):
+        levels = [self.args.level] if self.args.level else self.config.LEVELS
+        for level in levels:
+            conf = self.config.read(level)["machine"]
+            if self.args.name:
+                conf = conf.get(self.args.name, {})
+            self._hide_private(conf)
+            prefix = self._config_file_prefix(True, self.config, level)
+            configs = list(self._format_config(conf, prefix))
+            if configs:
+                ui.write("\n".join(configs))
+
+    def _show_table(self):
+        td = TabularData(self.TABLE_COLUMNS, fill_value="-")
+        conf = self.config.read()["machine"]
+        if self.args.name:
+            conf = {self.args.name: conf.get(self.args.name, {})}
+        self._hide_private(conf)
+        for machine, machine_config in conf.items():
+            machine_config["name"] = machine
+            td.row_from_dict(machine_config)
+        td.dropna("cols", "all")
+        td.render()
+
+    def run(self):
+        if self.args.show_origin:
+            self._show_origin()
+        else:
+            self._show_table()
+        return 0
+
+
+class CmdMachineModify(CmdMachineConfig):
+    def run(self):
+        from dvc.config import merge
+
+        with self.config.edit(self.args.level) as conf:
+            merged = self.config.load_config_to_level(self.args.level)
+            merge(merged, conf)
+            self._check_exists(merged)
+
+            if self.args.name not in conf["machine"]:
+                conf["machine"][self.args.name] = {}
+            section = conf["machine"][self.args.name]
+            if self.args.unset:
+                section.pop(self.args.option, None)
+            else:
+                section[self.args.option] = self.args.value
+        return 0
+
+
+class CmdMachineRename(CmdBase):
+    def _check_exists(self, conf):
+        if self.args.name not in conf["machine"]:
+            raise ConfigError(f"machine '{self.args.name}' doesn't exist.")
+
+    def _rename_default(self, conf):
+        if conf["core"].get("machine") == self.args.name:
+            conf["core"]["machine"] = self.args.new
+
+    def _check_before_rename(self):
+        from dvc.machine import validate_name
+
+        validate_name(self.args.new)
+
+        all_config = self.config.load_config_to_level(None)
+        if self.args.new in all_config.get("machine", {}):
+            raise ConfigError(
+                f"Rename failed. Machine '{self.args.new}' already exists."
+            )
+        ui.write(f"Rename machine '{self.args.name}' to '{self.args.new}'.")
+
+    def run(self):
+        self._check_before_rename()
+
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            conf["machine"][self.args.new] = conf["machine"][self.args.name]
+            try:
+                assert self.repo.machine
+                self.repo.machine.rename(self.args.name, self.args.new)
+            except DvcException as error:
+                del conf["machine"][self.args.new]
+                raise ConfigError("terraform rename failed") from error
+            del conf["machine"][self.args.name]
+            self._rename_default(conf)
+
+        up_to_level = self.args.level or "repo"
+        for level in reversed(self.config.LEVELS):
+            if level == up_to_level:
+                break
+            with self.config.edit(level) as level_conf:
+                self._rename_default(level_conf)
+
+        return 0
+
+
+class CmdMachineDefault(CmdMachineConfig):
+    def run(self):
+        if self.args.name is None and not self.args.unset:
+            conf = self.config.read(self.args.level)
+            try:
+                ui.write(conf["core"]["machine"])
+            except KeyError:
+                ui.write("No default machine set")
+                return 1
+        else:
+            with self.config.edit(self.args.level) as conf:
+                if self.args.unset:
+                    conf["core"].pop("machine", None)
+                else:
+                    merged_conf = self.config.load_config_to_level(self.args.level)
+                    if (
+                        self.args.name in conf["machine"]
+                        or self.args.name in merged_conf["machine"]
+                    ):
+                        conf["core"]["machine"] = self.args.name
+                    else:
+                        raise ConfigError(
+                            "default machine must be present in machine list."
+                        )
+        return 0
+
+
+class CmdMachineCreate(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.create(self.args.name)
+        return 0
+
+
+class CmdMachineStatus(CmdBase):
+    INSTANCE_FIELD = ["name", "instance", "status"]
+    SHOWN_FIELD = [
+        "cloud",
+        "instance_ip",
+        "instance_type",
+        "instance_hdd_size",
+        "instance_gpu",
+    ]
+    FILL_VALUE = "-"
+
+    def _add_row(
+        self,
+        name: str,
+        all_status: List[Dict],
+        td: TabularData,
+    ):
+        if not all_status:
+            row = [
+                name,
+                self.FILL_VALUE,
+                "offline",
+            ]  # back to `None` after #7167
+            td.append(row)
+        for i, status in enumerate(all_status, start=1):
+            row = [name, f"num_{i}", "running" if status else "offline"]
+            for field in self.SHOWN_FIELD:
+                value = str(status.get(field, ""))
+                row.append(value)
+            td.append(row)
+
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        td = TabularData(
+            self.INSTANCE_FIELD + self.SHOWN_FIELD, fill_value=self.FILL_VALUE
+        )
+
+        if self.args.name:
+            all_status = list(self.repo.machine.status(self.args.name))
+            self._add_row(self.args.name, all_status, td)
+        else:
+            name_set = set()
+            for level in self.repo.config.LEVELS:
+                conf = self.repo.config.read(level)["machine"]
+                name_set.update(conf.keys())
+            name_list = list(name_set)
+            for name in sorted(name_list):
+                all_status = list(self.repo.machine.status(name))
+                self._add_row(name, all_status, td)
+
+        td.dropna("cols", "all")
+        td.render()
+        return 0
+
+
+class CmdMachineDestroy(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.destroy(self.args.name)
+        return 0
+
+
+class CmdMachineSsh(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.run_shell(self.args.name)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):  # noqa: PLR0915
+    from dvc.commands.config import parent_config_parser
+
+    machine_HELP = "Set up and manage cloud machines."
+    machine_parser = subparsers.add_parser(
+        "machine",
+        parents=[parent_parser],
+        description=append_doc_link(machine_HELP, "machine"),
+        # NOTE: suppress help during development to hide command
+        # help=machine_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    machine_subparsers = machine_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc machine CMD --help` for command-specific help.",
+    )
+
+    fix_subparsers(machine_subparsers)
+
+    machine_ADD_HELP = "Add a new data machine."
+    machine_add_parser = machine_subparsers.add_parser(
+        "add",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_ADD_HELP, "machine/add"),
+        help=machine_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_add_parser.add_argument("name", help="Name of the machine")
+    machine_add_parser.add_argument(
+        "cloud",
+        help="Machine cloud. See full list of supported clouds at {}".format(
+            format_link(
+                "https://github.com/iterative/terraform-provider-iterative#machine"
+            )
+        ),
+    )
+    machine_add_parser.add_argument(
+        "-d",
+        "--default",
+        action="store_true",
+        default=False,
+        help="Set as default machine.",
+    )
+    machine_add_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force overwriting existing configs",
+    )
+    machine_add_parser.set_defaults(func=CmdMachineAdd)
+
+    machine_DEFAULT_HELP = "Set/unset the default machine."
+    machine_default_parser = machine_subparsers.add_parser(
+        "default",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_DEFAULT_HELP, "machine/default"),
+        help=machine_DEFAULT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_default_parser.add_argument("name", nargs="?", help="Name of the machine")
+    machine_default_parser.add_argument(
+        "-u",
+        "--unset",
+        action="store_true",
+        default=False,
+        help="Unset default machine.",
+    )
+    machine_default_parser.set_defaults(func=CmdMachineDefault)
+
+    machine_LIST_HELP = "List the configuration of one/all machines."
+    machine_list_parser = machine_subparsers.add_parser(
+        "list",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_LIST_HELP, "machine/list"),
+        help=machine_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_list_parser.add_argument(
+        "--show-origin",
+        default=False,
+        action="store_true",
+        help="Show the source file containing each config value.",
+    )
+    machine_list_parser.add_argument(
+        "name",
+        nargs="?",
+        type=str,
+        help="name of machine to specify",
+    )
+    machine_list_parser.set_defaults(func=CmdMachineList)
+    machine_MODIFY_HELP = "Modify the configuration of an machine."
+    machine_modify_parser = machine_subparsers.add_parser(
+        "modify",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_MODIFY_HELP, "machine/modify"),
+        help=machine_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_modify_parser.add_argument("name", help="Name of the machine")
+    machine_modify_parser.add_argument("option", help="Name of the option to modify.")
+    machine_modify_parser.add_argument(
+        "value", nargs="?", help="(optional) Value of the option."
+    )
+    machine_modify_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    machine_modify_parser.set_defaults(func=CmdMachineModify)
+
+    machine_RENAME_HELP = "Rename a machine "
+    machine_rename_parser = machine_subparsers.add_parser(
+        "rename",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_RENAME_HELP, "remote/rename"),
+        help=machine_RENAME_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_rename_parser.add_argument("name", help="Machine to be renamed")
+    machine_rename_parser.add_argument("new", help="New name of the machine")
+    machine_rename_parser.set_defaults(func=CmdMachineRename)
+
+    machine_REMOVE_HELP = "Remove an machine."
+    machine_remove_parser = machine_subparsers.add_parser(
+        "remove",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_REMOVE_HELP, "machine/remove"),
+        help=machine_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_remove_parser.add_argument("name", help="Name of the machine to remove.")
+    machine_remove_parser.set_defaults(func=CmdMachineRemove)
+
+    machine_CREATE_HELP = "Create and start a machine instance."
+    machine_create_parser = machine_subparsers.add_parser(
+        "create",
+        parents=[parent_parser],
+        description=append_doc_link(machine_CREATE_HELP, "machine/create"),
+        help=machine_CREATE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_create_parser.add_argument("name", help="Name of the machine to create.")
+    machine_create_parser.set_defaults(func=CmdMachineCreate)
+
+    machine_STATUS_HELP = "List the status of running instances for one/all machines."
+    machine_status_parser = machine_subparsers.add_parser(
+        "status",
+        parents=[parent_parser],
+        description=append_doc_link(machine_STATUS_HELP, "machine/status"),
+        help=machine_STATUS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_status_parser.add_argument(
+        "name", nargs="?", help="(optional) Name of the machine."
+    )
+    machine_status_parser.set_defaults(func=CmdMachineStatus)
+
+    machine_DESTROY_HELP = "Destroy an machine instance."
+    machine_destroy_parser = machine_subparsers.add_parser(
+        "destroy",
+        parents=[parent_parser],
+        description=append_doc_link(machine_DESTROY_HELP, "machine/destroy"),
+        help=machine_DESTROY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_destroy_parser.add_argument(
+        "name", help="Name of the machine instance to destroy."
+    )
+    machine_destroy_parser.set_defaults(func=CmdMachineDestroy)
+
+    machine_SSH_HELP = "Connect to a machine via SSH."
+    machine_ssh_parser = machine_subparsers.add_parser(
+        "ssh",
+        parents=[parent_parser],
+        description=append_doc_link(machine_SSH_HELP, "machine/ssh"),
+        help=machine_SSH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_ssh_parser.add_argument(
+        "name", help="Name of the machine instance to connect to."
+    )
+    machine_ssh_parser.set_defaults(func=CmdMachineSsh)

dvc/commands/metrics.py

@@ -1,11 +1,14 @@
-from dvc.cli import completion, formatter
+import argparse
+import logging
+
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.log import logger
+from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
 from dvc.ui import ui
 from dvc.utils.serialize import encode_exception
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 DEFAULT_PRECISION = 5
@@ -17,25 +20,17 @@ class CmdMetricsBase(CmdBase):
 
 class CmdMetricsShow(CmdMetricsBase):
     def run(self):
-        from dvc.repo.metrics.show import to_relpath
-        from dvc.utils import errored_revisions
-
-        metrics = self.repo.metrics.show(
-            self.args.targets,
-            all_branches=self.args.all_branches,
-            all_tags=self.args.all_tags,
-            all_commits=self.args.all_commits,
-        )
-        metrics = {
-            k: to_relpath(self.repo.fs, self.repo.root_dir, v)
-            for k, v in metrics.items()
-        }
-
-        if errored := errored_revisions(metrics):
-            ui.error_write(
-                "DVC failed to load some metrics for following revisions:"
-                f" '{', '.join(errored)}'."
+        try:
+            metrics = self.repo.metrics.show(
+                self.args.targets,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                recursive=self.args.recursive,
             )
+        except DvcException:
+            logger.exception("")
+            return 1
 
         if self.args.json:
             ui.write_json(metrics, default=encode_exception)
@@ -57,26 +52,17 @@ class CmdMetricsShow(CmdMetricsBase):
 
 class CmdMetricsDiff(CmdMetricsBase):
     def run(self):
-        import os
-        from os.path import relpath
-
-        diff_result = self.repo.metrics.diff(
-            a_rev=self.args.a_rev,
-            b_rev=self.args.b_rev,
-            targets=self.args.targets,
-            all=self.args.all,
-        )
-
-        errored = [rev for rev, err in diff_result.get("errors", {}).items() if err]
-        if errored:
-            ui.error_write(
-                "DVC failed to load some metrics for following revisions:"
-                f" '{', '.join(errored)}'."
+        try:
+            diff = self.repo.metrics.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                targets=self.args.targets,
+                recursive=self.args.recursive,
+                all=self.args.all,
             )
-
-        start = relpath(os.getcwd(), self.repo.root_dir)
-        diff = diff_result.get("diff", {})
-        diff = {relpath(path, start): result for path, result in diff.items()}
+        except DvcException:
+            logger.exception("failed to show metrics diff")
+            return 1
 
         if self.args.json:
             ui.write_json(diff)
@@ -105,22 +91,23 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(METRICS_HELP, "metrics"),
         help=METRICS_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     metrics_subparsers = metrics_parser.add_subparsers(
         dest="cmd",
         help="Use `dvc metrics CMD --help` to display command-specific help.",
-        required=True,
     )
 
+    fix_subparsers(metrics_subparsers)
+
     METRICS_SHOW_HELP = "Print metrics, with optional formatting."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
         parents=[parent_parser],
         description=append_doc_link(METRICS_SHOW_HELP, "metrics/show"),
         help=METRICS_SHOW_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_show_parser.add_argument(
         "targets",
@@ -153,12 +140,14 @@ def add_parser(subparsers, parent_parser):
     )
     metrics_show_parser.add_argument(
         "--json",
+        "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
     metrics_show_parser.add_argument(
         "--md",
+        "--show-md",
         action="store_true",
         default=False,
         dest="markdown",
@@ -194,17 +183,13 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(METRICS_DIFF_HELP, "metrics/diff"),
         help=METRICS_DIFF_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_diff_parser.add_argument(
-        "a_rev",
-        nargs="?",
-        help="Old Git commit to compare (defaults to HEAD)",
-        default="HEAD",
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
     )
     metrics_diff_parser.add_argument(
         "b_rev",
-        default="workspace",
         nargs="?",
         help="New Git commit to compare (defaults to the current workspace)",
     )
@@ -238,12 +223,14 @@ def add_parser(subparsers, parent_parser):
     )
     metrics_diff_parser.add_argument(
         "--json",
+        "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
     metrics_diff_parser.add_argument(
         "--md",
+        "--show-md",
         action="store_true",
         default=False,
         dest="markdown",

dvc/commands/move.py

@@ -1,10 +1,12 @@
-from dvc.cli import completion, formatter
+import argparse
+import logging
+
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdMove(CmdBase):
@@ -28,11 +30,10 @@ def add_parser(subparsers, parent_parser):
 
     move_parser = subparsers.add_parser(
         "move",
-        aliases=["mv"],
         parents=[parent_parser],
         description=append_doc_link(MOVE_DESCRIPTION, "move"),
         help=MOVE_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
         "src", help="Source path to a data file or directory."

dvc/commands/params.py

@@ -1,37 +1,30 @@
-from dvc.cli import completion, formatter
+import argparse
+import logging
+
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.log import logger
+from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdParamsDiff(CmdBase):
     UNINITIALIZED = True
 
     def run(self):
-        import os
-        from os.path import relpath
-
-        diff_result = self.repo.params.diff(
-            a_rev=self.args.a_rev,
-            b_rev=self.args.b_rev,
-            targets=self.args.targets,
-            all=self.args.all,
-            deps_only=self.args.deps,
-        )
-
-        errored = [rev for rev, err in diff_result.get("errors", {}).items() if err]
-        if errored:
-            ui.error_write(
-                "DVC failed to load some metrics for following revisions:"
-                f" '{', '.join(errored)}'."
+        try:
+            diff = self.repo.params.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                targets=self.args.targets,
+                all=self.args.all,
+                deps=self.args.deps,
             )
-
-        start = relpath(os.getcwd(), self.repo.root_dir)
-        diff = diff_result.get("diff", {})
-        diff = {relpath(path, start): result for path, result in diff.items()}
+        except DvcException:
+            logger.exception("failed to show params diff")
+            return 1
 
         if self.args.json:
             ui.write_json(diff)
@@ -59,15 +52,16 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(PARAMS_HELP, "params"),
         help=PARAMS_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     params_subparsers = params_parser.add_subparsers(
         dest="cmd",
         help="Use `dvc params CMD --help` to display command-specific help.",
-        required=True,
     )
 
+    fix_subparsers(params_subparsers)
+
     PARAMS_DIFF_HELP = (
         "Show changes in params between commits in the DVC repository, or "
         "between a commit and the workspace."
@@ -77,17 +71,13 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(PARAMS_DIFF_HELP, "params/diff"),
         help=PARAMS_DIFF_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     params_diff_parser.add_argument(
-        "a_rev",
-        nargs="?",
-        default="HEAD",
-        help="Old Git commit to compare (defaults to HEAD)",
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
     )
     params_diff_parser.add_argument(
         "b_rev",
-        default="workspace",
         nargs="?",
         help="New Git commit to compare (defaults to the current workspace)",
     )
@@ -115,12 +105,14 @@ def add_parser(subparsers, parent_parser):
     )
     params_diff_parser.add_argument(
         "--json",
+        "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
     params_diff_parser.add_argument(
         "--md",
+        "--show-md",
         action="store_true",
         default=False,
         dest="markdown",

dvc/commands/plots.py

@@ -1,14 +1,14 @@
 import argparse
+import logging
 import os
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Dict, List, Optional
 
 from funcy import compact, first, get_in
 
-from dvc.cli import completion, formatter
+from dvc.cli import completion
 from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.log import logger
 from dvc.ui import ui
 from dvc.utils import format_link
 
@@ -16,30 +16,39 @@ if TYPE_CHECKING:
     from dvc.render.match import RendererWithErrors
 
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 def _show_json(
-    renderers_with_errors: list["RendererWithErrors"],
+    renderers_with_errors: List["RendererWithErrors"],
     split=False,
-    errors: Optional[dict[str, Exception]] = None,
+    errors: Optional[Dict[str, Exception]] = None,
 ):
     from dvc.render.convert import to_json
     from dvc.utils.serialize import encode_exception
 
-    all_errors: list[dict] = []
+    all_errors: List[Dict] = []
     data = {}
 
     for renderer, src_errors, def_errors in renderers_with_errors:
         name = renderer.name
         data[name] = to_json(renderer, split)
         all_errors.extend(
-            {"name": name, "rev": rev, "source": source, **encode_exception(e)}
+            {
+                "name": name,
+                "rev": rev,
+                "source": source,
+                **encode_exception(e),
+            }
             for rev, per_rev_src_errors in src_errors.items()
             for source, e in per_rev_src_errors.items()
         )
         all_errors.extend(
-            {"name": name, "rev": rev, **encode_exception(e)}
+            {
+                "name": name,
+                "rev": rev,
+                **encode_exception(e),
+            }
             for rev, e in def_errors.items()
         )
 
@@ -50,6 +59,49 @@ def _show_json(
     ui.write_json(compact({"errors": all_errors, "data": data}), highlight=False)
 
 
+def _adjust_vega_renderers(renderers):
+    from dvc.render import REVISION_FIELD, VERSION_FIELD
+    from dvc_render import VegaRenderer
+
+    for r in renderers:
+        if isinstance(r, VegaRenderer):
+            if _data_versions_count(r) > 1:
+                summary = _summarize_version_infos(r)
+                for dp in r.datapoints:
+                    vi = dp.pop(VERSION_FIELD, {})
+                    keys = list(vi.keys())
+                    for key in keys:
+                        if not (len(summary.get(key, set())) > 1):
+                            vi.pop(key)
+                    if vi:
+                        dp["rev"] = "::".join(vi.values())
+            else:
+                for dp in r.datapoints:
+                    dp[REVISION_FIELD] = dp[VERSION_FIELD]["revision"]
+                    dp.pop(VERSION_FIELD, {})
+
+
+def _summarize_version_infos(renderer):
+    from collections import defaultdict
+
+    from dvc.render import VERSION_FIELD
+
+    result = defaultdict(set)
+
+    for dp in renderer.datapoints:
+        for key, value in dp.get(VERSION_FIELD, {}).items():
+            result[key].add(value)
+    return dict(result)
+
+
+def _data_versions_count(renderer):
+    from itertools import product
+
+    summary = _summarize_version_infos(renderer)
+    x = product(summary.get("filename", {None}), summary.get("field", {None}))
+    return len(set(x))
+
+
 class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
@@ -72,7 +124,7 @@ class CmdPlots(CmdBase):
                 html_template_path = os.path.join(self.repo.dvc_dir, html_template_path)
         return html_template_path
 
-    def run(self) -> int:  # noqa: C901, PLR0911
+    def run(self) -> int:  # noqa: C901, PLR0911, PLR0912
         from pathlib import Path
 
         from dvc.render.match import match_defs_renderers
@@ -92,7 +144,10 @@ class CmdPlots(CmdBase):
                 return 1
 
         try:
-            plots_data = self._func(targets=self.args.targets, props=self._props())
+            plots_data = self._func(
+                targets=self.args.targets,
+                props=self._props(),
+            )
 
             if not plots_data and not self.args.json:
                 ui.error_write(
@@ -120,10 +175,11 @@ class CmdPlots(CmdBase):
                 return 0
 
             renderers = [r.renderer for r in renderers_with_errors]
+            _adjust_vega_renderers(renderers)
             if self.args.show_vega:
                 renderer = first(filter(lambda r: r.TYPE == "vega", renderers))
                 if renderer:
-                    ui.write_json(renderer.get_filled_template())
+                    ui.write_json(renderer.get_filled_template(as_string=False))
                 return 0
 
             output_file: Path = (Path.cwd() / out).resolve() / "index.html"
@@ -211,14 +267,15 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(PLOTS_HELP, "plots"),
         help=PLOTS_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_subparsers = plots_parser.add_subparsers(
         dest="cmd",
         help="Use `dvc plots CMD --help` to display command-specific help.",
-        required=True,
     )
 
+    fix_subparsers(plots_subparsers)
+
     SHOW_HELP = (
         "Generate plots from target files or from `plots` definitions in `dvc.yaml`."
     )
@@ -227,7 +284,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(SHOW_HELP, "plots/show"),
         help=SHOW_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_show_parser.add_argument(
         "targets",
@@ -250,7 +307,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(PLOTS_DIFF_HELP, "plots/diff"),
         help=PLOTS_DIFF_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_diff_parser.add_argument(
         "--targets",
@@ -286,7 +343,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(PLOTS_MODIFY_HELP, "plots/modify"),
         help=PLOTS_MODIFY_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_modify_parser.add_argument(
         "target",
@@ -307,7 +364,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(TEMPLATES_HELP, "plots/templates"),
         help=TEMPLATES_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_templates_parser.add_argument(
         "template",
@@ -373,6 +430,7 @@ def _add_ui_arguments(parser):
     )
     parser.add_argument(
         "--json",
+        "--show-json",
         action="store_true",
         default=False,
         help=argparse.SUPPRESS,

dvc/commands/queue/__init__.py

@@ -1,8 +1,16 @@
-from dvc.cli import formatter
-from dvc.cli.utils import append_doc_link
+import argparse
+
+from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.commands.queue import kill, logs, remove, start, status, stop
 
-SUB_COMMANDS = [start, stop, status, logs, remove, kill]
+SUB_COMMANDS = [
+    start,
+    stop,
+    status,
+    logs,
+    remove,
+    kill,
+]
 
 
 def add_parser(subparsers, parent_parser):
@@ -12,15 +20,15 @@ def add_parser(subparsers, parent_parser):
         "queue",
         parents=[parent_parser],
         description=append_doc_link(QUEUE_HELP, "queue"),
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
         help=QUEUE_HELP,
     )
 
     queue_subparsers = queue_parser.add_subparsers(
         dest="cmd",
         help="Use `dvc queue CMD --help` to display command-specific help.",
-        required=True,
     )
 
+    fix_subparsers(queue_subparsers)
     for cmd in SUB_COMMANDS:
         cmd.add_parser(queue_subparsers, parent_parser)

dvc/commands/queue/kill.py

@@ -1,9 +1,10 @@
-from dvc.cli import formatter
+import argparse
+import logging
+
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdQueueKill(CmdBase):
@@ -26,7 +27,7 @@ def add_parser(queue_subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(QUEUE_KILL_HELP, "queue/kill"),
         help=QUEUE_KILL_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     queue_kill_parser.add_argument(
         "-f",

dvc/commands/queue/logs.py

@@ -1,9 +1,10 @@
-from dvc.cli import formatter
+import argparse
+import logging
+
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdQueueLogs(CmdBase):
@@ -28,7 +29,7 @@ def add_parser(queue_subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(QUEUE_LOGS_HELP, "queue/logs"),
         help=QUEUE_LOGS_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     queue_logs_parser.add_argument(
         "-e",
@@ -45,5 +46,9 @@ def add_parser(queue_subparsers, parent_parser):
         ),
         action="store_true",
     )
-    queue_logs_parser.add_argument("task", help="Task to show.", metavar="<task>")
+    queue_logs_parser.add_argument(
+        "task",
+        help="Task to show.",
+        metavar="<task>",
+    )
     queue_logs_parser.set_defaults(func=CmdQueueLogs)

dvc/commands/queue/remove.py

@@ -1,11 +1,12 @@
-from dvc.cli import formatter
+import argparse
+import logging
+
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.exceptions import InvalidArgumentError
-from dvc.log import logger
 from dvc.ui import ui
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdQueueRemove(CmdBase):
@@ -61,7 +62,7 @@ def add_parser(queue_subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(QUEUE_REMOVE_HELP, "queue/remove"),
         help=QUEUE_REMOVE_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     queue_remove_parser.add_argument(
         "--all",