dvc: bump to 1.11.7

(cherry picked from commit 84a3a59bc7)

Co-authored-by: João Rafael <[email protected]>

.codecov.yml

@@ -0,0 +1,10 @@
+comment: false
+coverage:
+  status:
+    project:
+      default:
+        target: 85
+        threshold: 10%
+    patch:
+      default:
+        threshold: 25%

.coveragerc

@@ -0,0 +1,15 @@
+[run]
+branch = True
+source = dvc
+omit =
+    *__main__.py
+
+[report]
+exclude_lines =
+    if __name__ == .__main__.:
+    if typing.TYPE_CHECKING:
+    if TYPE_CHECKING:
+    pass
+    raise NotImplementedError
+    # pragma: no cover
+    #  pragma: no cover

.dvc/.gitignore

@@ -1,3 +1,12 @@
+/state
+/lock
 /config.local
-/tmp
+/updater
+/updater.lock
+/state-journal
+/state-wal
 /cache
+/pkg
+/repos
+/tmp
+/experiments

.dvc/config

@@ -0,0 +1,4 @@
+['remote "dvc"']
+url = https://remote.dvc.org/dvc
+[core]
+remote = dvc

.dvcignore

@@ -1,3 +0,0 @@
-/tests
-/dvc
-/.github

.git-blame-ignore-revs

@@ -1,10 +0,0 @@
-# The following commits will be ignored by the GitHub blame view.
-# You can configure `git blame` to use this file as the default ignore file:
-#     git config blame.ignoreRevsFile .git-blame-ignore-revs
-# or, you can also manually ignore commits from this file by doing:
-#     git blame --ignore-revs-file .git-blame-ignore-revs <filepath>
-# see the `blame.markIgnoredLines` and `blame.markUnblamableLines` options as well.
-# Refer to: https://git-scm.com/docs/git-blame#Documentation/git-blame.txt---ignore-revs-fileltfilegt
-
-# update black to 2023 stable style and change line-length to 88
-a86470ccbc693d6f0a4f8066cbd1357e5191c4ff

.git_archival.txt

@@ -1,4 +0,0 @@
-node: $Format:%H$
-node-date: $Format:%cI$
-describe-name: $Format:%(describe:tags=true)$
-ref-names: $Format:%D$

.gitattributes

@@ -1 +0,0 @@
-.git_archival.txt  export-subst

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+patreon: DVCorg

.github/ISSUE_TEMPLATE/bug_report.md

@@ -3,62 +3,16 @@ name: "\U0001F41B Bug Report"
 about: Create a bug report to help us improve DVC
 ---
 
-# Bug Report
+## Bug Report
 
-<!--
-## Issue name
+### Please provide information about your setup
 
-Issue names must follow the pattern `command: description` where the command is the dvc command that you are trying to run. The description should describe the consequence of the bug.
-
-Example: `repro: doesn't detect input changes`
--->
-
-## Description
-
-<!--
-A clear and concise description of what the bug is.
--->
-
-### Reproduce
-
-<!--
-Step list of how to reproduce the bug
--->
-
-<!--
-Example:
-
-1. dvc init
-2. Copy dataset.zip to the directory
-3. dvc add dataset.zip
-4. dvc run -d dataset.zip -o model ./train.sh
-5. modify dataset.zip
-6. dvc repro
--->
-
-### Expected
-
-<!--
-A clear and concise description of what you expect to happen.
--->
-
-### Environment information
-
-<!--
-This is required to ensure that we can reproduce the bug.
--->
-
-**Output of `dvc doctor`:**
+**Output of `dvc version`:**
 
 ```console
-$ dvc doctor
+$ dvc version
 ```
 
 **Additional Information (if any):**
 
-<!--
-Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
-
 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
-If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
--->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,3 +2,4 @@
 name: "\U0001F680 Feature Request"
 about: Suggest an idea for this project
 ---
+

.github/codecov.yml

@@ -1,12 +0,0 @@
-codecov:
-  notify:
-    wait_for_ci: true
-coverage:
-  status:
-    project:
-      default:
-        target: auto
-        threshold: 2%
-    patch: off
-github_checks:
-  annotations: false

.github/dependabot.yml

@@ -1,16 +0,0 @@
-version: 2
-
-updates:
-  - directory: "/"
-    package-ecosystem: "pip"
-    schedule:
-      interval: "daily"
-    labels:
-      - "maintenance"
-
-  - directory: "/"
-    package-ecosystem: "github-actions"
-    schedule:
-      interval: "daily"
-    labels:
-      - "maintenance"

.github/release-drafter.yml

@@ -0,0 +1,49 @@
+# Config for https://github.com/apps/release-drafter
+name-template: "$RESOLVED_VERSION 🦉 "
+tag-template: "$RESOLVED_VERSION"
+categories:
+  - title: "🚀 New Features and Enhancements "
+    labels:
+      - "feature"
+      - "enhancement"
+      - "ui"
+  - title: "💣 Breaking Changes "
+    labels:
+      - "breaking-change"
+  - title: "🏇 Optimizations "
+    labels:
+      - "optimize"
+      - "performance"
+  - title: "🐛 Bug Fixes "
+    labels:
+      - "bugfix"
+      - "bug"
+  - title: "🔨 Maintenance "
+    labels:
+      - "maintenance"
+      - "refactoring"
+      - "chore"
+      - "build"
+      - "ci"
+change-template: "- $TITLE (#$NUMBER) @$AUTHOR"
+version-resolver:
+  major:
+    labels:
+      - "major"
+  minor:
+    labels:
+      - "minor"
+  patch:
+    labels:
+      - "patch"
+  default: patch
+exclude-labels:
+  - "skip-changelog"
+template: |
+  > Refer to https://dvc.org/doc/install for installation instructions.
+
+  ## Changes
+
+  $CHANGES
+
+  Thanks again to $CONTRIBUTORS for the contributions! 🎉

.github/release.yml

@@ -1,16 +0,0 @@
-changelog:
-  exclude:
-    labels: ["skip-changelog"]
-  categories:
-    - title: 🛠 Breaking Changes
-      labels: ["breaking-change"]
-    - title: 🚀 New Features and Enhancements
-      labels: ["feature", "enhancement", "ui"]
-    - title: ⚡ Optimizations
-      labels: ["optimize", "performance"]
-    - title: 🐛 Bug Fixes
-      labels: ["bugfix", "bug"]
-    - title: 🔨 Maintenance
-      labels: ["maintenance", "refactoring", "chore", "build", "ci"]
-    - title: Other Changes
-      labels: ["*"]

.github/workflows/benchmarks.yaml

@@ -1,44 +0,0 @@
-name: benchmarks
-on: [pull_request, workflow_dispatch]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  FORCE_COLOR: "1"
-  PY_COLORS: "1"
-
-jobs:
-  bench:
-    name: run benchmarks
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-python@v6
-        with:
-            python-version: "3.13"
-
-      - uses: actions/checkout@v5
-        with:
-          ref: ${{ github.event.pull_request.base.sha }}
-          fetch-depth: 0
-
-      - uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: true
-          cache-dependency-glob: pyproject.toml
-
-      - run: uv pip install '.[tests]' --system
-      - name: run benchmarks on base branch
-        run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
-
-      - uses: actions/checkout@v5
-        with:
-          fetch-depth: 0
-          clean: false
-      - run: uv pip install '.[tests]' --system
-      - name: run benchmarks for PR
-        run: >
-          pytest --benchmark-compare --benchmark-compare-fail=min:5%
-          --benchmark-group-by name
-          dvc/testing/benchmarks/ -k 'test_init or test_help'

.github/workflows/build.yaml

@@ -1,95 +0,0 @@
-name: Build and upload package
-on:
-  push:
-    branches:
-    - main
-  release:
-    types:
-    - published
-  workflow_dispatch:
-
-env:
-  FORCE_COLOR: 1
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    name: Build & Verify package
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v5
-      with:
-        fetch-depth: 0
-
-    - uses: actions/setup-python@v6
-      with:
-        python-version: 3.13
-
-    - uses: astral-sh/setup-uv@v6
-
-    - name: Omit local version for Test PyPI upload
-      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-      run: echo SETUPTOOLS_SCM_OVERRIDES_FOR_DVC='{local_scheme="no-local-version"}' >> $GITHUB_ENV
-
-    - name: Build Python Package
-      run: |
-        echo 'PKG = "pip"'>dvc/_build.py
-        uv build
-
-    - name: Check dist
-      run: uv tool run twine check --strict dist/*
-
-    - uses: actions/upload-artifact@v4
-      with:
-        name: Packages
-        path: dist/
-        if-no-files-found: error
-
-  test-pypi-publish:
-    name: Publish dev package to test.pypi.org
-    runs-on: ubuntu-latest
-    needs: build
-    if: ${{ github.event.action == 'published' || (github.repository == 'iterative/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
-
-    environment:
-      name: test-pypi
-      url: https://test.pypi.org/p/dvc/${{ github.event.release.tag_name }}
-
-    permissions:
-      id-token: write
-
-    steps:
-    - uses: actions/download-artifact@v5
-      with:
-        name: Packages
-        path: dist
-
-    - name: Upload package to Test PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        repository-url: https://test.pypi.org/legacy/
-        skip-existing: true
-
-  pypi-publish:
-    name: Publish released package to pypi.org
-    runs-on: ubuntu-latest
-    needs: build
-    if: github.event.action == 'published'
-
-    environment:
-      name: pypi
-      url: https://pypi.org/p/dvc/${{ github.event.release.tag_name }}
-
-    permissions:
-      id-token: write
-
-    steps:
-    - uses: actions/download-artifact@v5
-      with:
-        name: Packages
-        path: dist
-
-    - name: Upload package to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/codeql.yml

@@ -1,40 +0,0 @@
-name: "CodeQL"
-
-on:
-  push:
-    branches: ["main"]
-  pull_request:
-    branches: ["main"]
-  schedule:
-    - cron: '32 19 * * 2'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ['python']
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v5
-
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
-      with:
-        languages: ${{ matrix.language }}
-        queries: security-extended
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3

.github/workflows/packages.yaml

@@ -0,0 +1,129 @@
+name: Build packages
+on:
+  push: {}
+  release:
+    types: [released]
+jobs:
+  linux:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
+      with:
+        python-version: 3.7
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Build
+      run: ./scripts/build_posix.sh
+    - name: Publish deb
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_content_type: binary/octet-stream
+    - name: Publish rpm
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_content_type: binary/octet-stream
+  mac:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
+      with:
+        python-version: 3.7
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Build
+      run: ./scripts/build_posix.sh
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_name: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_content_type: binary/octet-stream
+  windows:
+    runs-on: windows-2019
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
+      with:
+        python-version: 3.7
+    - name: Build
+      run: scripts\build_windows.cmd
+      shell: cmd
+    - name: Publish
+      if: github.event_name == 'release'
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      uses: actions/[email protected]
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}.exe
+        asset_name: dvc-${{ github.event.release.tag_name }}.exe
+        asset_content_type: binary/octet-stream
+  python:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
+      with:
+        python-version: 3.7
+    - name: Install
+      run: pip install ".[all,tests]"
+    - name: Build packages
+      run: ./scripts/build_package.sh
+    - name: Publish packages to PyPI
+      if: github.event_name == 'release'
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_TOKEN }}
+  snap:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/[email protected]
+    - name: Build the package
+      uses: snapcore/action-build@v1
+      id: snapcraft
+    - name: Test the package
+      run: |
+        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
+        dvc version
+        sudo snap remove dvc
+    - name: Publish to the edge channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'push'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: edge,v1/edge
+    - name: Publish to the stable channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'release'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: stable,v1/stable

.github/workflows/plugin_tests.yaml

@@ -1,55 +0,0 @@
-name: Remote Plugin Tests
-
-on:
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  tests:
-    timeout-minutes: 45
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest]
-        pyv: ["3.13"]
-        plugin: ["dvc-s3"]
-
-    steps:
-    - uses: actions/checkout@v5
-      with:
-        path: dvc
-
-    - uses: actions/checkout@v5
-      with:
-        repository: iterative/${{ matrix.plugin }}
-        ref: main
-        path: ${{ matrix.plugin }}
-
-    - name: Set up Python
-      uses: actions/setup-python@v6
-      with:
-        python-version: ${{ matrix.pyv }}
-
-    - uses: astral-sh/setup-uv@v6
-      with:
-        enable-cache: true
-        cache-dependency-glob: "**/pyproject.toml"
-        cache-suffix: ${{ matrix.pyv }}
-
-    - name: Install plugin + DVC@PR
-      run: |
-        uv pip install "./dvc[testing]" --system
-        uv pip install -e "./${{ matrix.plugin }}[tests]" --system
-
-    - name: Run plugin tests
-      timeout-minutes: 15
-      working-directory: ${{ matrix.plugin }}
-      run: pytest -v -n=auto

.github/workflows/tests.yaml

@@ -1,167 +1,82 @@
 name: Tests
 
 on:
-  push:
-    branches: [main]
-  pull_request:
-  schedule:
-    - cron: '5 1 * * *'  # every day at 01:05
-  workflow_dispatch:
+  - pull_request
+  - push
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
+env:
+  DVC_TEST: "true"
+  HOMEBREW_NO_AUTO_UPDATE: 1
+  SHELL: /bin/bash
+  GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
 
 jobs:
   lint:
-    timeout-minutes: 10
-    runs-on: ${{ matrix.os }}
-    env:
-       DVC_NO_ANALYTICS: true
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v2.3.2
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
       with:
-        fetch-depth: 0
-    - name: Set up Python
-      uses: actions/setup-python@v6
-      with:
-        python-version: '3.13'
-    - uses: astral-sh/setup-uv@v6
-      with:
-        enable-cache: true
-        cache-dependency-glob: pyproject.toml
-    - run: uv pip install -e ".[dev]" --system
-      env:
-        UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
-
-    - name: Cache mypy
-      uses: actions/cache@v4
-      with:
-        path: .mypy_cache
-        key: mypy-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
-    - uses: pre-commit/[email protected]
-
+        python-version: 3.8
+    - name: Install requirements
+      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+    - name: Check README
+      run: python setup.py checkdocs
+    - name: Check formatting
+      run: pre-commit run --all-files
   tests:
-    timeout-minutes: 20
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
-        pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        os: [ubuntu-18.04, windows-2019, macos-10.15]
+        pyv: ["3.6", "3.7", "3.8", "3.9"]
         include:
-        - os: windows-latest
-          pyv: "3.9"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
-        - os: windows-latest
-          pyv: "3.9"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
-        - os: windows-latest
-          pyv: "3.9"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
-        - os: windows-latest
-          pyv: "3.9"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
-        - os: windows-latest
-          pyv: "3.10"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
-        - os: windows-latest
-          pyv: "3.10"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
-        - os: windows-latest
-          pyv: "3.10"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
-        - os: windows-latest
-          pyv: "3.10"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
-        - os: windows-latest
-          pyv: "3.11"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
-        - os: windows-latest
-          pyv: "3.11"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
-        - os: windows-latest
-          pyv: "3.11"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
-        - os: windows-latest
-          pyv: "3.11"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
-        - os: windows-latest
-          pyv: "3.12"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
-        - os: windows-latest
-          pyv: "3.12"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
-        - os: windows-latest
-          pyv: "3.12"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
-        - os: windows-latest
-          pyv: "3.12"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
-        - os: windows-latest
-          pyv: "3.13"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
-        - os: windows-latest
-          pyv: "3.13"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
-        - os: windows-latest
-          pyv: "3.13"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
-        - os: windows-latest
-          pyv: "3.13"
-          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+          - os: "macos-11.0"
+            pyv: "3.9"
+        exclude:
+          - os: macos-10.15
+            pyv: "3.9"
+
     steps:
-    - uses: actions/checkout@v5
-      with:
-        fetch-depth: 0
+    - uses: actions/checkout@v2.3.2
     - name: Set up Python
-      uses: actions/setup-python@v6
+      uses: actions/setup-python@v2.1.4
       with:
         python-version: ${{ matrix.pyv }}
-    - uses: astral-sh/setup-uv@v6
-      with:
-        enable-cache: true
-        cache-dependency-glob: pyproject.toml
-        cache-suffix: ${{ matrix.pyv }}
-    - run: uv pip install -e ".[dev]" pytest-split --system
-      env:
-        UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
-
+    - name: install
+      run: |
+        pip install wheel
+        pip install -e ".[all,tests]"
+    - name: setup git
+      run: |
+        git config --global user.email "[email protected]"
+        git config --global user.name "DVC Tester"
+    - name: setup gs creds
+      shell: bash
+      if: env.GS_CREDS_JSON != ''
+      run: |
+        mkdir -p scripts/ci
+        echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
     - name: run tests
-      timeout-minutes: 40
-      env:
-        COVERAGE_CORE: sysmon
-        PYTHONUTF8: 1
-      run: >
-        pytest ${{ matrix.pytestargs }} -n=logical --dist=worksteal --timeout=300 --durations=0
-        --cov --cov-report=xml --cov-report=term  --durations-path=./.github/.test_durations
+      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined
     - name: upload coverage report
-      uses: codecov/codecov-action@v5
-  check:
-    if: always()
-    needs: [tests]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: re-actors/alls-green@release/v1
-        with:
-          jobs: ${{ toJSON(needs) }}
-  notify:
-    if: github.ref == 'refs/heads/main' && failure()
-    needs: [tests]
-    runs-on: ubuntu-latest
-    steps:
-    - name: Slack Notification
-      uses: rtCamp/[email protected]
-      env:
-        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-        SLACK_COLOR: ${{ job.status }}
-        SLACK_MESSAGE: 'CI Failed on main :boom:'
-        SLACK_TITLE: CI Status
-        SLACK_USERNAME: DVC-CI
+      uses: codecov/codecov-action@v1.0.13
+      with:
+        file: ./coverage.xml
+        fail_ci_if_error: false
+    - name: Post TAP
+      # Running custom fork with an option, that does not fail CI on error
+      uses: skshetry/flaky-service/packages/action@master
+      if: always() && github.ref == 'refs/heads/master'
+      with:
+        file-path: ${{github.workspace}}/testresults.tap
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        os: ${{runner.os}}
+        matrix: ${{toJson(matrix)}}
+        repo-description: ${{github.event.repository.description}}
+        fail_ci_if_error: false

.gitignore

@@ -1,13 +1,10 @@
-__pycache__/
 neatlynx.conf
+neatlynx/__pycache__
 .idea
 /cache
 *.pyc
-
-.env*/
-.venv
-env/
-venv/
+.env/
+.env2.7/
 .python-version
 
 .dvc.conf.lock
@@ -16,7 +13,6 @@ build/
 dist/
 
 *.egg-info/
-.eggs
 
 hooks/*.pyc
 
@@ -29,7 +25,6 @@ innosetup/config.ini
 
 .coverage
 .coverage.*
-coverage.xml
 
 *.sw?
 
@@ -43,10 +38,3 @@ scripts/ci/gcp-creds.json
 .gcp-creds.json
 
 *~
-/dvc/_version.py
-
-.mypy_cache/
-.pytest_cache/
-.nox/
-.tox/
-htmlcov/

.pre-commit-config.yaml

@@ -1,46 +1,39 @@
-default_language_version:
-  python: python3
-
-ci:
-  skip: [mypy, dvc-pre-commit]
-
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
-    hooks:
-      - id: check-added-large-files
-      - id: check-case-conflict
-      - id: check-executables-have-shebangs
-      - id: check-json
-      - id: check-merge-conflict
-        exclude: "tests/func/utils/test_strict_yaml.py"
-        args: ['--assume-in-merge']
-      - id: check-toml
-      - id: check-yaml
-      - id: end-of-file-fixer
-      - id: mixed-line-ending
-      - id: sort-simple-yaml
-      - id: trailing-whitespace
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.13.2'
-    hooks:
-      - id: ruff-check
-        args: [--fix, --exit-non-zero-on-fix]
-      - id: ruff-format
-  - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.1
-    hooks:
-      - id: codespell
-        additional_dependencies: ["tomli"]
+  - hooks:
+      - id: black
+        language_version: python3
+    repo: https://github.com/ambv/black
+    rev: 19.10b0
+  - hooks:
+      - id: isort
+        language_version: python3
+    repo: https://github.com/timothycrosley/isort
+    rev: 5.6.4
+  - hooks:
+      - id: flake8
+        language_version: python3
+        additional_dependencies:
+          - flake8-bugbear
+          - flake8-comprehensions
+          - flake8-debugger
+          - flake8-string-format
+    repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.4
   - repo: local
     hooks:
-    - id: mypy
-      name: mypy
-      entry: mypy
-      files: ^dvc/
+    - id: pylint
+      name: pylint
+      entry: pylint
       language: system
       types: [python]
-      require_serial: true
+  - hooks:
+      - args:
+          - -i
+          - "2"
+        id: beautysh
+        language_version: python3
+    repo: https://github.com/lovesegfault/beautysh
+    rev: 6.0.1
   - hooks:
       - args:
           - git-hook
@@ -50,7 +43,7 @@ repos:
         language: system
         name: DVC pre-commit
         stages:
-          - pre-commit
+          - commit
         verbose: true
         require_serial: true
       - args:
@@ -61,7 +54,7 @@ repos:
         language: system
         name: DVC pre-push
         stages:
-          - pre-push
+          - push
         require_serial: true
       - always_run: true
         args:

.pre-commit-hooks.yaml

@@ -8,7 +8,7 @@
   name: DVC pre-commit
   require_serial: true
   stages:
-  - pre-commit
+  - commit
   verbose: true
 - args:
   - git-hook
@@ -20,7 +20,7 @@
   name: DVC pre-push
   require_serial: true
   stages:
-  - pre-push
+  - push
 - always_run: true
   args:
   - git-hook
@@ -29,7 +29,7 @@
   id: dvc-post-checkout
   language: python
   language_version: python3
-  minimum_pre_commit_version: 3.2.0
+  minimum_pre_commit_version: 2.2.0
   name: DVC post-checkout
   require_serial: true
   stages:

.pylintrc

@@ -0,0 +1,607 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-whitelist=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+init-hook="import sys; \
+           from pathlib import Path; \
+           from pylint.config import find_pylintrc; \
+           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=1
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=pylint_pytest,pylint_plugin_disable
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Specify a configuration file.
+#rcfile=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=format,
+        # ^ most of them are covered by flake8,
+        # and are conventions rather than warnings or errors
+        refactoring,  # will be noisy if it starts giving suggestions
+        spelling,
+        # design is too-few-* and too-many-* checks, too-much-* for us right now
+        design,
+        # following were generated by default
+        # most of them are redundant Py3 checks for Py2+3 codebase
+        print-statement,
+        parameter-unpacking,
+        unpacking-in-except,
+        old-raise-syntax,
+        backtick,
+        long-suffix,
+        old-ne-operator,
+        old-octal-literal,
+        import-star-module-level,
+        non-ascii-bytes-literal,
+        raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        apply-builtin,
+        basestring-builtin,
+        buffer-builtin,
+        cmp-builtin,
+        coerce-builtin,
+        execfile-builtin,
+        file-builtin,
+        long-builtin,
+        raw_input-builtin,
+        reduce-builtin,
+        standarderror-builtin,
+        unicode-builtin,
+        xrange-builtin,
+        coerce-method,
+        delslice-method,
+        getslice-method,
+        setslice-method,
+        no-absolute-import,
+        old-division,
+        dict-iter-method,
+        dict-view-method,
+        next-method-called,
+        metaclass-assignment,
+        indexing-exception,
+        raising-string,
+        reload-builtin,
+        oct-method,
+        hex-method,
+        nonzero-method,
+        cmp-method,
+        input-builtin,
+        round-builtin,
+        intern-builtin,
+        unichr-builtin,
+        map-builtin-not-iterating,
+        zip-builtin-not-iterating,
+        range-builtin-not-iterating,
+        filter-builtin-not-iterating,
+        using-cmp-argument,
+        eq-without-hash,
+        div-method,
+        idiv-method,
+        rdiv-method,
+        exception-message-attribute,
+        invalid-str-codec,
+        sys-max-int,
+        bad-python3-import,
+        deprecated-string-function,
+        deprecated-str-translate-call,
+        deprecated-itertools-function,
+        deprecated-types-field,
+        next-method-defined,
+        dict-items-not-iterating,
+        dict-keys-not-iterating,
+        dict-values-not-iterating,
+        deprecated-operator-function,
+        deprecated-urllib-function,
+        xreadlines-attribute,
+        deprecated-sys-function,
+        exception-escape,
+        comprehension-escape,
+        # custom disables start here
+        no-self-use,
+        invalid-name,
+        # already a check enabled for `wildcard-import`
+        unused-wildcard-import,
+        misplaced-comparison-constant,
+        # probably soon: https://github.com/iterative/dvc/issues/1843
+        logging-format-interpolation,
+        logging-fstring-interpolation,
+        # we don't have proper docstrings yet
+        # even if we do, we'll be doing it gradually
+        missing-function-docstring,
+        missing-module-docstring,
+        missing-class-docstring,
+        duplicate-code,
+        # too noisy, and detects issues even when import is inside a function
+        # see: https://github.com/PyCQA/pylint/issues/850
+        cyclic-import,
+        # isort should cover these:
+        wrong-import-order,
+        wrong-import-position,
+        ungrouped-imports,
+        multiple-imports,
+        # we use it to speedup/optimize
+        import-outside-toplevel,
+        fixme,
+
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[REPORTS]
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit
+
+
+[LOGGING]
+
+# Format style used to check logging format string. `old` means using %
+# formatting, while `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package..
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=hdfs.*,pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*|^ignored_|^unused_|args|kwargs
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=79
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,
+               dict-separator
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style.
+#class-attribute-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _,
+           s3,
+           gs,
+           x, y,
+           x0, y0,
+           x1, y1,
+           dx, dy
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style.
+#variable-rgx=
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement.
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=cls
+
+
+[IMPORTS]
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled).
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled).
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception".
+overgeneral-exceptions=Exception

CITATION.cff

@@ -1,25 +0,0 @@
-cff-version: 1.2.0
-title: 'DVC: Data Version Control - Git for Data & Models"'
-message: >-
-  If you use this software as part of a publication and wish
-  to cite it, please use the metadata from this file.
-type: software
-authors:
-  - name: The DVC team and contributors
-    website: 'https://github.com/iterative'
-repository-code: 'https://github.com/iterative/dvc'
-url: 'https://dvc.org'
-abstract: >-
-  DVC is a tool for data science that takes advantage of existing software engineering toolset.
-  It helps machine learning teams manage large datasets, make projects reproducible, and collaborate better.
-keywords:
-  - ai
-  - collaboration
-  - data-science
-  - data-version-control
-  - developer-tools
-  - git
-  - machine-learning
-  - python
-  - reproducibility
-license: Apache-2.0

CODE_OF_CONDUCT.md

@@ -74,3 +74,4 @@ available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.ht
 
 For answers to common questions about this code of conduct, see
 https://www.contributor-covenant.org/faq
+

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2017-2021 Iterative, Inc.
+   Copyright {yyyy} {name of copyright owner}
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

MANIFEST.in

@@ -0,0 +1,2 @@
+include fastentrypoints.py
+include LICENSE

README.rst

@@ -3,114 +3,96 @@
 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_
 • `Blog <http://blog.dataversioncontrol.com>`_
+• `Twitter <https://twitter.com/DVCorg>`_
+• `Chat (Community & Support) <https://dvc.org/chat>`_
 • `Tutorial <https://dvc.org/doc/get-started>`_
-• `Related Technologies <https://dvc.org/doc/user-guide/related-technologies>`_
-• `How DVC works`_
-• `VS Code Extension`_
-• `Installation`_
-• `Contributing`_
-• `Community and Support`_
+• `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-|CI| |Python Version| |Coverage| |VS Code| |DOI|
+|Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|
 
-|PyPI| |PyPI Downloads| |Packages| |Brew| |Conda| |Choco| |Snap|
+|PyPI| |Packages| |Brew| |Conda| |Choco| |Snap|
 
 |
 
-**Data Version Control** or **DVC** is a command line tool and `VS Code Extension`_ to help you develop reproducible machine learning projects:
+**Data Version Control** or **DVC** is an **open-source** tool for data science and machine
+learning projects. Key features:
 
-#. **Version** your data and models.
-   Store them in your cloud storage but keep their version info in your Git repo.
+#. Simple **command line** Git-like experience. Does not require installing and maintaining
+   any databases. Does not depend on any proprietary online services.
 
-#. **Iterate** fast with lightweight pipelines.
-   When you make changes, only run the steps impacted by those changes.
+#. Management and versioning of **datasets** and **machine learning models**. Data is saved in
+   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
 
-#. **Track** experiments in your local Git repo (no servers needed).
+#. Makes projects **reproducible** and **shareable**; helping to answer questions about how
+   a model was built.
 
-#. **Compare** any data, code, parameters, model, or performance plots.
+#. Helps manage experiments with Git tags/branches and **metrics** tracking.
 
-#. **Share** experiments and automatically reproduce anyone's experiment.
+**DVC** aims to replace spreadsheet and document sharing tools (such as Excel or Google Docs)
+which are being used frequently as both knowledge repositories and team ledgers.
+DVC also replaces both ad-hoc scripts to track, move, and deploy different model versions;
+as well as ad-hoc data file suffixes and prefixes.
 
-Quick start
-===========
-
-    Please read our `Command Reference <https://dvc.org/doc/command-reference>`_ for a complete list.
-
-A common CLI workflow includes:
-
-
-+-----------------------------------+----------------------------------------------------------------------------------------------------+
-| Task                              | Terminal                                                                                           |
-+===================================+====================================================================================================+
-| Track data                        | | ``$ git add train.py params.yaml``                                                               |
-|                                   | | ``$ dvc add images/``                                                                            |
-+-----------------------------------+----------------------------------------------------------------------------------------------------+
-| Connect code and data             | | ``$ dvc stage add -n featurize -d images/ -o features/ python featurize.py``                     |
-|                                   | | ``$ dvc stage add -n train -d features/ -d train.py -o model.p -M metrics.json python train.py`` |
-+-----------------------------------+----------------------------------------------------------------------------------------------------+
-| Make changes and experiment       | | ``$ dvc exp run -n exp-baseline``                                                                |
-|                                   | | ``$ vi train.py``                                                                                |
-|                                   | | ``$ dvc exp run -n exp-code-change``                                                             |
-+-----------------------------------+----------------------------------------------------------------------------------------------------+
-| Compare and select experiments    | | ``$ dvc exp show``                                                                               |
-|                                   | | ``$ dvc exp apply exp-baseline``                                                                 |
-+-----------------------------------+----------------------------------------------------------------------------------------------------+
-| Share code                        | | ``$ git add .``                                                                                  |
-|                                   | | ``$ git commit -m 'The baseline model'``                                                         |
-|                                   | | ``$ git push``                                                                                   |
-+-----------------------------------+----------------------------------------------------------------------------------------------------+
-| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``                                         |
-|                                   | | ``$ dvc push``                                                                                   |
-+-----------------------------------+----------------------------------------------------------------------------------------------------+
+.. contents:: **Contents**
+  :backlinks: none
 
 How DVC works
 =============
 
-    We encourage you to read our `Get Started
-    <https://dvc.org/doc/get-started>`_ docs to better understand what DVC
-    does and how it can fit your scenarios.
+We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ guide to better understand what DVC
+is and how it can fit your scenarios.
 
-The closest *analogies* to describe the main DVC features are these:
+The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-LFS to be precise) & Makefiles
+made right and tailored specifically for ML and Data Science scenarios.
 
-#. **Git for data**: Store and share data artifacts (like Git-LFS but without a server) and models, connecting them with a Git repository. Data management meets GitOps!
-#. **Makefiles** for ML: Describes how data or model artifacts are built from other data and code in a standard format. Now you can version your data pipelines with Git.
-#. Local **experiment tracking**: Turn your machine into an ML experiment management platform, and collaborate with others using existing Git hosting (Github, Gitlab, etc.).
+#. ``Git/Git-LFS`` part - DVC helps store and share data artifacts and models, connecting them with a Git repository.
+#. ``Makefile``\ s part - DVC describes how one data or model artifact was built from other data and code.
 
-Git is employed as usual to store and version code (including DVC meta-files as placeholders for data).
-DVC `stores data and model files <https://dvc.org/doc/start/data-management>`_ seamlessly in a cache outside of Git, while preserving almost the same user experience as if they were in the repo.
-To share and back up the *data cache*, DVC supports multiple remote storage platforms - any cloud (S3, Azure, Google Cloud, etc.) or on-premise network storage (via SSH, for example).
+DVC usually runs along with Git. Git is used as usual to store and version code (including DVC meta-files). DVC helps
+to store data and model files seamlessly out of Git, while preserving almost the same user experience as if they
+were stored in Git itself. To store and share the data cache, DVC supports multiple remotes - any cloud (S3, Azure,
+Google Cloud, etc) or any on-premise network storage (via SSH, for example).
 
 |Flowchart|
 
-`DVC pipelines <https://dvc.org/doc/start/data-management/data-pipelines>`_ (computational graphs) connect code and data together.
-They specify all steps required to produce a model: input dependencies including code, data, commands to run; and output information to be saved.
+The DVC pipelines (computational graph) feature connects code and data together. It is possible to explicitly
+specify all steps required to produce a model: input dependencies including data, commands to run,
+and output information to be saved. See the quick start section below or
+the `Get Started <https://dvc.org/doc/get-started>`_ tutorial to learn more.
 
-Last but not least, `DVC Experiment Versioning <https://dvc.org/doc/start/experiments>`_ lets you prepare and run a large number of experiments.
-Their results can be filtered and compared based on hyperparameters and metrics, and visualized with multiple plots.
+Quick start
+===========
 
-.. _`VS Code Extension`:
+Please read `Get Started <https://dvc.org/doc/get-started>`_ guide for a full version. Common workflow commands include:
 
-VS Code Extension
-=================
-
-|VS Code|
-
-To use DVC as a GUI right from your VS Code IDE, install the `DVC Extension <https://marketplace.visualstudio.com/items?itemName=Iterative.dvc>`_ from the Marketplace.
-It currently features experiment tracking and data management, and more features (data pipeline support, etc.) are coming soon!
-
-|VS Code Extension Overview|
-
-    Note: You'll have to install core DVC on your system separately (as detailed
-    below). The Extension will guide you if needed.
++-----------------------------------+-------------------------------------------------------------------+
+| Step                              | Command                                                           |
++===================================+===================================================================+
+| Track data                        | | ``$ git add train.py``                                          |
+|                                   | | ``$ dvc add images.zip``                                        |
++-----------------------------------+-------------------------------------------------------------------+
+| Connect code and data by commands | | ``$ dvc run -d images.zip -o images/ unzip -q images.zip``      |
+|                                   | | ``$ dvc run -d images/ -d train.py -o model.p python train.py`` |
++-----------------------------------+-------------------------------------------------------------------+
+| Make changes and reproduce        | | ``$ vi train.py``                                               |
+|                                   | | ``$ dvc repro model.p.dvc``                                     |
++-----------------------------------+-------------------------------------------------------------------+
+| Share code                        | | ``$ git add .``                                                 |
+|                                   | | ``$ git commit -m 'The baseline model'``                        |
+|                                   | | ``$ git push``                                                  |
++-----------------------------------+-------------------------------------------------------------------+
+| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``        |
+|                                   | | ``$ dvc push``                                                  |
++-----------------------------------+-------------------------------------------------------------------+
 
 Installation
 ============
 
-There are several ways to install DVC: in VS Code; using ``snap``, ``choco``, ``brew``, ``conda``, ``pip``; or with an OS-specific package.
+There are four options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package.
 Full instructions are `available here <https://dvc.org/doc/get-started/install>`_.
 
-Snapcraft (Linux)
------------------
+Snap (Snapcraft/Linux)
+----------------------
 
 |Snap|
 
@@ -119,10 +101,11 @@ Snapcraft (Linux)
    snap install dvc --classic
 
 This corresponds to the latest tagged release.
-Add ``--beta`` for the latest tagged release candidate, or ``--edge`` for the latest ``main`` version.
+Add ``--beta`` for the latest tagged release candidate,
+or ``--edge`` for the latest ``master`` version.
 
-Chocolatey (Windows)
---------------------
+Choco (Chocolatey/Windows)
+--------------------------
 
 |Choco|
 
@@ -130,8 +113,8 @@ Chocolatey (Windows)
 
    choco install dvc
 
-Brew (mac OS)
--------------
+Brew (Homebrew/Mac OS)
+----------------------
 
 |Brew|
 
@@ -139,20 +122,17 @@ Brew (mac OS)
 
    brew install dvc
 
-Anaconda (Any platform)
------------------------
+Conda (Anaconda)
+----------------
 
 |Conda|
 
 .. code-block:: bash
 
-   conda install -c conda-forge mamba # installs much faster than conda
-   mamba install -c conda-forge dvc
+   conda install -c conda-forge dvc
 
-Depending on the remote storage type you plan to use to keep and share your data, you might need to install optional dependencies: `dvc-s3`, `dvc-azure`, `dvc-gdrive`, `dvc-gs`, `dvc-oss`, `dvc-ssh`.
-
-PyPI (Python)
--------------
+pip (PyPI)
+----------
 
 |PyPI|
 
@@ -160,8 +140,10 @@ PyPI (Python)
 
    pip install dvc
 
-Depending on the remote storage type you plan to use to keep and share your data, you might need to specify one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all`` to include them all.
-The command should look like this: ``pip install 'dvc[s3]'`` (in this case AWS S3 dependencies such as ``boto3`` will be installed automatically).
+Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
+one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all`` to include them all.
+The command should look like this: ``pip install dvc[s3]`` (in this case AWS S3 dependencies such as ``boto3``
+will be installed automatically).
 
 To install the development version, run:
 
@@ -169,58 +151,100 @@ To install the development version, run:
 
    pip install git+git://github.com/iterative/dvc
 
-Package (Platform-specific)
----------------------------
+Package
+-------
 
 |Packages|
 
-Self-contained packages for Linux, Windows, and Mac are available.
-The latest version of the packages can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+Self-contained packages for Linux, Windows, and Mac are available. The latest version of the packages
+can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
 Ubuntu / Debian (deb)
 ^^^^^^^^^^^^^^^^^^^^^
 .. code-block:: bash
 
    sudo wget https://dvc.org/deb/dvc.list -O /etc/apt/sources.list.d/dvc.list
-   wget -qO - https://dvc.org/deb/iterative.asc | sudo apt-key add -
-   sudo apt update
-   sudo apt install dvc
+   sudo apt-get update
+   sudo apt-get install dvc
 
 Fedora / CentOS (rpm)
 ^^^^^^^^^^^^^^^^^^^^^
 .. code-block:: bash
 
    sudo wget https://dvc.org/rpm/dvc.repo -O /etc/yum.repos.d/dvc.repo
-   sudo rpm --import https://dvc.org/rpm/iterative.asc
    sudo yum update
    sudo yum install dvc
 
+Comparison to related technologies
+==================================
+
+#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should
+   not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of
+   copying/duplicating files.
+
+#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
+   etc). DVC also uses reflinks or hardlinks to avoid copy operations on checkouts; thus handling large data files
+   much more efficiently.
+
+#. *Makefile* (and analogues including ad-hoc scripts) - DVC tracks dependencies (in a directed acyclic graph).
+
+#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow
+   management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
+
+#. `DAGsHub <https://dagshub.com/>`_ - This is a Github equivalent for DVC. Pushing Git+DVC based repositories to DAGsHub will produce in a high level project dashboard; including DVC pipelines and metrics visualizations, as well as links to any DVC-managed files present in cloud storage.
+
 Contributing
 ============
 
-|Maintainability|
+|Maintainability| |Donate|
 
-Contributions are welcome!
-Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more details.
-Thanks to all our contributors!
+Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
+details.
 
-|Contribs|
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+   :alt: 0
 
-Community and Support
-=====================
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+   :alt: 1
 
-* `Twitter <https://twitter.com/DVCorg>`_
-* `Forum <https://discuss.dvc.org/>`_
-* `Discord Chat <https://dvc.org/chat>`_
-* `Email <mailto:[email protected]>`_
-* `Mailing List <https://dvc.org/community#subscribe>`_
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+   :alt: 2
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
+   :alt: 3
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+   :alt: 4
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+   :alt: 5
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+   :alt: 6
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+   :alt: 7
+
+Mailing List
+============
+
+Want to stay up to date? Want to help improve DVC by participating in our occasional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
 
 Copyright
 =========
 
 This project is distributed under the Apache license version 2.0 (see the LICENSE file in the project root).
 
-By submitting a pull request to this project, you agree to license your contribution under the Apache license version 2.0 to this project.
+By submitting a pull request to this project, you agree to license your contribution under the Apache license version
+2.0 to this project.
 
 Citation
 ========
@@ -230,32 +254,30 @@ Citation
 Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
 `DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
 
-Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects <https://mcis.cs.queensu.ca/publications/2021/saner.pdf>`_ , in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA.
-
-
 .. |Banner| image:: https://dvc.org/img/logo-github-readme.png
    :target: https://dvc.org
    :alt: DVC logo
 
-.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/iterative/vscode-dvc/main/extension/docs/overview.gif
-   :alt: DVC Extension for VS Code
+.. |Release| image:: https://img.shields.io/badge/release-ok-brightgreen
+   :target: https://travis-ci.com/iterative/dvc/branches
+   :alt: Release
 
-.. |CI| image:: https://github.com/iterative/dvc/actions/workflows/tests.yaml/badge.svg
-   :target: https://github.com/iterative/dvc/actions/workflows/tests.yaml
+.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=master
+   :target: https://github.com/iterative/dvc/actions
    :alt: GHA Tests
 
-.. |Maintainability| image:: https://img.shields.io/codeclimate/maintainability/iterative/dvc
+.. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
    :target: https://codeclimate.com/github/iterative/dvc
    :alt: Code Climate
 
-.. |Python Version| image:: https://img.shields.io/pypi/pyversions/dvc
-   :target: https://pypi.org/project/dvc
-   :alt: Python Version
-
-.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/main/graph/badge.svg
+.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
    :target: https://codecov.io/gh/iterative/dvc
    :alt: Codecov
 
+.. |Donate| image:: https://img.shields.io/badge/patreon-donate-green.svg?logo=patreon
+   :target: https://www.patreon.com/DVCorg/overview
+   :alt: Donate
+
 .. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
    :target: https://snapcraft.io/dvc
    :alt: Snapcraft
@@ -268,7 +290,7 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://formulae.brew.sh/formula/dvc
    :alt: Homebrew
 
-.. |Conda| image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
+.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/dvc.svg?label=conda&logo=conda-forge
    :target: https://anaconda.org/conda-forge/dvc
    :alt: Conda-forge
 
@@ -276,12 +298,8 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://pypi.org/project/dvc
    :alt: PyPI
 
-.. |PyPI Downloads| image:: https://img.shields.io/pypi/dm/dvc.svg?color=blue&label=Downloads&logo=pypi&logoColor=gold
-   :target: https://pypi.org/project/dvc
-   :alt: PyPI Downloads
-
-.. |Packages| image:: https://img.shields.io/badge/deb|pkg|rpm|exe-blue
-   :target: https://dvc.org/doc/install
+.. |Packages| image:: https://img.shields.io/github/v/release/iterative/dvc?label=deb|pkg|rpm|exe&logo=GitHub
+   :target: https://github.com/iterative/dvc/releases/latest
    :alt: deb|pkg|rpm|exe
 
 .. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.3677553-blue.svg
@@ -291,11 +309,3 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
 .. |Flowchart| image:: https://dvc.org/img/flow.gif
    :target: https://dvc.org/img/flow.gif
    :alt: how_dvc_works
-
-.. |Contribs| image:: https://contrib.rocks/image?repo=iterative/dvc
-   :target: https://github.com/iterative/dvc/graphs/contributors
-   :alt: Contributors
-
-.. |VS Code| image:: https://img.shields.io/visual-studio-marketplace/v/Iterative.dvc?color=blue&label=VSCode&logo=visualstudiocode&logoColor=blue
-   :target: https://marketplace.visualstudio.com/items?itemName=Iterative.dvc
-   :alt: VS Code Extension

bin/dvc

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+PYTHONPATH=$DVC_HOME python -m dvc $@

bin/dvc.bat

@@ -0,0 +1,3 @@
+@echo OFF
+set PYTHONPATH=%DVC_HOME%
+python -m dvc %*

debian/changelog

@@ -0,0 +1,5 @@
+dvc (0.75.b1) experimental; urgency=low
+
+  * First native Debian package version.
+
+ -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -0,0 +1,33 @@
+Source: dvc
+Maintainer: Dmitry Petrov <[email protected]>
+Section: python
+Priority: optional
+Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
+Standards-Version: 4.0.0
+Homepage: http://dvc.org
+Vcs-Git: https://github.com/iterative/dvc.git
+Vcs-browser: https://github.com/iterative/dvc
+
+Package: dvc
+Architecture: all
+Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
+Recommends: python3-arrow (=0.14.0),
+            python3-paramiko (>=2.5.0),
+            python3-gssapi (>= 2.5.0),
+            python3-boto3 (>=1.9.86),
+            python3-pydrive (=1.3.1),
+            python3-azure-storage-blob (=12.1.0),
+            python3-oss2 (=2.6.1),
+            python3-google-cloud-storage (=1.19.0)
+Description: Git for data scientists - manage your code and data together
+ Data Version Control or DVC is an open-source tool for data science and machine learning projects.
+ .
+ Key features:
+ .
+ 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
+ .
+ 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
+ .
+ 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
+ .
+ 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -0,0 +1,11 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: DVC
+Upstream-Contact: Dmitry Petrov <[email protected]>
+Source: https://github.com/iterative/dvc
+
+Files: *
+Copyright: Dmitry Petrov <[email protected]>
+License: Apache 2.0
+ On Debian systems, the full text of the Apache 2.0
+ License can be found in the file
+ `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+
+export PYBUILD_NAME=dvc
+%:
+	dh $@ --with python3 --buildsystem=pybuild
+

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

debian/watch

@@ -0,0 +1,4 @@
+# please also check http://pypi.debian.net/dvc/watch
+version=3
+opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
+http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/__init__.py

@@ -3,9 +3,7 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-
 import dvc.logger
-from dvc.build import PKG  # noqa: F401
-from dvc.version import __version__, version_tuple  # noqa: F401
+from dvc.version import __version__  # noqa: F401
 
 dvc.logger.setup()

dvc/__main__.py

@@ -1,8 +1,7 @@
 """Main entry point for DVC command line tool."""
-
 import sys
 
-from dvc.cli import main
+from dvc.main import main
 
 if __name__ == "__main__":
     sys.exit(main(sys.argv[1:]))

dvc/__pyinstaller/__init__.py

@@ -1,5 +0,0 @@
-import os
-
-
-def get_hook_dirs():
-    return [os.path.dirname(__file__)]

dvc/__pyinstaller/hook-celery.py

@@ -1,14 +0,0 @@
-# ruff: noqa: N999
-
-from PyInstaller.utils.hooks import collect_submodules, is_module_or_submodule
-
-# Celery dynamically imports most celery internals at runtime
-# pyinstaller hook must expose all modules loaded by
-# kombu.utils.imports:symbol_by_name()
-_EXCLUDES = ("celery.bin", "celery.contrib")
-hiddenimports = collect_submodules(
-    "celery",
-    filter=lambda name: not any(
-        is_module_or_submodule(name, module) for module in _EXCLUDES
-    ),
-)

dvc/__pyinstaller/hook-dvc.py

@@ -1,30 +0,0 @@
-# ruff: noqa: N999
-from PyInstaller.utils.hooks import copy_metadata
-
-# needed for `dvc doctor` to show dep versions
-datas = copy_metadata("adlfs", recursive=True)
-datas += copy_metadata("knack")
-datas += copy_metadata("gcsfs")
-datas += copy_metadata("pyarrow")
-datas += copy_metadata("pydrive2")
-datas += copy_metadata("s3fs", recursive=True)
-datas += copy_metadata("boto3")
-datas += copy_metadata("ossfs")
-datas += copy_metadata("sshfs")
-datas += copy_metadata("webdav4")
-datas += copy_metadata("aiohttp")
-datas += copy_metadata("aiohttp_retry")
-
-hiddenimports = [
-    "dvc_azure",
-    "dvc_gdrive",
-    "dvc_gs",
-    "dvc_hdfs",
-    "dvc_oss",
-    "dvc_s3",
-    "dvc_ssh",
-    "dvc_webdav",
-    "dvc_webhdfs",
-    # https://github.com/pypa/setuptools/issues/1963
-    "pkg_resources.py2_warn",
-]

dvc/__pyinstaller/hook-dvc.utils.flatten.py

@@ -1,4 +0,0 @@
-# ruff: noqa: N999
-from PyInstaller.utils.hooks import copy_metadata
-
-datas = copy_metadata("flatten-dict")

dvc/__pyinstaller/hook-dvc_task.py

@@ -1,5 +0,0 @@
-# ruff: noqa: N999
-
-from PyInstaller.utils.hooks import collect_submodules
-
-hiddenimports = collect_submodules("dvc_task")

dvc/__pyinstaller/hook-fsspec.py

@@ -1,2 +0,0 @@
-# ruff: noqa: N999
-hiddenimports = ["fsspec.implementations.memory"]

dvc/__pyinstaller/hook-pydrive2.py

@@ -1,4 +0,0 @@
-# ruff: noqa: N999
-from PyInstaller.utils.hooks import copy_metadata
-
-datas = copy_metadata("pydrive2")

dvc/_debug.py

@@ -1,288 +0,0 @@
-from contextlib import ExitStack, contextmanager
-from datetime import datetime
-from typing import TYPE_CHECKING, Callable, Optional, Union
-
-if TYPE_CHECKING:
-    from argparse import Namespace
-    from types import FrameType
-
-
-@contextmanager
-def viztracer_profile(
-    path: Union[Callable[[], str], str],
-    depth: int = -1,
-    log_async: bool = False,
-):
-    try:
-        import viztracer  # ty: ignore[unresolved-import]
-    except ImportError:
-        print("Failed to run profiler, viztracer is not installed")  # noqa: T201
-        yield
-        return
-
-    tracer = viztracer.VizTracer(max_stack_depth=depth, log_async=log_async)
-
-    tracer.start()
-    try:
-        yield
-    finally:
-        tracer.stop()
-        tracer.save(path() if callable(path) else path)
-
-
-@contextmanager
-def yappi_profile(
-    path: Optional[Union[Callable[[], str], str]] = None,
-    wall_clock: Optional[bool] = True,
-    separate_threads: Optional[bool] = False,
-):
-    try:
-        import yappi  # ty: ignore[unresolved-import]
-    except ImportError:
-        print("Failed to run profiler, yappi is not installed")  # noqa: T201
-        yield
-        return
-
-    yappi.set_clock_type("wall" if wall_clock else "cpu")
-
-    yappi.start()
-    try:
-        yield
-    finally:
-        yappi.stop()
-
-        threads = yappi.get_thread_stats()
-        stats = {}
-        if separate_threads:
-            for thread in threads:
-                ctx_id = thread.id
-                stats[ctx_id] = yappi.get_func_stats(ctx_id=ctx_id)
-        else:
-            stats[None] = yappi.get_func_stats()
-
-        fpath = path() if callable(path) else path
-        for ctx_id, st in stats.items():
-            if fpath:
-                out = f"{fpath}-{ctx_id}" if ctx_id is not None else fpath
-                st.save(out, type="callgrind")
-            else:
-                if ctx_id is not None:
-                    print(f"\nThread {ctx_id}")  # noqa: T201
-                st.print_all()
-                if ctx_id is None:
-                    threads.print_all()
-
-        yappi.clear_stats()
-
-
-@contextmanager
-def instrument(html_output=False):
-    """Run a statistical profiler"""
-    try:
-        from pyinstrument import Profiler  # ty: ignore[unresolved-import]
-    except ImportError:
-        print("Failed to run profiler, pyinstrument is not installed")  # noqa: T201
-        yield
-        return
-
-    profiler = Profiler()
-
-    profiler.start()
-    try:
-        yield
-    finally:
-        profiler.stop()
-
-        if html_output:
-            profiler.open_in_browser()
-        else:
-            print(profiler.output_text(unicode=True, color=True))  # noqa: T201
-
-
-@contextmanager
-def profile(dump_path: Optional[str] = None):
-    """Run a cprofile"""
-    import cProfile
-
-    prof = cProfile.Profile()
-    prof.enable()
-
-    try:
-        yield
-    finally:
-        prof.disable()
-        if dump_path:
-            prof.dump_stats(dump_path)
-        else:
-            prof.print_stats(sort="cumtime")
-
-
-@contextmanager
-def debug():
-    try:
-        yield
-    except Exception:
-        try:
-            import ipdb as pdb  # noqa: T100  # ty: ignore[unresolved-import]
-        except ImportError:
-            import pdb  # type: ignore[no-redef]  # noqa: T100
-        pdb.post_mortem()
-
-        raise  # prevent from jumping ahead
-
-
-def _sigshow(_, frame: Optional["FrameType"]) -> None:
-    import sys
-    from shutil import get_terminal_size
-    from traceback import format_stack
-
-    lines = "\u2015" * get_terminal_size().columns
-    stack = format_stack(frame)
-    print(lines, "\n", *stack, lines, sep="", file=sys.stderr)  # noqa: T201
-
-
-@contextmanager
-def show_stack():
-    r"""Show stack trace on SIGQUIT (Ctrl-\) or SIGINFO (Ctrl-T on macOS)."""
-    import signal
-    import sys
-
-    if sys.platform != "win32":
-        signal.signal(signal.SIGQUIT, _sigshow)
-
-    try:
-        # only available on macOS
-        signal.signal(signal.SIGINFO, _sigshow)  # type: ignore[attr-defined]
-    except AttributeError:
-        pass
-    yield
-
-
-def _get_path_func(tool: str, ext: str):
-    fmt = f"{tool}.dvc-{{now:%Y%m%d}}_{{now:%H%M%S}}.{ext}"
-
-    def func(now: Optional["datetime"] = None) -> str:
-        return fmt.format(now=now or datetime.now())  # noqa: DTZ005
-
-    return func
-
-
-@contextmanager
-def debugtools(args: Optional["Namespace"] = None, **kwargs):
-    kw = vars(args) if args else {}
-    kw.update(kwargs)
-
-    with ExitStack() as stack:
-        if kw.get("pdb"):
-            stack.enter_context(debug())
-        if kw.get("cprofile") or kw.get("cprofile_dump"):
-            stack.enter_context(profile(kw.get("cprofile_dump")))
-        if kw.get("instrument") or kw.get("instrument_open"):
-            stack.enter_context(instrument(kw.get("instrument_open", False)))
-        if kw.get("show_stack", False):
-            stack.enter_context(show_stack())
-        if kw.get("yappi"):
-            path_func = _get_path_func("callgrind", "out")
-            stack.enter_context(
-                yappi_profile(
-                    path=path_func,
-                    separate_threads=kw.get("yappi_separate_threads"),
-                )
-            )
-        if (
-            kw.get("viztracer")
-            or kw.get("viztracer_depth")
-            or kw.get("viztracer_async")
-        ):
-            path_func = _get_path_func("viztracer", "json")
-            depth = kw.get("viztracer_depth") or -1
-            log_async = kw.get("viztracer_async") or False
-            prof = viztracer_profile(path=path_func, depth=depth, log_async=log_async)
-            stack.enter_context(prof)
-        yield
-
-
-def add_debugging_flags(parser):
-    from argparse import SUPPRESS
-
-    # For detailed info see:
-    # https://github.com/iterative/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
-    args, _ = parser.parse_known_args()
-    verbose = args.verbose
-
-    def debug_help(msg):
-        if verbose:
-            return msg
-        return SUPPRESS
-
-    parser = parser.add_argument_group("debug options")
-
-    parser.add_argument(
-        "--cprofile",
-        action="store_true",
-        default=False,
-        help=debug_help("Generate cprofile data for tools like snakeviz / tuna"),
-    )
-    parser.add_argument(
-        "--cprofile-dump", help=debug_help("Location to dump cprofile file")
-    )
-    parser.add_argument(
-        "--yappi",
-        action="store_true",
-        default=False,
-        help=debug_help(
-            "Generate a callgrind file for use with tools like "
-            "kcachegrind / qcachegrind"
-        ),
-    )
-    parser.add_argument(
-        "--yappi-separate-threads",
-        action="store_true",
-        default=False,
-        help=debug_help("Generate one callgrind file per thread"),
-    )
-    parser.add_argument(
-        "--viztracer",
-        action="store_true",
-        default=False,
-        help=debug_help("Generate a viztracer file for use with vizviewer"),
-    )
-    parser.add_argument(
-        "--viztracer-depth",
-        type=int,
-        help=debug_help("Set viztracer maximum stack depth"),
-    )
-    parser.add_argument(
-        "--viztracer-async",
-        action="store_true",
-        default=False,
-        help=debug_help("Treat async tasks as threads"),
-    )
-    parser.add_argument(
-        "--pdb",
-        action="store_true",
-        default=False,
-        help=debug_help("Drop into the pdb/ipdb debugger on any exception"),
-    )
-    parser.add_argument(
-        "--instrument",
-        action="store_true",
-        default=False,
-        help=debug_help("Use pyinstrument CLI profiler"),
-    )
-    parser.add_argument(
-        "--instrument-open",
-        action="store_true",
-        default=False,
-        help=debug_help("Use pyinstrument web profiler"),
-    )
-    parser.add_argument(
-        "--show-stack",
-        "--ss",
-        action="store_true",
-        default=False,
-        help=debug_help(
-            r"Use Ctrl+T on macOS or Ctrl+\ on Linux to print the stack "
-            "frame currently executing. Unavailable on Windows."
-        ),
-    )

dvc/analytics.py

@@ -1,15 +1,26 @@
 import json
+import logging
 import os
-from typing import TYPE_CHECKING, Optional
+import platform
+import sys
+import tempfile
+import uuid
 
-from dvc.log import logger
+import distro
+import requests
 
-from .env import DVC_ANALYTICS_ENDPOINT, DVC_NO_ANALYTICS
+from dvc import __version__
+from dvc.config import Config, to_bool
+from dvc.daemon import daemon
+from dvc.exceptions import NotDvcRepoError
+from dvc.lock import Lock, LockError
+from dvc.repo import Repo
+from dvc.scm import SCM, NoSCM
+from dvc.scm.base import SCMError
+from dvc.utils import env2bool, is_binary
+from dvc.utils.fs import makedirs
 
-if TYPE_CHECKING:
-    from dvc.scm import Base
-
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 def collect_and_send_report(args=None, return_code=None):
@@ -24,10 +35,6 @@ def collect_and_send_report(args=None, return_code=None):
     report as a JSON, where the _collector_ generates it and the _sender_
     removes it after sending it.
     """
-    import tempfile
-
-    from dvc.daemon import daemon
-
     report = {}
 
     # Include command execution information on the report only when available.
@@ -39,25 +46,17 @@ def collect_and_send_report(args=None, return_code=None):
 
     with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
         json.dump(report, fobj)
-
-    logger.trace("Saving analytics report to %s", fobj.name)
-    daemon(["analytics", fobj.name])
+        daemon(["analytics", fobj.name])
 
 
 def is_enabled():
-    from dvc.config import Config, to_bool
-    from dvc.utils import env2bool
-
     if env2bool("DVC_TEST"):
         return False
 
-    enabled = not os.getenv(DVC_NO_ANALYTICS)
-    if enabled:
-        enabled = to_bool(
-            Config.from_cwd(validate=False).get("core", {}).get("analytics", "true")
-        )
-
-    logger.debug("Analytics is %sabled.", "en" if enabled else "dis")
+    enabled = to_bool(
+        Config(validate=False).get("core", {}).get("analytics", "true")
+    )
+    logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
     return enabled
 
@@ -70,136 +69,50 @@ def send(path):
     `collect_and_send_report`. Sending happens on another process,
     thus, the need of removing such file afterwards.
     """
-    import requests
-    from requests.exceptions import RequestException
-
-    url = os.environ.get(DVC_ANALYTICS_ENDPOINT, "https://analytics.dvc.org")
+    url = "https://analytics.dvc.org"
     headers = {"content-type": "application/json"}
 
-    with open(path, encoding="utf-8") as fobj:
+    with open(path) as fobj:
         report = json.load(fobj)
 
     report.update(_runtime_info())
 
-    logger.debug("uploading report to %s", url)
-    logger.trace("Sending %s to %s", report, url)
-
     try:
         requests.post(url, json=report, headers=headers, timeout=5)
-    except RequestException as e:
-        logger.trace("", exc_info=True)
-        logger.debug("failed to send analytics report %s", str(e))
+    except requests.exceptions.RequestException:
+        logger.debug("failed to send analytics report", exc_info=True)
 
-    logger.trace("removing report %s", path)
     os.remove(path)
 
 
-def _git_remote_url(scm: Optional["Base"]) -> Optional[str]:
-    from dvc.scm import Git
-
-    if not isinstance(scm, Git):
-        return None
-
-    from dulwich.porcelain import get_remote_repo
-
-    dulwich_repo = scm.dulwich.repo
+def _scm_in_use():
     try:
-        _remote, url = get_remote_repo(dulwich_repo)
-    except IndexError:
-        # IndexError happens when the head is detached
-        _remote, url = get_remote_repo(dulwich_repo, b"origin")
-    # Dulwich returns (None, "origin") if no remote set
-    if (_remote, url) == (None, "origin"):
-        return None
-    return url
-
-
-def _scm_in_use(scm: Optional["Base"]) -> Optional[str]:
-    return type(scm).__name__ if scm else None
-
-
-def _parse_git_remote_path(remote_url: str) -> str:
-    from urllib.parse import urlparse
-
-    from scmrepo.urls import is_scp_style_url
-
-    parsed = urlparse(remote_url)
-    # Windows Path also gets parsed with a drive letter as scheme
-    # https://github.com/python/cpython/issues/86381
-    if parsed.scheme and parsed.scheme in ("http", "https", "git", "ssh"):
-        return parsed.path.strip("/")
-
-    if is_scp_style_url(remote_url):
-        # handle scp-style URL
-        parts = remote_url.split(":", 1)
-        if len(parts) == 2:
-            _, path = parts
-            return path.rstrip("/")
-    return remote_url
-
-
-def _git_remote_path_hash(scm: Optional["Base"]) -> Optional[str]:
-    """Return a hash of the git remote path."""
-    import hashlib
-
-    try:
-        if remote_url := _git_remote_url(scm):
-            path = _parse_git_remote_path(remote_url)
-            h = hashlib.md5(path.encode("utf-8"), usedforsecurity=False)  # for FIPS
-            return h.hexdigest()
-    except Exception:  # noqa: BLE001
-        logger.debug("Failed to get git remote path", exc_info=True)
-    return None
+        scm = SCM(root_dir=Repo.find_root())
+        return type(scm).__name__
+    except SCMError:
+        return NoSCM.__name__
+    except NotDvcRepoError:
+        pass
 
 
 def _runtime_info():
     """
     Gather information from the environment where DVC runs to fill a report.
     """
-    from iterative_telemetry import _generate_ci_id, find_or_create_user_id
-
-    from dvc import __version__
-    from dvc.info import _get_remotes
-    from dvc.repo import Repo
-    from dvc.utils import is_binary
-
-    ci_id = _generate_ci_id()
-    if ci_id:
-        group_id, user_id = ci_id
-    else:
-        group_id, user_id = None, find_or_create_user_id()
-
-    scm = None
-    remotes = None
-    try:
-        repo = Repo()
-        scm = repo.scm
-        remotes = _get_remotes(repo.config)
-    except Exception as exc:  # noqa: BLE001
-        logger.debug("failed to open repo: %s", exc)
-
     return {
         "dvc_version": __version__,
         "is_binary": is_binary(),
-        "scm_class": _scm_in_use(scm),
+        "scm_class": _scm_in_use(),
         "system_info": _system_info(),
-        "user_id": user_id,
-        "group_id": group_id,
-        "remotes": remotes,
-        "git_remote_hash": _git_remote_path_hash(scm),
+        "user_id": _find_or_create_user_id(),
     }
 
 
 def _system_info():
-    import platform
-    import sys
-
-    import distro
-
     system = platform.system()
 
     if system == "Windows":
-        version = sys.getwindowsversion()  # type: ignore[attr-defined]
+        version = sys.getwindowsversion()
 
         return {
             "os": "windows",
@@ -222,3 +135,39 @@ def _system_info():
 
     # We don't collect data for any other system.
     raise NotImplementedError
+
+
+def _find_or_create_user_id():
+    """
+    The user's ID is stored on a file under the global config directory.
+
+    The file should contain a JSON with a "user_id" key:
+
+        {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"}
+
+    IDs are generated randomly with UUID.
+    """
+    config_dir = Config.get_dir("global")
+    fname = os.path.join(config_dir, "user_id")
+    lockfile = os.path.join(config_dir, "user_id.lock")
+
+    # Since the `fname` and `lockfile` are under the global config,
+    # we need to make sure such directory exist already.
+    makedirs(config_dir, exist_ok=True)
+
+    try:
+        with Lock(lockfile):
+            try:
+                with open(fname) as fobj:
+                    user_id = json.load(fobj)["user_id"]
+
+            except (FileNotFoundError, ValueError, KeyError):
+                user_id = str(uuid.uuid4())
+
+                with open(fname, "w") as fobj:
+                    json.dump({"user_id": user_id}, fobj)
+
+            return user_id
+
+    except LockError:
+        logger.debug(f"Failed to acquire '{lockfile}'")

dvc/annotations.py

@@ -1,52 +0,0 @@
-from dataclasses import asdict, dataclass, field, fields
-from typing import Any, ClassVar, Optional
-
-from funcy import compact
-from voluptuous import Required
-
-
-@dataclass
-class Annotation:
-    PARAM_DESC: ClassVar[str] = "desc"
-    PARAM_TYPE: ClassVar[str] = "type"
-    PARAM_LABELS: ClassVar[str] = "labels"
-    PARAM_META: ClassVar[str] = "meta"
-
-    desc: Optional[str] = None
-    type: Optional[str] = None
-    labels: list[str] = field(default_factory=list)
-    meta: dict[str, Any] = field(default_factory=dict)
-
-    def to_dict(self) -> dict[str, str]:
-        return compact(asdict(self))
-
-
-@dataclass
-class Artifact:
-    PARAM_PATH: ClassVar[str] = "path"
-    PARAM_DESC: ClassVar[str] = "desc"
-    PARAM_TYPE: ClassVar[str] = "type"
-    PARAM_LABELS: ClassVar[str] = "labels"
-    PARAM_META: ClassVar[str] = "meta"
-
-    path: str
-    desc: Optional[str] = None
-    type: Optional[str] = None
-    labels: list[str] = field(default_factory=list)
-    meta: dict[str, Any] = field(default_factory=dict)
-
-    def to_dict(self) -> dict[str, str]:
-        return compact(asdict(self))
-
-
-ANNOTATION_FIELDS = [field.name for field in fields(Annotation)]
-ANNOTATION_SCHEMA = {
-    Annotation.PARAM_DESC: str,
-    Annotation.PARAM_TYPE: str,
-    Annotation.PARAM_LABELS: [str],
-    Annotation.PARAM_META: object,
-}
-ARTIFACT_SCHEMA: dict[Any, Any] = {
-    Required(Artifact.PARAM_PATH): str,
-    **ANNOTATION_SCHEMA,  # type: ignore[arg-type]
-}

dvc/api.py

@@ -0,0 +1,142 @@
+import os
+from contextlib import _GeneratorContextManager as GCM
+from contextlib import contextmanager
+
+from funcy import reraise
+
+from dvc.exceptions import (
+    NotDvcRepoError,
+    OutputNotFoundError,
+    PathMissingError,
+)
+from dvc.external_repo import external_repo
+from dvc.path_info import PathInfo
+from dvc.repo import Repo
+
+
+def get_url(path, repo=None, rev=None, remote=None):
+    """
+    Returns the URL to the storage location of a data file or directory tracked
+    in a DVC repo. For Git repos, HEAD is used unless a rev argument is
+    supplied. The default remote is tried unless a remote argument is supplied.
+
+    Raises OutputNotFoundError if the file is not tracked by DVC.
+
+    NOTE: This function does not check for the actual existence of the file or
+    directory in the remote storage.
+    """
+    with _make_repo(repo, rev=rev) as _repo:
+        path_info = PathInfo(_repo.root_dir) / path
+        with reraise(FileNotFoundError, PathMissingError(path, repo)):
+            metadata = _repo.repo_tree.metadata(path_info)
+
+        if not metadata.is_dvc:
+            raise OutputNotFoundError(path, repo)
+
+        cloud = metadata.repo.cloud
+        hash_info = _repo.repo_tree.get_hash(path_info)
+        return cloud.get_url_for(remote, checksum=hash_info.value)
+
+
+def open(  # noqa, pylint: disable=redefined-builtin
+    path, repo=None, rev=None, remote=None, mode="r", encoding=None
+):
+    """
+    Open file in the supplied path tracked in a repo (both DVC projects and
+    plain Git repos are supported). For Git repos, HEAD is used unless a rev
+    argument is supplied. The default remote is tried unless a remote argument
+    is supplied. It may only be used as a context manager:
+
+        with dvc.api.open(
+                'path/to/file',
+                repo='https://example.com/url/to/repo'
+                ) as fd:
+            # ... Handle file object fd
+    """
+    args = (path,)
+    kwargs = {
+        "repo": repo,
+        "remote": remote,
+        "rev": rev,
+        "mode": mode,
+        "encoding": encoding,
+    }
+    return _OpenContextManager(_open, args, kwargs)
+
+
+class _OpenContextManager(GCM):
+    def __init__(
+        self, func, args, kwds
+    ):  # pylint: disable=super-init-not-called
+        self.gen = func(*args, **kwds)
+        self.func, self.args, self.kwds = func, args, kwds
+
+    def __getattr__(self, name):
+        raise AttributeError(
+            "dvc.api.open() should be used in a with statement."
+        )
+
+
+def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
+    with _make_repo(repo, rev=rev) as _repo:
+        with _repo.open_by_relpath(
+            path, remote=remote, mode=mode, encoding=encoding
+        ) as fd:
+            yield fd
+
+
+def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
+    """
+    Returns the contents of a tracked file (by DVC or Git). For Git repos, HEAD
+    is used unless a rev argument is supplied. The default remote is tried
+    unless a remote argument is supplied.
+    """
+    with open(
+        path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
+    ) as fd:
+        return fd.read()
+
+
+@contextmanager
+def _make_repo(repo_url=None, rev=None):
+    repo_url = repo_url or os.getcwd()
+    if rev is None and os.path.exists(repo_url):
+        try:
+            yield Repo(repo_url, subrepos=True)
+            return
+        except NotDvcRepoError:
+            pass  # fallthrough to external_repo
+    with external_repo(url=repo_url, rev=rev) as repo:
+        yield repo
+
+
+def make_checkpoint():
+    """
+    Signal DVC to create a checkpoint experiment.
+
+    If the current process is being run from DVC, this function will block
+    until DVC has finished creating the checkpoint. Otherwise, this function
+    will return immediately.
+    """
+    import builtins
+    from time import sleep
+
+    from dvc.env import DVC_CHECKPOINT, DVC_ROOT
+    from dvc.stage.run import CHECKPOINT_SIGNAL_FILE
+
+    if os.getenv(DVC_CHECKPOINT) is None:
+        return
+
+    root_dir = os.getenv(DVC_ROOT, Repo.find_root())
+    signal_file = os.path.join(
+        root_dir, Repo.DVC_DIR, "tmp", CHECKPOINT_SIGNAL_FILE
+    )
+
+    with builtins.open(signal_file, "w") as fobj:
+        # NOTE: force flushing/writing empty file to disk, otherwise when
+        # run in certain contexts (pytest) file may not actually be written
+        fobj.write("")
+        fobj.flush()
+        os.fsync(fobj.fileno())
+    while os.path.exists(signal_file):
+        sleep(1)

dvc/api/__init__.py

@@ -1,24 +0,0 @@
-from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem
-
-from .artifacts import artifacts_show
-from .data import get_url, open, read  # noqa: A004
-from .dataset import get as get_dataset
-from .experiments import exp_save, exp_show
-from .scm import all_branches, all_commits, all_tags
-from .show import metrics_show, params_show
-
-__all__ = [
-    "DVCFileSystem",
-    "all_branches",
-    "all_commits",
-    "all_tags",
-    "artifacts_show",
-    "exp_save",
-    "exp_show",
-    "get_dataset",
-    "get_url",
-    "metrics_show",
-    "open",
-    "params_show",
-    "read",
-]

dvc/api/artifacts.py

@@ -1,58 +0,0 @@
-import os
-from typing import Any, Optional
-
-from dvc.repo import Repo
-
-
-def artifacts_show(
-    name: str,
-    version: Optional[str] = None,
-    stage: Optional[str] = None,
-    repo: Optional[str] = None,
-) -> dict[str, str]:
-    """
-    Return path and Git revision for an artifact in a DVC project.
-
-    The resulting path and revision can be used in conjunction with other dvc.api
-    calls to open and read the artifact.
-
-    Args:
-        name (str): name of the artifact to open.
-        version (str, optional): version of the artifact to open. Defaults to
-            the latest version.
-        stage (str, optional): name of the model registry stage.
-        repo: (str, optional): path or URL for the DVC repo.
-
-    Returns:
-        Dictionary of the form:
-            {
-                "rev": ...,
-                "path": ...,
-            }
-
-    Raises:
-        dvc.exceptions.ArtifactNotFoundError: The specified artifact was not found in
-            the repo.
-    """
-    if version and stage:
-        raise ValueError("Artifact version and stage are mutually exclusive.")
-
-    from dvc.repo.artifacts import Artifacts
-    from dvc.utils import as_posix
-
-    repo_kwargs: dict[str, Any] = {
-        "subrepos": True,
-        "uninitialized": True,
-    }
-
-    dirname, _ = Artifacts.parse_path(name)
-    with Repo.open(repo, **repo_kwargs) as _repo:
-        rev = _repo.artifacts.get_rev(name, version=version, stage=stage)
-        with _repo.switch(rev):
-            root = _repo.fs.root_marker
-            _dirname = _repo.fs.join(root, dirname) if dirname else root
-            with Repo(_dirname, fs=_repo.fs, scm=_repo.scm) as r:
-                path = r.artifacts.get_path(name)
-                path = _repo.fs.join(_repo.fs.root_marker, as_posix(path))
-                parts = _repo.fs.relparts(path, _repo.root_dir)
-                return {"rev": rev, "path": os.path.join(*parts)}

dvc/api/data.py

@@ -1,330 +0,0 @@
-from contextlib import _GeneratorContextManager as GCM
-from contextlib import contextmanager
-from typing import Any, Optional
-
-from funcy import reraise
-
-from dvc.exceptions import FileMissingError, OutputNotFoundError, PathMissingError
-from dvc.repo import Repo
-
-
-@contextmanager
-def _wrap_exceptions(repo, url):
-    from dvc.config import NoRemoteError
-    from dvc.exceptions import NoOutputInExternalRepoError, NoRemoteInExternalRepoError
-
-    try:
-        yield
-    except NoRemoteError as exc:
-        raise NoRemoteInExternalRepoError(url) from exc
-    except OutputNotFoundError as exc:
-        if exc.repo is repo:
-            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
-        raise
-    except FileMissingError as exc:
-        raise PathMissingError(exc.path, url) from exc
-
-
-def get_url(
-    path: str,
-    repo: Optional[str] = None,
-    rev: Optional[str] = None,
-    remote: Optional[str] = None,
-    config: Optional[dict[str, Any]] = None,
-    remote_config: Optional[dict[str, Any]] = None,
-):
-    """
-    Returns the URL to the storage location of a data file or directory tracked
-    in a DVC repo. For Git repos, HEAD is used unless a rev argument is
-    supplied. The default remote is tried unless a remote argument is supplied.
-
-    Raises OutputNotFoundError if the file is not tracked by DVC.
-
-    NOTE: This function does not check for the actual existence of the file or
-    directory in the remote storage.
-
-    Args:
-        path (str): location and file name of the target, relative to the root
-            of `repo`.
-        repo (str, optional): location of the DVC project or Git Repo.
-            Defaults to the current DVC project (found by walking up from the
-            current working directory tree).
-            It can be a URL or a file system path.
-            Both HTTP and SSH protocols are supported for online Git repos
-            (e.g. [user@]server:project.git).
-        rev (str, optional): Any `Git revision`_ such as a branch or tag name,
-            a commit hash or a dvc experiment name.
-            Defaults to HEAD.
-            If `repo` is not a Git repo, this option is ignored.
-        remote (str, optional): Name of the `DVC remote`_ used to form the
-            returned URL string.
-            Defaults to the `default remote`_ of `repo`.
-            For local projects, the cache is tried before the default remote.
-        config(dict, optional): config to be passed to the DVC repository.
-            Defaults to None.
-        remote_config(dict, optional): remote config to be passed to the DVC
-            repository.
-            Defaults to None.
-
-    Returns:
-        str: URL to the file or directory.
-    """
-    from dvc.config import NoRemoteError
-    from dvc_data.index import StorageKeyError
-
-    with Repo.open(
-        repo,
-        rev=rev,
-        subrepos=True,
-        uninitialized=True,
-        remote=remote,
-        config=config,
-        remote_config=remote_config,
-    ) as _repo:
-        index, entry = _repo.get_data_index_entry(path)
-        with reraise(
-            (StorageKeyError, ValueError),
-            NoRemoteError(f"no remote specified in {_repo}"),
-        ):
-            remote_fs, remote_path = index.storage_map.get_remote(entry)
-            return remote_fs.unstrip_protocol(remote_path)
-
-
-class _OpenContextManager(GCM):
-    def __init__(self, func, args, kwds):
-        self.gen = func(*args, **kwds)
-        self.func, self.args, self.kwds = (func, args, kwds)  # type: ignore[assignment]
-
-    def __getattr__(self, name):
-        raise AttributeError("dvc.api.open() should be used in a with statement.")
-
-
-def open(  # noqa: A001
-    path: str,
-    repo: Optional[str] = None,
-    rev: Optional[str] = None,
-    remote: Optional[str] = None,
-    mode: str = "r",
-    encoding: Optional[str] = None,
-    config: Optional[dict[str, Any]] = None,
-    remote_config: Optional[dict[str, Any]] = None,
-):
-    """
-    Opens a file tracked in a DVC project.
-
-    This function may only be used as a context manager (using the `with`
-    keyword, as shown in the examples).
-
-    This function makes a direct connection to the remote storage, so the file
-    contents can be streamed. Your code can process the data buffer as it's
-    streamed, which optimizes memory usage.
-
-    Note:
-        Use dvc.api.read() to load the complete file contents
-        in a single function call, no context manager involved.
-        Neither function utilizes disc space.
-
-    Args:
-        path (str): location and file name of the target to open,
-        relative to the root of `repo`.
-        repo (str, optional): location of the DVC project or Git Repo.
-            Defaults to the current DVC project (found by walking up from the
-            current working directory tree).
-            It can be a URL or a file system path.
-            Both HTTP and SSH protocols are supported for online Git repos
-            (e.g. [user@]server:project.git).
-        rev (str, optional): Any `Git revision`_ such as a branch or tag name,
-            a commit hash or a dvc experiment name.
-            Defaults to HEAD.
-            If `repo` is not a Git repo, this option is ignored.
-        remote (str, optional): Name of the `DVC remote`_ used to form the
-            returned URL string.
-            Defaults to the `default remote`_ of `repo`.
-            For local projects, the cache is tried before the default remote.
-        mode (str, optional): Specifies the mode in which the file is opened.
-            Defaults to "r" (read).
-            Mirrors the namesake parameter in builtin `open()`_.
-            Only reading `mode` is supported.
-        encoding(str, optional): `Codec`_ used to decode the file contents.
-            Defaults to None.
-            This should only be used in text mode.
-            Mirrors the namesake parameter in builtin `open()`_.
-        config(dict, optional): config to be passed to the DVC repository.
-            Defaults to None.
-        remote_config(dict, optional): remote config to be passed to the DVC
-            repository.
-            Defaults to None.
-
-    Returns:
-        _OpenContextManager: A context manager that generatse a corresponding
-            `file object`_.
-            The exact type of file object depends on the mode used.
-            For more details, please refer to Python's `open()`_ built-in,
-            which is used under the hood.
-
-    Raises:
-        AttributeError: If this method is not used as a context manager.
-        ValueError: If non-read `mode` is used.
-
-    Examples:
-
-        - Use data or models from a DVC repository.
-
-        Any file tracked in a DVC project (and stored remotely) can be
-        processed directly in your Python code with this API.
-        For example, an XML file tracked in a public DVC repo on GitHub can be
-        processed like this:
-
-        >>> from xml.sax import parse
-        >>> import dvc.api
-        >>> from mymodule import mySAXHandler
-
-        >>> with dvc.api.open(
-        ...     'get-started/data.xml',
-        ...     repo='https://github.com/iterative/dataset-registry'
-        ... ) as fd:
-        ...     parse(fd, mySAXHandler)
-
-        We use a SAX XML parser here because dvc.api.open() is able to stream
-        the data from remote storage.
-        The mySAXHandler object should handle the event-driven parsing of the
-        document in this case.
-        This increases the performance of the code (minimizing memory usage),
-        and is typically faster than loading the whole data into memory.
-
-        - Accessing private repos
-
-        This is just a matter of using the right repo argument, for example an
-        SSH URL (requires that the credentials are configured locally):
-
-        >>> import dvc.api
-
-        >>> with dvc.api.open(
-        ...     'features.dat',
-        ...     repo='[email protected]:path/to/repo.git'
-        ... ) as fd:
-        ...     # ... Process 'features'
-        ...     pass
-
-        - Use different versions of data
-
-        Any git revision (see `rev`) can be accessed programmatically.
-        For example, if your DVC repo has tagged releases of a CSV dataset:
-
-        >>> import csv
-        >>> import dvc.api
-        >>> with dvc.api.open(
-        ...     'clean.csv',
-        ...     rev='v1.1.0'
-        ... ) as fd:
-        ...     reader = csv.reader(fd)
-        ...     # ... Process 'clean' data from version 1.1.0
-
-    .. _Git revision:
-        https://git-scm.com/docs/revisions
-
-    .. _DVC remote:
-        https://dvc.org/doc/command-reference/remote
-
-    .. _default remote:
-        https://dvc.org/doc/command-reference/remote/default
-
-    .. _open():
-        https://docs.python.org/3/library/functions.html#open
-
-    .. _Codec:
-        https://docs.python.org/3/library/codecs.html#standard-encodings
-
-    .. _file object:
-        https://docs.python.org/3/glossary.html#term-file-object
-
-    """
-    if "r" not in mode:
-        raise ValueError("Only reading `mode` is supported.")
-
-    args = (path,)
-    kwargs = {
-        "repo": repo,
-        "remote": remote,
-        "rev": rev,
-        "mode": mode,
-        "encoding": encoding,
-        "config": config,
-        "remote_config": remote_config,
-    }
-    return _OpenContextManager(_open, args, kwargs)
-
-
-def _open(
-    path,
-    repo=None,
-    rev=None,
-    remote=None,
-    mode="r",
-    encoding=None,
-    config=None,
-    remote_config=None,
-):
-    repo_kwargs: dict[str, Any] = {
-        "subrepos": True,
-        "uninitialized": True,
-        "remote": remote,
-        "config": config,
-        "remote_config": remote_config,
-    }
-
-    with Repo.open(repo, rev=rev, **repo_kwargs) as _repo:
-        with _wrap_exceptions(_repo, path):
-            import os
-            from typing import TYPE_CHECKING, Union
-
-            from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
-            from dvc.fs.data import DataFileSystem
-            from dvc.fs.dvc import DVCFileSystem
-
-            if TYPE_CHECKING:
-                from dvc.fs import FileSystem
-
-            fs: Union[FileSystem, DataFileSystem, DVCFileSystem]
-            if os.path.isabs(path):
-                fs = DataFileSystem(index=_repo.index.data["local"])
-                fs_path = path
-            else:
-                fs = DVCFileSystem(repo=_repo, subrepos=True)
-                fs_path = fs.from_os_path(path)
-
-            try:
-                with fs.open(fs_path, mode=mode, encoding=encoding) as fobj:
-                    yield fobj
-            except FileNotFoundError as exc:
-                raise FileMissingError(path) from exc
-            except IsADirectoryError as exc:
-                raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
-
-
-def read(
-    path,
-    repo=None,
-    rev=None,
-    remote=None,
-    mode="r",
-    encoding=None,
-    config=None,
-    remote_config=None,
-):
-    """
-    Returns the contents of a tracked file (by DVC or Git). For Git repos, HEAD
-    is used unless a rev argument is supplied. The default remote is tried
-    unless a remote argument is supplied.
-    """
-    with open(
-        path,
-        repo=repo,
-        rev=rev,
-        remote=remote,
-        mode=mode,
-        encoding=encoding,
-        config=config,
-        remote_config=remote_config,
-    ) as fd:
-        return fd.read()

dvc/api/dataset.py

@@ -1,67 +0,0 @@
-from typing import Literal, TypedDict, Union
-
-
-class DatachainDataset(TypedDict):
-    type: Literal["dc"]
-    name: str
-    version: int
-
-
-class DVCDataset(TypedDict):
-    type: Literal["dvc"]
-    url: str
-    path: str
-    sha: str
-
-
-class URLDataset(TypedDict):
-    type: Literal["url"]
-    files: list[str]
-    path: str
-
-
-def get(name: str) -> Union[DatachainDataset, DVCDataset, URLDataset]:
-    from difflib import get_close_matches
-
-    from dvc.fs import get_cloud_fs
-    from dvc.repo import Repo, datasets
-
-    repo = Repo()
-    try:
-        dataset = repo.datasets[name]
-    except datasets.DatasetNotFoundError as e:
-        add_note = getattr(e, "add_note", lambda _: None)
-        if matches := get_close_matches(name, repo.datasets):
-            add_note(f"Did you mean: {matches[0]!r}?")
-        raise
-
-    if dataset._invalidated:
-        raise ValueError(f"dataset not in sync. Sync with 'dvc ds update {name}'.")
-    if not dataset.lock:
-        raise ValueError("missing lock information")
-    if dataset.type == "dvc":
-        return DVCDataset(
-            type="dvc",
-            url=dataset.lock.url,
-            path=dataset.lock.path,
-            sha=dataset.lock.rev_lock,
-        )
-    if dataset.type == "dc":
-        return DatachainDataset(
-            type="dc", name=dataset.name_version[0], version=dataset.lock.version
-        )
-    if dataset.type == "url":
-        fs_cls, _, path = get_cloud_fs(repo.config, url=dataset.lock.url)
-        assert fs_cls
-        join_version = getattr(fs_cls, "join_version", lambda path, _: path)
-        protocol = fs_cls.protocol
-        versioned_path = join_version(path, dataset.lock.meta.version_id)
-        versioned_path = f"{protocol}://{versioned_path}"
-        files = [
-            join_version(
-                fs_cls.join(versioned_path, file.relpath), file.meta.version_id
-            )
-            for file in dataset.lock.files
-        ]
-        return URLDataset(type="url", files=files, path=versioned_path)
-    raise AssertionError("unreachable")

dvc/api/experiments.py

@@ -1,120 +0,0 @@
-from typing import Optional, Union
-
-from rich.text import Text
-
-from dvc.repo import Repo
-from dvc.repo.experiments.show import tabulate
-
-
-def exp_save(
-    name: Optional[str] = None,
-    force: bool = False,
-    include_untracked: Optional[list[str]] = None,
-):
-    """
-    Create a new DVC experiment using `exp save`.
-
-    See https://dvc.org/doc/command-reference/exp/save.
-
-    Args:
-        name (str, optional): specify a name for this experiment.
-            If `None`, a default one will be generated, such as `urban-sign`.
-            Defaults to `None`.
-        force (bool):  overwrite the experiment if an experiment with the same
-            name already exists.
-            Defaults to `False`.
-        include_untracked (List[str], optional): specify untracked file(s) to
-            be included in the saved experiment.
-            Defaults to `None`.
-
-    Returns:
-        str: The `Git revision`_ of the created experiment.
-
-    Raises:
-        ExperimentExistsError: If an experiment with `name` already exists and
-            `force=False`.
-
-    .. _Git revision:
-        https://git-scm.com/docs/revisions
-    """
-    with Repo() as repo:
-        return repo.experiments.save(
-            name=name, force=force, include_untracked=include_untracked
-        )
-
-
-def _postprocess(exp_rows):
-    for exp_row in exp_rows:
-        for k, v in exp_row.items():
-            if isinstance(v, Text):
-                v_str = str(v)
-                try:
-                    exp_row[k] = float(v_str)
-                except ValueError:
-                    exp_row[k] = v_str
-
-            elif not exp_row[k]:
-                exp_row[k] = None
-
-    return exp_rows
-
-
-def exp_show(
-    repo: Optional[str] = None,
-    revs: Optional[Union[str, list[str]]] = None,
-    num: int = 1,
-    param_deps: bool = False,
-    force: bool = False,
-    config: Optional[dict] = None,
-) -> list[dict]:
-    """Get DVC experiments tracked in `repo`.
-
-    Without arguments, this function will retrieve all experiments derived from
-    the Git `HEAD`.
-
-    See the options below to customize the experiments retrieved.
-
-    Args:
-        repo (str, optional): location of the DVC repository.
-            Defaults to the current project (found by walking up from the
-            current working directory tree).
-            It can be a URL or a file system path.
-            Both HTTP and SSH protocols are supported for online Git repos
-            (e.g. [user@]server:project.git).
-        revs (Union[str, List[str]], optional): Git revision(s) (e.g. branch,
-            tag, SHA commit) to use as a reference point to start listing
-            experiments.
-            Defaults to `None`, which will use `HEAD` as starting point.
-        num (int, optional): show experiments from the last `num` commits
-            (first parents) starting from the `revs` baseline.
-            Give a negative value to include all first-parent commits (similar
-            to `git log -n`).
-            Defaults to 1.
-        param_deps (bool, optional): include only parameters that are stage
-            dependencies.
-            Defaults to `False`.
-        force (bool, optional): force re-collection of experiments instead of
-            loading from internal experiments cache.
-            DVC caches `exp_show` data for completed experiments to improve
-            performance of subsequent calls.
-            When `force` is specified, DVC will reload all experiment data and
-            ignore any previously cached results.
-            Defaults to `False`.
-        config (dict, optional): config to be passed through to DVC project.
-            Defaults to `None`.
-
-    Returns:
-        List[Dict]: Each item in the list will contain a dictionary with
-            the info for an individual experiment.
-            See Examples below.
-    """
-    with Repo.open(repo, config=config) as _repo:
-        experiments = _repo.experiments.show(
-            revs=revs,
-            num=num,
-            param_deps=param_deps,
-            force=force,
-        )
-        td, _ = tabulate(experiments, fill_value=None)
-
-        return _postprocess(td.as_dict())

dvc/api/scm.py

@@ -1,54 +0,0 @@
-from typing import Optional
-
-from dvc.repo import Repo
-
-
-def all_branches(repo: Optional[str] = None) -> list[str]:
-    """Get all Git branches in a DVC repository.
-
-    Args:
-        repo (str, optional): location of the DVC repository.
-            Defaults to the current project (found by walking up from the
-            current working directory tree).
-            It can be a URL or a file system path.
-            Both HTTP and SSH protocols are supported for online Git repos
-            (e.g. [user@]server:project.git).
-    Returns:
-        List[str]: Names of the Git branches.
-    """
-    with Repo.open(repo) as _repo:
-        return _repo.scm.list_branches()
-
-
-def all_commits(repo: Optional[str] = None) -> list[str]:
-    """Get all Git commits in a DVC repository.
-
-    Args:
-        repo (str, optional): location of the DVC repository.
-            Defaults to the current project (found by walking up from the
-            current working directory tree).
-            It can be a URL or a file system path.
-            Both HTTP and SSH protocols are supported for online Git repos
-            (e.g. [user@]server:project.git).
-    Returns:
-        List[str]: SHAs of the Git commits.
-    """
-    with Repo.open(repo) as _repo:
-        return _repo.scm.list_all_commits()
-
-
-def all_tags(repo: Optional[str] = None) -> list[str]:
-    """Get all Git tags in a DVC repository.
-
-    Args:
-        repo (str, optional): location of the DVC repository.
-            Defaults to the current project (found by walking up from the
-            current working directory tree).
-            It can be a URL or a file system path.
-            Both HTTP and SSH protocols are supported for online Git repos
-            (e.g. [user@]server:project.git).
-    Returns:
-        List[str]: Names of the Git tags.
-    """
-    with Repo.open(repo) as _repo:
-        return _repo.scm.list_tags()

dvc/api/show.py

@@ -1,404 +0,0 @@
-import typing
-from collections import Counter
-from collections.abc import Iterable
-from typing import Optional, Union
-
-from funcy import first
-
-from dvc.repo import Repo
-
-
-def _postprocess(results):
-    processed: dict[str, dict] = {}
-    for rev, rev_data in results.items():
-        if not rev_data:
-            continue
-
-        processed[rev] = {}
-
-        counts: typing.Counter[str] = Counter()
-        for file_data in rev_data["data"].values():
-            for k in file_data["data"]:
-                counts[k] += 1
-        for file_name, file_data in rev_data["data"].items():
-            to_merge = {
-                (k if counts[k] == 1 else f"{file_name}:{k}"): v
-                for k, v in file_data["data"].items()
-            }
-            processed[rev] = processed[rev] | to_merge
-
-    processed.pop("workspace", None)
-
-    return processed
-
-
-def metrics_show(
-    *targets: str,
-    repo: Optional[str] = None,
-    rev: Optional[str] = None,
-    config: Optional[dict] = None,
-) -> dict:
-    """Get metrics tracked in `repo`.
-
-    Without arguments, this function will retrieve all metrics from all tracked
-    metric files, for the current working tree.
-
-    See the options below to restrict the metrics retrieved.
-
-    Args:
-        *targets (str, optional): Names of the metric files to retrieve
-        metrics from. For example, "classifier_eval.json,
-        clustering_eval.json".
-        If no `targets` are provided, all metric files tracked in `dvc.yaml`
-        will be used.
-        Note that targets don't necessarily have to be defined in `dvc.yaml`.
-        repo (str, optional): Location of the DVC repository.
-            Defaults to the current project (found by walking up from the
-            current working directory tree).
-            It can be a URL or a file system path.
-            Both HTTP and SSH protocols are supported for online Git repos
-            (e.g. [user@]server:project.git).
-        rev (str, optional): Name of the `Git revision`_ to retrieve metrics
-            from.
-            Defaults to `None`.
-            An example of git revision can be a branch or tag name, a commit
-            hash or a dvc experiment name.
-            If `repo` is not a Git repo, this option is ignored.
-            If `None`, the current working tree will be used.
-        config (dict, optional): config to be passed through to DVC project.
-            Defaults to `None`.
-
-    Returns:
-        Dict: See Examples below.
-
-    Examples:
-
-        - No arguments.
-
-        Working on https://github.com/iterative/example-get-started
-
-        >>> import dvc.api
-        >>> metrics = dvc.api.metrics_show()
-        >>> print(json.dumps(metrics, indent=4))
-        {
-            "avg_prec": 0.9249974999612706,
-            "roc_auc": 0.9460213440787918
-        }
-
-        ---
-
-        - Using `rev`.
-
-        Working on https://github.com/iterative/example-get-started
-
-        >>> import json
-        >>> import dvc.api
-        >>> metrics = dvc.api.metrics_show(rev="tune-hyperparams")
-        >>> print(json.dumps(metrics, indent=4))
-        {
-            "avg_prec": 0.9268792615819422,
-            "roc_auc": 0.945093365854111
-        }
-
-        ---
-
-        - Using `targets`.
-
-        Working on https://github.com/iterative/example-get-started
-
-        >>> import json
-        >>> import dvc.api
-        >>> metrics = dvc.api.metrics_show("evaluation.json")
-        >>> print(json.dumps(metrics, indent=4))
-        {
-            "avg_prec": 0.9249974999612706,
-            "roc_auc": 0.9460213440787918
-        }
-
-        ---
-
-        - Git URL as `repo`.
-
-        >>> import json
-        >>> import dvc.api
-        >>> metrics = dvc.api.metrics_show(
-        ...     repo="https://github.com/iterative/demo-fashion-mnist")
-        >>> print(json.dumps(metrics, indent=4))
-        {
-            "loss": 0.25284987688064575,
-            "accuracy": 0.9071000218391418
-        }
-
-
-    .. _Git revision:
-        https://git-scm.com/docs/revisions
-    """
-    from dvc.repo.metrics.show import to_relpath
-
-    with Repo.open(repo, config=config) as _repo:
-        metrics = _repo.metrics.show(
-            targets=targets,
-            revs=rev if rev is None else [rev],
-            on_error="raise",
-        )
-        metrics = {
-            k: to_relpath(_repo.fs, _repo.root_dir, v) for k, v in metrics.items()
-        }
-
-    metrics = _postprocess(metrics)
-
-    if not metrics:
-        return {}
-
-    return metrics[first(metrics)]
-
-
-def params_show(
-    *targets: str,
-    repo: Optional[str] = None,
-    stages: Optional[Union[str, Iterable[str]]] = None,
-    rev: Optional[str] = None,
-    deps: bool = False,
-    config: Optional[dict] = None,
-) -> dict:
-    """Get parameters tracked in `repo`.
-
-    Without arguments, this function will retrieve all params from all tracked
-    parameter files, for the current working tree.
-
-    See the options below to restrict the parameters retrieved.
-
-    Args:
-        *targets (str, optional): Names of the parameter files to retrieve
-        params from. For example, "params.py, myparams.toml".
-        If no `targets` are provided, all parameter files tracked in `dvc.yaml`
-        will be used.
-        Note that targets don't necessarily have to be defined in `dvc.yaml`.
-        repo (str, optional): location of the DVC repository.
-            Defaults to the current project (found by walking up from the
-            current working directory tree).
-            It can be a URL or a file system path.
-            Both HTTP and SSH protocols are supported for online Git repos
-            (e.g. [user@]server:project.git).
-        stages (Union[str, Iterable[str]], optional): Name or names of the
-            stages to retrieve parameters from.
-            Defaults to `None`.
-            If `None`, all parameters from all stages will be retrieved.
-            If this method is called from a different location to the one where
-            the `dvc.yaml` is found, the relative path to the `dvc.yaml` must
-            be provided as a prefix with the syntax `{relpath}:{stage}`.
-            For example: `subdir/dvc.yaml:stage-0` or `../dvc.yaml:stage-1`.
-        rev (str, optional): Name of the `Git revision`_ to retrieve parameters
-            from.
-            Defaults to `None`.
-            An example of git revision can be a branch or tag name, a commit
-            hash or a dvc experiment name.
-            If `repo` is not a Git repo, this option is ignored.
-            If `None`, the current working tree will be used.
-        deps (bool, optional): Whether to retrieve only parameters that are
-            stage dependencies or not.
-            Defaults to `False`.
-        config (dict, optional): config to be passed through to DVC project.
-            Defaults to `None`.
-
-    Returns:
-        Dict: See Examples below.
-
-    Examples:
-
-        - No arguments.
-
-        Working on https://github.com/iterative/example-get-started
-
-        >>> import json
-        >>> import dvc.api
-        >>> params = dvc.api.params_show()
-        >>> print(json.dumps(params, indent=4))
-        {
-            "prepare": {
-                "split": 0.2,
-                "seed": 20170428
-            },
-            "featurize": {
-                "max_features": 200,
-                "ngrams": 2
-            },
-            "train": {
-                "seed": 20170428,
-                "n_est": 50,
-                "min_split": 0.01
-            }
-        }
-
-        ---
-
-        - Filtering with `stages`.
-
-        Working on https://github.com/iterative/example-get-started
-
-        `stages` can a single string:
-
-        >>> import json
-        >>> import dvc.api
-        >>> params = dvc.api.params_show(stages="prepare")
-        >>> print(json.dumps(params, indent=4))
-        {
-            "prepare": {
-                "split": 0.2,
-                "seed": 20170428
-            }
-        }
-
-        Or an iterable of strings:
-
-        >>> import json
-        >>> import dvc.api
-        >>> params = dvc.api.params_show(stages=["prepare", "train"])
-        >>> print(json.dumps(params, indent=4))
-        {
-            "prepare": {
-                "split": 0.2,
-                "seed": 20170428
-            },
-            "train": {
-                "seed": 20170428,
-                "n_est": 50,
-                "min_split": 0.01
-            }
-        }
-
-        ---
-
-        - Using `rev`.
-
-        Working on https://github.com/iterative/example-get-started
-
-        >>> import json
-        >>> import dvc.api
-        >>> params = dvc.api.params_show(rev="tune-hyperparams")
-        >>> print(json.dumps(params, indent=4))
-        {
-            "prepare": {
-                "split": 0.2,
-                "seed": 20170428
-            },
-            "featurize": {
-                "max_features": 200,
-                "ngrams": 2
-            },
-            "train": {
-                "seed": 20170428,
-                "n_est": 100,
-                "min_split": 8
-            }
-        }
-
-        ---
-
-        - Using `targets`.
-
-        Working on `multi-params-files` folder of
-        https://github.com/iterative/pipeline-conifguration
-
-        You can pass a single target:
-
-        >>> import json
-        >>> import dvc.api
-        >>> params = dvc.api.params_show("params.yaml")
-        >>> print(json.dumps(params, indent=4))
-        {
-            "run_mode": "prod",
-            "configs": {
-                "dev": "configs/params_dev.yaml",
-                "test": "configs/params_test.yaml",
-                "prod": "configs/params_prod.yaml"
-            },
-            "evaluate": {
-                "dataset": "micro",
-                "size": 5000,
-                "metrics": ["f1", "roc-auc"],
-                "metrics_file": "reports/metrics.json",
-                "plots_cm": "reports/plot_confusion_matrix.png"
-            }
-        }
-
-
-        Or multiple targets:
-
-        >>> import json
-        >>> import dvc.api
-        >>> params = dvc.api.params_show(
-        ...     "configs/params_dev.yaml", "configs/params_prod.yaml")
-        >>> print(json.dumps(params, indent=4))
-        {
-            "configs/params_prod.yaml:run_mode": "prod",
-            "configs/params_prod.yaml:config_file": "configs/params_prod.yaml",
-            "configs/params_prod.yaml:data_load": {
-                "dataset": "large",
-                "sampling": {
-                "enable": true,
-                "size": 50000
-                }
-            },
-            "configs/params_prod.yaml:train": {
-                "epochs": 1000
-            },
-            "configs/params_dev.yaml:run_mode": "dev",
-            "configs/params_dev.yaml:config_file": "configs/params_dev.yaml",
-            "configs/params_dev.yaml:data_load": {
-                "dataset": "development",
-                "sampling": {
-                "enable": true,
-                "size": 1000
-                }
-            },
-            "configs/params_dev.yaml:train": {
-                "epochs": 10
-            }
-        }
-
-        ---
-
-        - Git URL as `repo`.
-
-        >>> import json
-        >>> import dvc.api
-        >>> params = dvc.api.params_show(
-        ...     repo="https://github.com/iterative/demo-fashion-mnist")
-        {
-            "train": {
-                "batch_size": 128,
-                "hidden_units": 64,
-                "dropout": 0.4,
-                "num_epochs": 10,
-                "lr": 0.001,
-                "conv_activation": "relu"
-            }
-        }
-
-
-    .. _Git revision:
-        https://git-scm.com/docs/revisions
-
-    """
-    from dvc.repo.metrics.show import to_relpath
-
-    if isinstance(stages, str):
-        stages = [stages]
-
-    with Repo.open(repo, config=config) as _repo:
-        params = _repo.params.show(
-            revs=rev if rev is None else [rev],
-            targets=targets,
-            deps_only=deps,
-            on_error="raise",
-            stages=stages,
-        )
-        params = {k: to_relpath(_repo.fs, _repo.root_dir, v) for k, v in params.items()}
-
-    params = _postprocess(params)
-
-    if not params:
-        return {}
-
-    return params[first(params)]

dvc/build.py

@@ -1,9 +0,0 @@
-from typing import Optional
-
-try:
-    # file is created during dvc build
-    from . import _build  # type: ignore[attr-defined, import]
-
-    PKG: Optional[str] = _build.PKG  # type: ignore[assignment]
-except ImportError:
-    PKG = None  # type: ignore[assignment]

dvc/cache/__init__.py

@@ -0,0 +1,176 @@
+"""Manages cache of a DVC repo."""
+from collections import defaultdict
+
+from ..scheme import Schemes
+
+
+def get_cloud_cache(tree):
+    from .base import CloudCache
+    from .local import LocalCache
+    from .ssh import SSHCache
+
+    if tree.scheme == Schemes.LOCAL:
+        return LocalCache(tree)
+
+    if tree.scheme == Schemes.SSH:
+        return SSHCache(tree)
+
+    return CloudCache(tree)
+
+
+def _get_cache(repo, settings):
+    from ..tree import get_cloud_tree
+
+    if not settings:
+        return None
+
+    tree = get_cloud_tree(repo, **settings)
+    return get_cloud_cache(tree)
+
+
+class Cache:
+    """Class that manages cache locations of a DVC repo.
+
+    Args:
+        repo (dvc.repo.Repo): repo instance that this cache belongs to.
+    """
+
+    CACHE_DIR = "cache"
+    CLOUD_SCHEMES = [
+        Schemes.S3,
+        Schemes.GS,
+        Schemes.SSH,
+        Schemes.HDFS,
+        Schemes.WEBHDFS,
+    ]
+
+    def __init__(self, repo):
+        self.repo = repo
+        self.config = config = repo.config["cache"]
+        self._cache = {}
+
+        local = config.get("local")
+
+        if local:
+            settings = {"name": local}
+        elif "dir" not in config:
+            settings = None
+        else:
+            from ..config import LOCAL_COMMON
+
+            settings = {"url": config["dir"]}
+            for opt in LOCAL_COMMON.keys():
+                if opt in config:
+                    settings[str(opt)] = config.get(opt)
+
+        self._cache[Schemes.LOCAL] = _get_cache(repo, settings)
+
+        for scheme in self.CLOUD_SCHEMES:
+            remote = self.config.get(scheme)
+            settings = {"name": remote} if remote else None
+            self._cache[scheme] = _get_cache(repo, settings)
+
+    def __getattr__(self, name):
+        try:
+            return self._cache[name]
+        except KeyError as exc:
+            raise AttributeError from exc
+
+    def by_scheme(self):
+        yield from self._cache.items()
+
+
+class NamedCacheItem:
+    def __init__(self):
+        self.names = set()
+        self.children = defaultdict(NamedCacheItem)
+
+    def __eq__(self, other):
+        return self.names == other.names and self.children == other.children
+
+    def child_keys(self):
+        for key, child in self.children.items():
+            yield key
+            yield from child.child_keys()
+
+    def child_names(self):
+        for key, child in self.children.items():
+            yield key, child.names
+            yield from child.child_names()
+
+    def add(self, checksum, item):
+        self.children[checksum].update(item)
+
+    def update(self, item, suffix=""):
+        if suffix:
+            self.names.update(n + suffix for n in item.names)
+        else:
+            self.names.update(item.names)
+        for checksum, child_item in item.children.items():
+            self.children[checksum].update(child_item)
+
+
+class NamedCache:
+    # pylint: disable=protected-access
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
+        self.external = defaultdict(set)
+
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
+
+    def __getitem__(self, key):
+        return self._items[key]
+
+    def add(self, scheme, checksum, name):
+        """Add a mapped name for the specified checksum."""
+        self._items[scheme][checksum].names.add(name)
+
+    def add_child_cache(self, checksum, cache, suffix=""):
+        """Add/update child cache for the specified checksum."""
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme][checksum].children
+            for child_checksum, item in src.items():
+                dst[child_checksum].update(item, suffix=suffix)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)
+
+    def add_external(self, url, rev, path):
+        self.external[url, rev].add(path)
+
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, item in src.items():
+                dst[checksum].update(item, suffix=suffix)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)
+
+    def scheme_keys(self, scheme):
+        """Iterate over a flat list of all keys for the specified scheme,
+        including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key
+            yield from item.child_keys()
+
+    def scheme_names(self, scheme):
+        """Iterate over a flat list of checksum, names items for the specified
+        scheme, including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key, item.names
+            yield from item.child_names()
+
+    def dir_keys(self, scheme):
+        return (
+            key for key, item in self._items[scheme].items() if item.children
+        )
+
+    def child_keys(self, scheme, checksum):
+        return self._items[scheme][checksum].child_keys()

dvc/cache/base.py

@@ -0,0 +1,937 @@
+import itertools
+import json
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from copy import copy
+from typing import Optional
+
+from funcy import decorator
+from shortuuid import uuid
+
+import dvc.prompt as prompt
+from dvc.dir_info import DirInfo
+from dvc.exceptions import (
+    CacheLinkError,
+    CheckoutError,
+    ConfirmRemoveError,
+    DvcException,
+)
+from dvc.progress import Tqdm
+from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
+    slow_link_guard,
+)
+
+from ..tree.base import RemoteActionNotImplemented
+
+logger = logging.getLogger(__name__)
+
+
+class DirCacheError(DvcException):
+    def __init__(self, hash_):
+        super().__init__(
+            f"Failed to load dir cache for hash value: '{hash_}'."
+        )
+
+
+@decorator
+def use_state(call):
+    tree = call._args[0].tree  # pylint: disable=protected-access
+    with tree.state:
+        return call()
+
+
+class CloudCache:
+    """Cloud cache class."""
+
+    DEFAULT_CACHE_TYPES = ["copy"]
+    CACHE_MODE: Optional[int] = None
+
+    def __init__(self, tree):
+        self.tree = tree
+        self.repo = tree.repo
+
+        self.cache_types = tree.config.get("type") or copy(
+            self.DEFAULT_CACHE_TYPES
+        )
+        self.cache_type_confirmed = False
+        self._dir_info = {}
+
+    def get_dir_cache(self, hash_info):
+        assert hash_info
+
+        dir_info = self._dir_info.get(hash_info.value)
+        if dir_info:
+            return dir_info
+
+        try:
+            dir_info = self.load_dir_cache(hash_info)
+        except DirCacheError:
+            dir_info = DirInfo()
+
+        self._dir_info[hash_info.value] = dir_info
+        return dir_info
+
+    def load_dir_cache(self, hash_info):
+        path_info = self.tree.hash_to_path_info(hash_info.value)
+
+        try:
+            with self.tree.open(path_info, "r") as fobj:
+                d = json.load(fobj)
+        except (ValueError, FileNotFoundError) as exc:
+            raise DirCacheError(hash_info) from exc
+
+        if not isinstance(d, list):
+            logger.error(
+                "dir cache file format error '%s' [skipping the file]",
+                path_info,
+            )
+            d = []
+
+        return DirInfo.from_list(d)
+
+    def changed(self, path_info, hash_info):
+        """Checks if data has changed.
+
+        A file is considered changed if:
+            - It doesn't exist on the working directory (was unlinked)
+            - Hash value is not computed (saving a new file)
+            - The hash value stored is different from the given one
+            - There's no file in the cache
+
+        Args:
+            path_info: dict with path information.
+            hash: expected hash value for this data.
+
+        Returns:
+            bool: True if data has changed, False otherwise.
+        """
+
+        logger.debug(
+            "checking if '%s'('%s') has changed.", path_info, hash_info
+        )
+
+        if not self.tree.exists(path_info):
+            logger.debug("'%s' doesn't exist.", path_info)
+            return True
+
+        if not hash_info:
+            logger.debug("hash value for '%s' is missing.", path_info)
+            return True
+
+        if self.changed_cache(hash_info):
+            logger.debug(
+                "cache for '%s'('%s') has changed.", path_info, hash_info
+            )
+            return True
+
+        actual = self.tree.get_hash(path_info)
+        if hash_info != actual:
+            logger.debug(
+                "hash value '%s' for '%s' has changed (actual '%s').",
+                hash_info,
+                actual,
+                path_info,
+            )
+            return True
+
+        logger.debug("'%s' hasn't changed.", path_info)
+        return False
+
+    def link(self, from_info, to_info):
+        self._link(from_info, to_info, self.cache_types)
+
+    def _link(self, from_info, to_info, link_types):
+        assert self.tree.isfile(from_info)
+
+        self.tree.makedirs(to_info.parent)
+
+        self._try_links(from_info, to_info, link_types)
+
+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self.tree, f"is_{link_type}", None)
+        if is_link and not is_link(path_info):
+            self.tree.remove(path_info)
+            raise DvcException(f"failed to verify {link_type}")
+
+        self.cache_type_confirmed = True
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        while link_types:
+            link_method = getattr(self.tree, link_types[0])
+            try:
+                self._do_link(from_info, to_info, link_method)
+                self._verify_link(to_info, link_types[0])
+                return
+
+            except DvcException as exc:
+                logger.debug(
+                    "Cache type '%s' is not supported: %s", link_types[0], exc
+                )
+                del link_types[0]
+
+        raise CacheLinkError([to_info])
+
+    def _do_link(self, from_info, to_info, link_method):
+        if self.tree.exists(to_info):
+            raise DvcException(f"Link '{to_info}' already exists!")
+
+        link_method(from_info, to_info)
+
+        logger.debug(
+            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
+        )
+
+    def _save_file(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        assert hash_info
+
+        cache_info = self.tree.hash_to_path_info(hash_info.value)
+        if tree == self.tree:
+            if self.changed_cache(hash_info):
+                self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
+                self.link(cache_info, path_info)
+            elif self.tree.iscopy(path_info) and self._cache_is_copy(
+                path_info
+            ):
+                # Default relink procedure involves unneeded copy
+                self.tree.unprotect(path_info)
+            else:
+                self.tree.remove(path_info)
+                self.link(cache_info, path_info)
+
+            if save_link:
+                self.tree.state.save_link(path_info)
+            # we need to update path and cache, since in case of reflink,
+            # or copy cache type moving original file results in updates on
+            # next executed command, which causes md5 recalculation
+            self.tree.state.save(path_info, hash_info)
+        else:
+            if self.changed_cache(hash_info):
+                with tree.open(path_info, mode="rb") as fobj:
+                    # if tree has fetch enabled, DVC out will be fetched on
+                    # open and we do not need to read/copy any data
+                    if not (
+                        tree.isdvc(path_info, strict=False) and tree.fetch
+                    ):
+                        self.tree.copy_fobj(fobj, cache_info)
+                callback = kwargs.get("download_callback")
+                if callback:
+                    callback(1)
+
+        self.tree.state.save(cache_info, hash_info)
+
+    def _cache_is_copy(self, path_info):
+        """Checks whether cache uses copies."""
+        if self.cache_type_confirmed:
+            return self.cache_types[0] == "copy"
+
+        if set(self.cache_types) <= {"copy"}:
+            return True
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.tree.path_info / ".cache_type_test_file"
+        if not self.tree.exists(test_cache_file):
+            with self.tree.open(test_cache_file, "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.tree.remove(workspace_file)
+            self.tree.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0] == "copy"
+
+    def _get_dir_info_hash(self, dir_info):
+        import tempfile
+
+        from dvc.path_info import PathInfo
+        from dvc.utils import tmp_fname
+
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        with open(tmp, "w+") as fobj:
+            json.dump(dir_info.to_list(), fobj, sort_keys=True)
+
+        from_info = PathInfo(tmp)
+        to_info = self.tree.path_info / tmp_fname("")
+        self.tree.upload(from_info, to_info, no_progress_bar=True)
+
+        hash_info = self.tree.get_file_hash(to_info)
+        hash_info.value += self.tree.CHECKSUM_DIR_SUFFIX
+        hash_info.dir_info = dir_info
+        hash_info.nfiles = dir_info.nfiles
+
+        return hash_info, to_info
+
+    @use_state
+    def save_dir_info(self, dir_info, hash_info=None):
+        if (
+            hash_info
+            and hash_info.name == self.tree.PARAM_CHECKSUM
+            and not self.changed_cache_file(hash_info)
+        ):
+            return hash_info
+
+        hi, tmp_info = self._get_dir_info_hash(dir_info)
+        new_info = self.tree.hash_to_path_info(hi.value)
+        if self.changed_cache_file(hi):
+            self.tree.makedirs(new_info.parent)
+            self.tree.move(tmp_info, new_info, mode=self.CACHE_MODE)
+
+        self.tree.state.save(new_info, hi)
+
+        return hi
+
+    def _save_dir(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if not hash_info.dir_info:
+            hash_info.dir_info = tree.cache.get_dir_cache(hash_info)
+        hi = self.save_dir_info(hash_info.dir_info, hash_info)
+        for entry_info, entry_hash in Tqdm(
+            hi.dir_info.items(path_info),
+            desc="Saving " + path_info.name,
+            unit="file",
+        ):
+            self._save_file(
+                entry_info, tree, entry_hash, save_link=False, **kwargs
+            )
+
+        if save_link:
+            self.tree.state.save_link(path_info)
+        if self.tree.exists(path_info):
+            self.tree.state.save(path_info, hi)
+
+        cache_info = self.tree.hash_to_path_info(hi.value)
+        self.tree.state.save(cache_info, hi)
+
+    @use_state
+    def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if path_info.scheme != self.tree.scheme:
+            raise RemoteActionNotImplemented(
+                f"save {path_info.scheme} -> {self.tree.scheme}",
+                self.tree.scheme,
+            )
+
+        self._save(path_info, tree, hash_info, save_link, **kwargs)
+
+    def _save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        to_info = self.tree.hash_to_path_info(hash_info.value)
+        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
+
+        if tree.isdir(path_info):
+            self._save_dir(path_info, tree, hash_info, save_link, **kwargs)
+        else:
+            self._save_file(path_info, tree, hash_info, save_link, **kwargs)
+
+    # Override to return path as a string instead of PathInfo for clouds
+    # which support string paths (see local)
+    def hash_to_path(self, hash_):
+        return self.tree.hash_to_path_info(hash_)
+
+    def changed_cache_file(self, hash_info):
+        """Compare the given hash with the (corresponding) actual one.
+
+        - Use `State` as a cache for computed hashes
+            + The entries are invalidated by taking into account the following:
+                * mtime
+                * inode
+                * size
+                * hash
+
+        - Remove the file from cache if it doesn't match the actual hash
+        """
+        # Prefer string path over PathInfo when possible due to performance
+        cache_info = self.hash_to_path(hash_info.value)
+        if self.tree.is_protected(cache_info):
+            logger.debug(
+                "Assuming '%s' is unchanged since it is read-only", cache_info
+            )
+            return False
+
+        actual = self.tree.get_hash(cache_info)
+
+        logger.debug(
+            "cache '%s' expected '%s' actual '%s'",
+            cache_info,
+            hash_info,
+            actual,
+        )
+
+        if not hash_info or not actual:
+            return True
+
+        if actual.value.split(".")[0] == hash_info.value.split(".")[0]:
+            # making cache file read-only so we don't need to check it
+            # next time
+            self.tree.protect(cache_info)
+            return False
+
+        if self.tree.exists(cache_info):
+            logger.warning("corrupted cache file '%s'.", cache_info)
+            self.tree.remove(cache_info)
+
+        return True
+
+    def _changed_dir_cache(self, hash_info, path_info=None, filter_info=None):
+        if self.changed_cache_file(hash_info):
+            return True
+
+        dir_info = self.get_dir_cache(hash_info)
+        for entry_info, entry_hash in dir_info.items(path_info):
+            if path_info and filter_info:
+                if not entry_info.isin_or_eq(filter_info):
+                    continue
+
+            if self.changed_cache_file(entry_hash):
+                return True
+
+        return False
+
+    def changed_cache(self, hash_info, path_info=None, filter_info=None):
+        if hash_info.isdir:
+            return self._changed_dir_cache(
+                hash_info, path_info=path_info, filter_info=filter_info
+            )
+        return self.changed_cache_file(hash_info)
+
+    def already_cached(self, path_info):
+        current = self.tree.get_hash(path_info)
+
+        if not current:
+            return False
+
+        return not self.changed_cache(current)
+
+    def safe_remove(self, path_info, force=False):
+        if not self.tree.exists(path_info):
+            return
+
+        if not force and not self.already_cached(path_info):
+            msg = (
+                "file '{}' is going to be removed."
+                " Are you sure you want to proceed?".format(str(path_info))
+            )
+
+            if not prompt.confirm(msg):
+                raise ConfirmRemoveError(str(path_info))
+
+        self.tree.remove(path_info)
+
+    def _checkout_file(
+        self, path_info, hash_info, force, progress_callback=None, relink=False
+    ):
+        """The file is changed we need to checkout a new copy"""
+        added, modified = True, False
+        cache_info = self.tree.hash_to_path_info(hash_info.value)
+        if self.tree.exists(path_info):
+            logger.debug("data '%s' will be replaced.", path_info)
+            self.safe_remove(path_info, force=force)
+            added, modified = False, True
+
+        self.link(cache_info, path_info)
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_info)
+        if progress_callback:
+            progress_callback(str(path_info))
+
+        return added, modified and not relink
+
+    def _checkout_dir(
+        self,
+        path_info,
+        hash_info,
+        force,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        added, modified = False, False
+        # Create dir separately so that dir is created
+        # even if there are no files in it
+        if not self.tree.exists(path_info):
+            added = True
+            self.tree.makedirs(path_info)
+
+        dir_info = self.get_dir_cache(hash_info)
+
+        logger.debug("Linking directory '%s'.", path_info)
+
+        for entry_info, entry_hash_info in dir_info.items(path_info):
+            entry_cache_info = self.tree.hash_to_path_info(
+                entry_hash_info.value
+            )
+
+            if filter_info and not entry_info.isin_or_eq(filter_info):
+                continue
+
+            if relink or self.changed(entry_info, entry_hash_info):
+                modified = True
+                self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.tree.state.save(entry_info, entry_hash_info)
+            if progress_callback:
+                progress_callback(str(entry_info))
+
+        modified = (
+            self._remove_redundant_files(path_info, dir_info, force)
+            or modified
+        )
+
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_info)
+
+        # relink is not modified, assume it as nochange
+        return added, not added and modified and not relink
+
+    def _remove_redundant_files(self, path_info, dir_info, force):
+        existing_files = set(self.tree.walk_files(path_info))
+
+        needed_files = {info for info, _ in dir_info.items(path_info)}
+        redundant_files = existing_files - needed_files
+        for path in redundant_files:
+            self.safe_remove(path, force)
+
+        return bool(redundant_files)
+
+    @use_state
+    def checkout(
+        self,
+        path_info,
+        hash_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+        quiet=False,
+    ):
+        if path_info.scheme not in ["local", self.tree.scheme]:
+            raise NotImplementedError
+
+        failed = None
+        skip = False
+        if not hash_info:
+            if not quiet:
+                logger.warning(
+                    "No file hash info found for '%s'. It won't be created.",
+                    path_info,
+                )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        elif not relink and not self.changed(path_info, hash_info):
+            logger.debug("Data '%s' didn't change.", path_info)
+            skip = True
+
+        elif self.changed_cache(
+            hash_info, path_info=path_info, filter_info=filter_info
+        ):
+            if not quiet:
+                logger.warning(
+                    "Cache '%s' not found. File '%s' won't be created.",
+                    hash_info,
+                    path_info,
+                )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        if failed or skip:
+            if progress_callback:
+                progress_callback(
+                    str(path_info),
+                    self.get_files_number(
+                        self.tree.path_info, hash_info, filter_info
+                    ),
+                )
+            if failed:
+                raise CheckoutError([failed])
+            return
+
+        logger.debug(
+            "Checking out '%s' with cache '%s'.", path_info, hash_info
+        )
+
+        return self._checkout(
+            path_info,
+            hash_info,
+            force,
+            progress_callback,
+            relink,
+            filter_info,
+        )
+
+    def _checkout(
+        self,
+        path_info,
+        hash_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        if not hash_info.isdir:
+            return self._checkout_file(
+                path_info, hash_info, force, progress_callback, relink
+            )
+
+        return self._checkout_dir(
+            path_info, hash_info, force, progress_callback, relink, filter_info
+        )
+
+    def get_files_number(self, path_info, hash_info, filter_info):
+        from funcy.py3 import ilen
+
+        if not hash_info:
+            return 0
+
+        if not hash_info.isdir:
+            return 1
+
+        if not filter_info:
+            return self.get_dir_cache(hash_info).nfiles
+
+        return ilen(
+            filter_info.isin_or_eq(path_info / relpath)
+            for relpath, _ in self.get_dir_cache(hash_info).items()
+        )
+
+    def _get_dir_size(self, dir_info):
+        try:
+            return sum(
+                self.tree.getsize(self.tree.hash_to_path_info(hi.value))
+                for _, hi in dir_info.items()
+            )
+        except FileNotFoundError:
+            return None
+
+    def merge(self, ancestor_info, our_info, their_info):
+        assert our_info
+        assert their_info
+
+        if ancestor_info:
+            ancestor = self.get_dir_cache(ancestor_info)
+        else:
+            ancestor = DirInfo()
+
+        our = self.get_dir_cache(our_info)
+        their = self.get_dir_cache(their_info)
+
+        merged = our.merge(ancestor, their)
+        hash_info = self.save_dir_info(merged)
+        hash_info.size = self._get_dir_size(merged)
+        return hash_info
+
+    @use_state
+    def get_hash(self, tree, path_info):
+        hash_info = tree.get_hash(path_info)
+        if not hash_info.isdir:
+            assert hash_info.name == self.tree.PARAM_CHECKSUM
+            return hash_info
+
+        hi = self.save_dir_info(hash_info.dir_info, hash_info)
+        hi.size = hash_info.size
+        return hi
+
+    def set_dir_info(self, hash_info):
+        assert hash_info.isdir
+
+        hash_info.dir_info = self.get_dir_cache(hash_info)
+        hash_info.nfiles = hash_info.dir_info.nfiles
+
+    def _list_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            if len(prefix) > 2:
+                path_info = self.tree.path_info / prefix[:2] / prefix[2:]
+            else:
+                path_info = self.tree.path_info / prefix[:2]
+            prefix = True
+        else:
+            path_info = self.tree.path_info
+            prefix = False
+        if progress_callback:
+            for file_info in self.tree.walk_files(path_info, prefix=prefix):
+                progress_callback()
+                yield file_info.path
+        else:
+            yield from self.tree.walk_files(path_info, prefix=prefix)
+
+    def _path_to_hash(self, path):
+        parts = self.tree.PATH_CLS(path).parts[-2:]
+
+        if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
+            raise ValueError(f"Bad cache file path '{path}'")
+
+        return "".join(parts)
+
+    def list_hashes(self, prefix=None, progress_callback=None):
+        """Iterate over hashes in this tree.
+
+        If `prefix` is specified, only hashes which begin with `prefix`
+        will be returned.
+        """
+        for path in self._list_paths(prefix, progress_callback):
+            try:
+                yield self._path_to_hash(path)
+            except ValueError:
+                logger.debug(
+                    "'%s' doesn't look like a cache file, skipping", path
+                )
+
+    def _hashes_with_limit(self, limit, prefix=None, progress_callback=None):
+        count = 0
+        for hash_ in self.list_hashes(prefix, progress_callback):
+            yield hash_
+            count += 1
+            if count > limit:
+                logger.debug(
+                    "`list_hashes()` returned max '{}' hashes, "
+                    "skipping remaining results".format(limit)
+                )
+                return
+
+    def _max_estimation_size(self, hashes):
+        # Max remote size allowed for us to use traverse method
+        return max(
+            self.tree.TRAVERSE_THRESHOLD_SIZE,
+            len(hashes)
+            / self.tree.TRAVERSE_WEIGHT_MULTIPLIER
+            * self.tree.LIST_OBJECT_PAGE_SIZE,
+        )
+
+    def _estimate_remote_size(self, hashes=None, name=None):
+        """Estimate tree size based on number of entries beginning with
+        "00..." prefix.
+        """
+        prefix = "0" * self.tree.TRAVERSE_PREFIX_LEN
+        total_prefixes = pow(16, self.tree.TRAVERSE_PREFIX_LEN)
+        if hashes:
+            max_hashes = self._max_estimation_size(hashes)
+        else:
+            max_hashes = None
+
+        with Tqdm(
+            desc="Estimating size of "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            unit="file",
+        ) as pbar:
+
+            def update(n=1):
+                pbar.update(n * total_prefixes)
+
+            if max_hashes:
+                hashes = self._hashes_with_limit(
+                    max_hashes / total_prefixes, prefix, update
+                )
+            else:
+                hashes = self.list_hashes(prefix, update)
+
+            remote_hashes = set(hashes)
+            if remote_hashes:
+                remote_size = total_prefixes * len(remote_hashes)
+            else:
+                remote_size = total_prefixes
+            logger.debug(f"Estimated remote size: {remote_size} files")
+        return remote_size, remote_hashes
+
+    def list_hashes_traverse(
+        self, remote_size, remote_hashes, jobs=None, name=None
+    ):
+        """Iterate over all hashes found in this tree.
+        Hashes are fetched in parallel according to prefix, except in
+        cases where the remote size is very small.
+
+        All hashes from the remote (including any from the size
+        estimation step passed via the `remote_hashes` argument) will be
+        returned.
+
+        NOTE: For large remotes the list of hashes will be very
+        big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
+        and we don't really need all of it at the same time, so it makes
+        sense to use a generator to gradually iterate over it, without
+        keeping all of it in memory.
+        """
+        num_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
+        if num_pages < 256 / self.tree.JOBS:
+            # Fetching prefixes in parallel requires at least 255 more
+            # requests, for small enough remotes it will be faster to fetch
+            # entire cache without splitting it into prefixes.
+            #
+            # NOTE: this ends up re-fetching hashes that were already
+            # fetched during remote size estimation
+            traverse_prefixes = [None]
+            initial = 0
+        else:
+            yield from remote_hashes
+            initial = len(remote_hashes)
+            traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
+            if self.tree.TRAVERSE_PREFIX_LEN > 2:
+                traverse_prefixes += [
+                    "{0:0{1}x}".format(i, self.tree.TRAVERSE_PREFIX_LEN)
+                    for i in range(
+                        1, pow(16, self.tree.TRAVERSE_PREFIX_LEN - 2)
+                    )
+                ]
+        with Tqdm(
+            desc="Querying "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            total=remote_size,
+            initial=initial,
+            unit="file",
+        ) as pbar:
+
+            def list_with_update(prefix):
+                return list(
+                    self.list_hashes(
+                        prefix=prefix, progress_callback=pbar.update
+                    )
+                )
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                in_remote = executor.map(list_with_update, traverse_prefixes,)
+                yield from itertools.chain.from_iterable(in_remote)
+
+    def all(self, jobs=None, name=None):
+        """Iterate over all hashes in this tree.
+
+        Hashes will be fetched in parallel threads according to prefix
+        (except for small remotes) and a progress bar will be displayed.
+        """
+        logger.debug(
+            "Fetching all hashes from '{}'".format(
+                name if name else "remote cache"
+            )
+        )
+
+        if not self.tree.CAN_TRAVERSE:
+            return self.list_hashes()
+
+        remote_size, remote_hashes = self._estimate_remote_size(name=name)
+        return self.list_hashes_traverse(
+            remote_size, remote_hashes, jobs, name
+        )
+
+    def _remove_unpacked_dir(self, hash_):
+        pass
+
+    def gc(self, used, jobs=None):
+        removed = False
+        # hashes must be sorted to ensure we always remove .dir files first
+        for hash_ in sorted(
+            self.all(jobs, str(self.tree.path_info)),
+            key=self.tree.is_dir_hash,
+            reverse=True,
+        ):
+            if hash_ in used:
+                continue
+            path_info = self.tree.hash_to_path_info(hash_)
+            if self.tree.is_dir_hash(hash_):
+                # backward compatibility
+                # pylint: disable=protected-access
+                self._remove_unpacked_dir(hash_)
+            self.tree.remove(path_info)
+            removed = True
+
+        return removed
+
+    def list_hashes_exists(self, hashes, jobs=None, name=None):
+        """Return list of the specified hashes which exist in this tree.
+        Hashes will be queried individually.
+        """
+        logger.debug(
+            "Querying {} hashes via object_exists".format(len(hashes))
+        )
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(hashes),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(path_info):
+                ret = self.tree.exists(path_info)
+                pbar.update_msg(str(path_info))
+                return ret
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                path_infos = map(self.tree.hash_to_path_info, hashes)
+                in_remote = executor.map(exists_with_progress, path_infos)
+                ret = list(itertools.compress(hashes, in_remote))
+                return ret
+
+    def hashes_exist(self, hashes, jobs=None, name=None):
+        """Check if the given hashes are stored in the remote.
+
+        There are two ways of performing this check:
+
+        - Traverse method: Get a list of all the files in the remote
+            (traversing the cache directory) and compare it with
+            the given hashes. Cache entries will be retrieved in parallel
+            threads according to prefix (i.e. entries starting with, "00...",
+            "01...", and so on) and a progress bar will be displayed.
+
+        - Exists method: For each given hash, run the `exists`
+            method and filter the hashes that aren't on the remote.
+            This is done in parallel threads.
+            It also shows a progress bar when performing the check.
+
+        The reason for such an odd logic is that most of the remotes
+        take much shorter time to just retrieve everything they have under
+        a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
+        check if particular file exists much quicker, use their own
+        implementation of hashes_exist (see ssh, local).
+
+        Which method to use will be automatically determined after estimating
+        the size of the remote cache, and comparing the estimated size with
+        len(hashes). To estimate the size of the remote cache, we fetch
+        a small subset of cache entries (i.e. entries starting with "00...").
+        Based on the number of entries in that subset, the size of the full
+        cache can be estimated, since the cache is evenly distributed according
+        to hash.
+
+        Returns:
+            A list with hashes that were found in the remote
+        """
+        # Remotes which do not use traverse prefix should override
+        # hashes_exist() (see ssh, local)
+        assert self.tree.TRAVERSE_PREFIX_LEN >= 2
+
+        hashes = set(hashes)
+        if len(hashes) == 1 or not self.tree.CAN_TRAVERSE:
+            remote_hashes = self.list_hashes_exists(hashes, jobs, name)
+            return remote_hashes
+
+        # Max remote size allowed for us to use traverse method
+        remote_size, remote_hashes = self._estimate_remote_size(hashes, name)
+
+        traverse_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
+        # For sufficiently large remotes, traverse must be weighted to account
+        # for performance overhead from large lists/sets.
+        # From testing with S3, for remotes with 1M+ files, object_exists is
+        # faster until len(hashes) is at least 10k~100k
+        if remote_size > self.tree.TRAVERSE_THRESHOLD_SIZE:
+            traverse_weight = (
+                traverse_pages * self.tree.TRAVERSE_WEIGHT_MULTIPLIER
+            )
+        else:
+            traverse_weight = traverse_pages
+        if len(hashes) < traverse_weight:
+            logger.debug(
+                "Large remote ('{}' hashes < '{}' traverse weight), "
+                "using object_exists for remaining hashes".format(
+                    len(hashes), traverse_weight
+                )
+            )
+            return list(hashes & remote_hashes) + self.list_hashes_exists(
+                hashes - remote_hashes, jobs, name
+            )
+
+        logger.debug("Querying '{}' hashes via traverse".format(len(hashes)))
+        remote_hashes = set(
+            self.list_hashes_traverse(remote_size, remote_hashes, jobs, name)
+        )
+        return list(hashes & set(remote_hashes))

dvc/cache/local.py

@@ -0,0 +1,91 @@
+import logging
+import os
+
+from funcy import cached_property
+
+from dvc.hash_info import HashInfo
+from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+
+from ..tree.local import LocalTree
+from ..utils.fs import walk_files
+from .base import CloudCache
+
+logger = logging.getLogger(__name__)
+
+
+class LocalCache(CloudCache):
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+    CACHE_MODE = LocalTree.CACHE_MODE
+    UNPACKED_DIR_SUFFIX = ".unpacked"
+
+    def __init__(self, tree):
+        super().__init__(tree)
+        self.cache_dir = tree.config.get("url")
+
+    @property
+    def cache_dir(self):
+        return self.tree.path_info.fspath if self.tree.path_info else None
+
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.tree.path_info = PathInfo(value) if value else None
+
+    @cached_property
+    def cache_path(self):
+        return os.path.abspath(self.cache_dir)
+
+    def hash_to_path(self, hash_):
+        # NOTE: `self.cache_path` is already normalized so we can simply use
+        # `os.sep` instead of `os.path.join`. This results in this helper
+        # being ~5.5 times faster.
+        return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
+
+    def hashes_exist(
+        self, hashes, jobs=None, name=None
+    ):  # pylint: disable=unused-argument
+        return [
+            hash_
+            for hash_ in Tqdm(
+                hashes,
+                unit="file",
+                desc="Querying "
+                + ("cache in " + name if name else "local cache"),
+            )
+            if not self.changed_cache_file(
+                HashInfo(self.tree.PARAM_CHECKSUM, hash_)
+            )
+        ]
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        return super().already_cached(path_info)
+
+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
+    def _list_paths(self, prefix=None, progress_callback=None):
+        assert self.tree.path_info is not None
+        if prefix:
+            path_info = self.tree.path_info / prefix[:2]
+            if not self.tree.exists(path_info):
+                return
+        else:
+            path_info = self.tree.path_info
+        # NOTE: use utils.fs walk_files since tree.walk_files will not follow
+        # symlinks
+        if progress_callback:
+            for path in walk_files(path_info):
+                progress_callback()
+                yield path
+        else:
+            yield from walk_files(path_info)
+
+    def _remove_unpacked_dir(self, hash_):
+        info = self.tree.hash_to_path_info(hash_)
+        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
+        self.tree.remove(path_info)

dvc/cache/ssh.py

@@ -0,0 +1,90 @@
+import errno
+import itertools
+import logging
+import posixpath
+from concurrent.futures import ThreadPoolExecutor
+
+from dvc.progress import Tqdm
+from dvc.utils import to_chunks
+
+from .base import CloudCache
+
+logger = logging.getLogger(__name__)
+
+
+class SSHCache(CloudCache):
+    def batch_exists(self, path_infos, callback):
+        def _exists(chunk_and_channel):
+            chunk, channel = chunk_and_channel
+            ret = []
+            for path in chunk:
+                try:
+                    channel.stat(path)
+                    ret.append(True)
+                except OSError as exc:
+                    if exc.errno != errno.ENOENT:
+                        raise
+                    ret.append(False)
+                callback(path)
+            return ret
+
+        with self.tree.ssh(path_infos[0]) as ssh:
+            channels = ssh.open_max_sftp_channels()
+            max_workers = len(channels)
+
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                paths = [path_info.path for path_info in path_infos]
+                chunks = to_chunks(paths, num_chunks=max_workers)
+                chunks_and_channels = zip(chunks, channels)
+                outcome = executor.map(_exists, chunks_and_channels)
+                results = list(itertools.chain.from_iterable(outcome))
+
+            return results
+
+    def hashes_exist(self, hashes, jobs=None, name=None):
+        """This is older implementation used in remote/base.py
+        We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
+        faster than current approach (relying on exists(path_info)) applied in
+        remote/base.
+        """
+        if not self.tree.CAN_TRAVERSE:
+            return list(set(hashes) & set(self.all()))
+
+        # possibly prompt for credentials before "Querying" progress output
+        self.tree.ensure_credentials()
+
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(hashes),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(chunks):
+                return self.batch_exists(chunks, callback=pbar.update_msg)
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                path_infos = [self.tree.hash_to_path_info(x) for x in hashes]
+                chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
+                results = executor.map(exists_with_progress, chunks)
+                in_remote = itertools.chain.from_iterable(results)
+                ret = list(itertools.compress(hashes, in_remote))
+                return ret
+
+    def _list_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            root = posixpath.join(self.tree.path_info.path, prefix[:2])
+        else:
+            root = self.tree.path_info.path
+        with self.tree.ssh(self.tree.path_info) as ssh:
+            if prefix and not ssh.exists(root):
+                return
+            # If we simply return an iterator then with above closes instantly
+            if progress_callback:
+                for path in ssh.walk_files(root):
+                    progress_callback()
+                    yield path
+            else:
+                yield from ssh.walk_files(root)

dvc/cachemgr.py

@@ -1,152 +0,0 @@
-import os
-from typing import TYPE_CHECKING, Optional
-
-from dvc.fs import GitFileSystem, Schemes
-from dvc_data.hashfile.db import get_odb
-from dvc_data.hashfile.hash import DEFAULT_ALGORITHM
-
-if TYPE_CHECKING:
-    from dvc.repo import Repo
-
-LEGACY_HASH_NAMES = {"md5-dos2unix", "params"}
-
-
-def _get_odb(
-    repo,
-    settings,
-    fs=None,
-    prefix: Optional[tuple[str, ...]] = None,
-    hash_name: Optional[str] = None,
-    **kwargs,
-):
-    from dvc.fs import get_cloud_fs
-
-    if not settings:
-        return None
-
-    cls, config, fs_path = get_cloud_fs(repo.config, **settings)
-    fs = fs or cls(**config)
-    if prefix:
-        fs_path = fs.join(fs_path, *prefix)
-    if hash_name:
-        config["hash_name"] = hash_name
-    return get_odb(fs, fs_path, state=repo.state, **config)
-
-
-class CacheManager:
-    CACHE_DIR = "cache"
-    FILES_DIR = "files"
-    FS_DIR = "fs"
-
-    def __init__(self, repo):
-        self._repo = repo
-        self.config = config = repo.config["cache"]
-        self._odb = {}
-
-        local = config.get("local")
-        default = self.default_local_cache_dir
-
-        if local:
-            settings = {"name": local}
-        elif "dir" not in config and not default:
-            settings = None
-        else:
-            from dvc.config_schema import LOCAL_COMMON
-
-            url = config.get("dir") or default
-            settings = {"url": url}
-            for opt in LOCAL_COMMON:
-                if opt in config:
-                    settings[str(opt)] = config.get(opt)
-
-        kwargs = {}
-        if not isinstance(repo.fs, GitFileSystem):
-            kwargs["fs"] = repo.fs
-
-        odb = _get_odb(
-            repo,
-            settings,
-            prefix=(self.FILES_DIR, DEFAULT_ALGORITHM),
-            **kwargs,
-        )
-        self._odb["repo"] = odb
-        self._odb[Schemes.LOCAL] = odb
-        legacy_odb = _get_odb(repo, settings, hash_name="md5-dos2unix", **kwargs)
-        self._odb["legacy"] = legacy_odb
-
-    @property
-    def fs_cache(self):
-        """Filesystem-based cache.
-
-        Currently used as a temporary location to download files that we don't
-        yet have a regular oid (e.g. md5) for.
-        """
-        from dvc_data.index import FileStorage
-
-        return FileStorage(
-            key=(),
-            fs=self.local.fs,
-            path=self.local.fs.join(self.local_cache_dir, self.FS_DIR),
-        )
-
-    def _init_odb(self, schemes):
-        for scheme in schemes:
-            remote = self.config.get(scheme)
-            settings = {"name": remote} if remote else None
-            self._odb[scheme] = _get_odb(
-                self._repo,
-                settings,
-                prefix=(self.FILES_DIR, DEFAULT_ALGORITHM),
-            )
-
-    def __getattr__(self, name):
-        try:
-            return self._odb[name]
-        except KeyError as exc:
-            raise AttributeError from exc
-
-    def by_scheme(self):
-        yield from self._odb.items()
-
-    @property
-    def local_cache_dir(self) -> str:
-        """Return base local cache directory without any prefixes.
-
-        (i.e. `dvc cache dir`).
-        """
-        return self.legacy.path
-
-    @property
-    def default_local_cache_dir(self) -> Optional[str]:
-        repo = self._repo
-        if repo and repo.local_dvc_dir:
-            return os.path.join(repo.local_dvc_dir, self.CACHE_DIR)
-        return None
-
-
-def migrate_2_to_3(repo: "Repo", dry: bool = False):
-    """Migrate legacy 2.x objects to 3.x cache.
-
-    Legacy 'md5-dos2unix' objects will be re-hashed with 'md5', added to 3.x cache,
-    and then a link from the legacy 2.x location to the 3.x location will be created.
-    """
-    from dvc.fs.callbacks import TqdmCallback
-    from dvc.ui import ui
-    from dvc_data.hashfile.db.migrate import migrate, prepare
-
-    src = repo.cache.legacy
-    dest = repo.cache.local
-    if dry:
-        oids = list(src._list_oids())
-        ui.write(
-            f"{len(oids)} files will be re-hashed and migrated to the DVC 3.0 cache "
-            "location."
-        )
-        return
-
-    with TqdmCallback(desc="Computing DVC 3.0 hashes", unit="files") as cb:
-        migration = prepare(src, dest, callback=cb)
-
-    with TqdmCallback(desc="Migrating to DVC 3.0 cache", unit="files") as cb:
-        count = migrate(migration, callback=cb)
-    ui.write(f"Migrated {count} files to DVC 3.0 cache location.")

dvc/cli.py

@@ -1,13 +1,11 @@
-"""Main parser for the dvc cli."""
-
+"""DVC command line interface"""
 import argparse
+import logging
 import os
-from functools import lru_cache
+import sys
 
-from dvc import __version__
-from dvc.commands import (
+from .command import (
     add,
-    artifacts,
     cache,
     check_ignore,
     checkout,
@@ -16,12 +14,9 @@ from dvc.commands import (
     config,
     daemon,
     dag,
-    data,
     data_sync,
-    dataset,
     destroy,
     diff,
-    du,
     experiments,
     freeze,
     gc,
@@ -29,87 +24,74 @@ from dvc.commands import (
     get_url,
     git_hook,
     imp,
-    imp_db,
     imp_url,
     init,
     install,
     ls,
-    ls_url,
     metrics,
     move,
     params,
     plots,
-    queue,
     remote,
     remove,
     repro,
     root,
-    stage,
-    studio,
+    run,
     unprotect,
     update,
     version,
 )
-from dvc.log import logger
+from .command.base import fix_subparsers
+from .exceptions import DvcParserError
 
-from . import DvcParserError, formatter
-
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 COMMANDS = [
-    add,
-    artifacts,
-    cache,
-    check_ignore,
-    checkout,
-    commit,
-    completion,
-    config,
-    daemon,
-    dag,
-    data,
-    data_sync,
-    dataset,
-    destroy,
-    diff,
-    du,
-    experiments,
-    freeze,
-    gc,
+    init,
     get,
     get_url,
-    git_hook,
-    imp,
-    imp_db,
-    imp_url,
-    init,
-    install,
-    ls,
-    ls_url,
-    metrics,
-    move,
-    params,
-    plots,
-    queue,
-    remote,
+    destroy,
+    add,
     remove,
-    repro,
-    root,
-    stage,
-    studio,
+    move,
     unprotect,
-    update,
+    run,
+    repro,
+    data_sync,
+    gc,
+    imp,
+    imp_url,
+    config,
+    checkout,
+    remote,
+    cache,
+    metrics,
+    params,
+    install,
+    root,
+    ls,
+    freeze,
+    dag,
+    daemon,
+    commit,
+    completion,
+    diff,
     version,
+    update,
+    git_hook,
+    plots,
+    experiments,
+    check_ignore,
 ]
 
 
 def _find_parser(parser, cmd_cls):
-    defaults = parser._defaults
+    defaults = parser._defaults  # pylint: disable=protected-access
     if not cmd_cls or cmd_cls == defaults.get("func"):
         parser.print_help()
-        raise DvcParserError
+        raise DvcParserError()
 
-    actions = parser._actions
+    actions = parser._actions  # pylint: disable=protected-access
     for action in actions:
         if not isinstance(action.choices, dict):
             # NOTE: we are only interested in subparsers
@@ -121,7 +103,7 @@ def _find_parser(parser, cmd_cls):
 class DvcParser(argparse.ArgumentParser):
     """Custom parser class for dvc CLI."""
 
-    def error(self, message, cmd_cls=None):
+    def error(self, message, cmd_cls=None):  # pylint: disable=arguments-differ
         logger.error(message)
         _find_parser(self, cmd_cls)
 
@@ -136,6 +118,17 @@ class DvcParser(argparse.ArgumentParser):
         return args
 
 
+class VersionAction(argparse.Action):  # pragma: no cover
+    # pylint: disable=too-few-public-methods
+    """Shows DVC version and exits."""
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        from dvc import __version__
+
+        print(__version__)
+        sys.exit(0)
+
+
 def get_parent_parser():
     """Create instances of a parser containing common arguments shared among
     all the commands.
@@ -143,9 +136,20 @@ def get_parent_parser():
     When overwriting `-q` or `-v`, you need to instantiate a new object
     in order to prevent some weird behavior.
     """
-    from dvc._debug import add_debugging_flags
-
     parent_parser = argparse.ArgumentParser(add_help=False)
+
+    parent_parser.add_argument(
+        "--cprofile",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parent_parser.add_argument("--cprofile-dump", help=argparse.SUPPRESS)
+
+    parent_parser.add_argument(
+        "--pdb", action="store_true", default=False, help=argparse.SUPPRESS,
+    )
+
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
         "-q", "--quiet", action="count", default=0, help="Be quiet."
@@ -153,12 +157,10 @@ def get_parent_parser():
     log_level_group.add_argument(
         "-v", "--verbose", action="count", default=0, help="Be verbose."
     )
-    add_debugging_flags(parent_parser)
 
     return parent_parser
 
 
-@lru_cache(maxsize=1)
 def get_main_parser():
     parent_parser = get_parent_parser()
 
@@ -168,7 +170,7 @@ def get_main_parser():
         prog="dvc",
         description=desc,
         parents=[parent_parser],
-        formatter_class=formatter.RawTextHelpFormatter,
+        formatter_class=argparse.RawTextHelpFormatter,
         add_help=False,
     )
 
@@ -184,11 +186,13 @@ def get_main_parser():
         help="Show this help message and exit.",
     )
 
+    # NOTE: On some python versions action='version' prints to stderr
+    # instead of stdout https://bugs.python.org/issue18920
     parser.add_argument(
         "-V",
         "--version",
-        action="version",
-        version=__version__,
+        action=VersionAction,
+        nargs=0,
         help="Show program's version.",
     )
 
@@ -200,24 +204,31 @@ def get_main_parser():
         type=str,
     )
 
-    parser.add_argument(
-        "--wait-for-lock",
-        action="store_true",
-        default=False,
-        help="Wait for the lock if it is already held by another process, instead of"
-        " failing immediately.",
-    )
-
     # Sub commands
     subparsers = parser.add_subparsers(
         title="Available Commands",
-        metavar="command",
+        metavar="COMMAND",
         dest="cmd",
-        help="Use `dvc command --help` for command-specific help.",
-        required=True,
+        help="Use `dvc COMMAND --help` for command-specific help.",
     )
 
+    fix_subparsers(subparsers)
+
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 
     return parser
+
+
+def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        dvc.exceptions.DvcParserError: raised for argument parsing errors.
+    """
+    parser = get_main_parser()
+    args = parser.parse_args(argv)
+    return args

dvc/cli/__init__.py

@@ -1,257 +0,0 @@
-"""This module provides an entrypoint to the dvc cli and parsing utils."""
-
-import logging
-import sys
-from typing import Optional
-
-from dvc.log import logger
-
-# Workaround for CPython bug. See [1] and [2] for more info.
-# [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
-# [2] https://bugs.python.org/issue29288
-"".encode("idna")
-
-
-logger = logger.getChild(__name__)
-
-
-class DvcParserError(Exception):
-    """Base class for CLI parser errors."""
-
-    def __init__(self):
-        super().__init__("parser error")
-
-
-def parse_args(argv=None):
-    """Parses CLI arguments.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Raises:
-        DvcParserError: raised for argument parsing errors.
-    """
-    from .parser import get_main_parser
-
-    parser = get_main_parser()
-    args = parser.parse_args(argv)
-    args.parser = parser
-    return args
-
-
-def _log_unknown_exceptions() -> None:
-    from dvc.info import get_dvc_info
-    from dvc.ui import ui
-    from dvc.utils import colorize
-
-    logger.exception("unexpected error")
-    if logger.isEnabledFor(logging.DEBUG):
-        logger.debug("Version info for developers:\n%s", get_dvc_info())
-
-    q = colorize("Having any troubles?", "yellow")
-    link = colorize("https://dvc.org/support", "blue")
-    footer = f"\n{q} Hit us up at {link}, we are always happy to help!"
-    ui.error_write(footer)
-
-
-def _log_exceptions(exc: Exception) -> Optional[int]:
-    """Try to log some known exceptions, that are not DVCExceptions."""
-    from dvc.utils import error_link, format_link
-
-    if isinstance(exc, OSError):
-        import errno
-
-        if exc.errno == errno.EMFILE:
-            logger.exception(
-                (
-                    "too many open files, please visit "
-                    "%s to see how to handle this problem"
-                ),
-                error_link("many-files"),
-                extra={"tb_only": True},
-            )
-        else:
-            _log_unknown_exceptions()
-        return None
-
-    from dvc.fs import AuthError, ConfigError, RemoteMissingDepsError
-
-    if isinstance(exc, RemoteMissingDepsError):
-        from dvc import PKG
-
-        proto = exc.protocol
-        by_pkg = {
-            "pip": f"pip install 'dvc[{proto}]'",
-            "conda": f"conda install -c conda-forge dvc-{proto}",
-        }
-
-        if PKG in by_pkg:
-            link = format_link("https://dvc.org/doc/install")
-            cmd = by_pkg.get(PKG)
-            hint = (
-                "To install dvc with those dependencies, run:\n"
-                "\n"
-                f"\t{cmd}\n"
-                "\n"
-                f"See {link} for more info."
-            )
-        else:
-            link = format_link("https://github.com/iterative/dvc/issues")
-            hint = f"\nPlease report this bug to {link}. Thank you!"
-
-        logger.exception(
-            "URL '%s' is supported but requires these missing dependencies: %s. %s",
-            exc.url,
-            exc.missing_deps,
-            hint,
-            extra={"tb_only": True},
-        )
-        return None
-
-    if isinstance(exc, (AuthError, ConfigError)):
-        link = format_link("https://man.dvc.org/remote/modify")
-        logger.exception("configuration error")
-        logger.exception(
-            "%s\nLearn more about configuration settings at %s.",
-            exc,
-            link,
-            extra={"tb_only": True},
-        )
-        return 251
-
-    from dvc_data.hashfile.cache import DiskError
-
-    if isinstance(exc, DiskError):
-        from dvc.utils import relpath
-
-        directory = relpath(exc.directory)
-        logger.exception(
-            (
-                "Could not open pickled '%s' cache.\n"
-                "Remove the '%s' directory and then retry this command."
-                "\nSee %s for more information."
-            ),
-            exc.type,
-            directory,
-            error_link("pickle"),
-            extra={"tb_only": True},
-        )
-        return None
-
-    from dvc_data.hashfile.build import IgnoreInCollectedDirError
-
-    if isinstance(exc, IgnoreInCollectedDirError):
-        logger.exception("")
-        return None
-
-    _log_unknown_exceptions()
-    return None
-
-
-def main(argv=None):  # noqa: C901, PLR0912, PLR0915
-    """Main entry point for dvc CLI.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Returns:
-        int: command's return code.
-    """
-    from contextlib import ExitStack
-
-    from dvc._debug import debugtools
-    from dvc.config import ConfigError
-    from dvc.exceptions import DvcException, NotDvcRepoError
-    from dvc.logger import set_loggers_level
-
-    # NOTE: stderr/stdout may be closed if we are running from dvc.daemon.
-    # On Linux we directly call cli.main after double forking and closing
-    # the copied parent's standard file descriptors. If we make any logging
-    # calls in this state it will cause an exception due to writing to a closed
-    # file descriptor.
-    if not sys.stderr or sys.stderr.closed:
-        logging.disable()
-    elif not sys.stdout or sys.stdout.closed:
-        logging.disable(logging.INFO)
-
-    args = None
-    stack = ExitStack()
-    try:
-        args = parse_args(argv)
-
-        level = None
-        if args.quiet:
-            level = logging.CRITICAL
-        elif args.verbose == 1:
-            level = logging.DEBUG
-        elif args.verbose > 1:
-            level = logging.TRACE  # type: ignore[attr-defined]
-
-        if level is not None:
-            stack.enter_context(set_loggers_level(level))
-
-        if level and level <= logging.DEBUG:
-            from platform import platform, python_implementation, python_version
-
-            from dvc import PKG, __version__
-
-            pyv = f"{python_implementation()} {python_version()}"
-            pkg = f" ({PKG})" if PKG else ""
-            logger.debug("v%s%s, %s on %s", __version__, pkg, pyv, platform())
-            logger.debug("command: %s", " ".join(argv or sys.argv))
-
-        logger.trace(args)
-
-        if sys.stdout and not sys.stdout.closed and not args.quiet:
-            from dvc.ui import ui
-
-            ui.enable()
-
-        with debugtools(args):
-            cmd = args.func(args)
-            ret = cmd.do_run()
-    except ConfigError:
-        logger.exception("configuration error")
-        ret = 251
-    except KeyboardInterrupt:
-        logger.exception("interrupted by the user")
-        ret = 252
-    except BrokenPipeError:
-        import os
-
-        # Python flushes standard streams on exit; redirect remaining output
-        # to devnull to avoid another BrokenPipeError at shutdown
-        # See: https://docs.python.org/3/library/signal.html#note-on-sigpipe
-        devnull = os.open(os.devnull, os.O_WRONLY)
-        os.dup2(devnull, sys.stdout.fileno())
-        ret = 141  # 128 + 13 (SIGPIPE)
-    except NotDvcRepoError:
-        logger.exception("")
-        ret = 253
-    except DvcException:
-        ret = 255
-        logger.exception("")
-    except DvcParserError:
-        ret = 254
-    except Exception as exc:  # noqa: BLE001
-        ret = _log_exceptions(exc) or 255
-
-    try:
-        import os
-
-        from dvc import analytics
-
-        if analytics.is_enabled():
-            analytics.collect_and_send_report(args, ret)
-
-        logger.trace("Process %s exiting with %s", os.getpid(), ret)
-
-        return ret
-    finally:
-        stack.close()
-
-        from dvc.repo.open_repo import clean_repos
-
-        # Remove cached repos in the end of the call, these are anonymous
-        # so won't be reused by any other subsequent run anyway.
-        clean_repos()

dvc/cli/command.py

@@ -1,44 +0,0 @@
-import os
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any
-
-from dvc.log import logger
-
-if TYPE_CHECKING:
-    from dvc.config import Config
-
-logger = logger.getChild(__name__)
-
-
-class CmdBase(ABC):
-    UNINITIALIZED = False
-
-    def __init__(self, args: Any):
-        from dvc.repo import Repo
-
-        os.chdir(args.cd)
-
-        self.repo: Repo = Repo(
-            uninitialized=self.UNINITIALIZED,
-            _wait_for_lock=args.wait_for_lock,
-        )
-        self.config: Config = self.repo.config
-        self.args = args
-
-    def do_run(self):
-        with self.repo:
-            return self.run()
-
-    @abstractmethod
-    def run(self):
-        pass
-
-
-class CmdBaseNoRepo(CmdBase):
-    def __init__(self, args):
-        self.args = args
-
-        os.chdir(args.cd)
-
-    def do_run(self):
-        return self.run()

dvc/cli/completion.py

@@ -1,233 +0,0 @@
-import shtab
-
-BASH_PREAMBLE = """
-# $1=COMP_WORDS[1]
-_dvc_compgen_DVCFiles() {
-  compgen -d -S '/' -- $1  # recurse into subdirs
-  compgen -f -X '!*?.dvc' -- $1
-  compgen -f -X '!*Dvcfile' -- $1
-  compgen -f -X '!*dvc.yaml' -- $1
-}
-
-_dvc_compgen_stages() {
-    local _dvc_stages=($(dvc stage list -q --names-only))
-    compgen -W "${_dvc_stages[*]}" -- $1
-}
-_dvc_compgen_stages_and_files() {
-    _dvc_compgen_DVCFiles $1
-    _dvc_compgen_stages $1
-}
-
-_dvc_compgen_exps() {
-    local _dvc_exps=($(dvc exp list -q --all-commits --names-only))
-    compgen -W "${_dvc_exps[*]}" -- $1
-}
-
-_dvc_compgen_remotes() {
-    local _dvc_remotes=($(dvc remote list | cut -d' ' -f1))
-    compgen -W "${_dvc_remotes[*]}" -- $1
-}
-
-_dvc_compgen_config_vars() {
-    compgen -W "${_dvc_config_vars[*]}" -- $1
-}
-
-_dvc_compgen_remote_config_vars() {
-    local cur prev words cword remote_name filtered
-    _init_completion || return
-
-    # last non-option word before cursor
-    local nonopts=()
-    for w in "${words[@]:1:$((cword-1))}"; do
-        [[ $w == -* ]] || nonopts+=("$w")
-    done
-
-    remote_name="${nonopts[-1]}"
-    if [[ -n $remote_name ]]; then
-        local _dvc_remote_config_vars=($(
-            dvc config --available-options -q 2> /dev/null \
-            | grep "^remote\\.${remote_name}\\." \
-            | sed "s/^remote\\.${remote_name}\\.//"
-        ))
-        compgen -W "${_dvc_remote_config_vars[*]}" -- $1
-    fi
-}
-"""
-
-ZSH_PREAMBLE = """
-_dvc_compadd_DVCFiles() {
-    _files -g '(*?.dvc|Dvcfile|dvc.yaml)'
-}
-_dvc_compadd_stages() {
-    # this will also show up the description of the stages
-    _describe 'stages' "($(_dvc_stages_output))"
-}
-
-_dvc_stages_output() {
-  dvc stage list -q | awk '{
-    # escape possible `:` on the stage name
-    sub(/:/, "\\\\\\\\:", $1);
-    # read all of the columns except the first
-    # reading `out` from $2, so as not to have a leading whitespace
-    out=$2; for(i=3;i<=NF;i++){out=out" "$i};
-    # print key, ":" and then single-quote the description
-    # colon is a delimiter used by `_describe` to separate field/description
-    print $1":""\\047"out"\\047"
-    # single quote -> \\047
-    }'
-}
-
-_dvc_compadd_stages_and_files() {
-    _dvc_compadd_DVCFiles
-    _dvc_compadd_stages
-}
-
-_dvc_compadd_exps() {
-    _describe 'experiments' "($(dvc exp list -q --all-commits --names-only))"
-}
-
-_dvc_compadd_remotes() {
-    _describe 'remotes' "($(dvc remote list | cut -d' ' -f1))"
-}
-
-_dvc_config_config_vars_for_completion() {
-    dvc config --available-options -q 2> /dev/null
-}
-
-_dvc_compadd_config_vars() {
-    _describe 'config_vars' _dvc_config_vars
-}
-
-_dvc_compadd_remote_config_vars() {
-  local remote_name filtered
-
-  # last non-option word before cursor
-  remote_name=${${words[1,CURRENT-1]##-*}[-1]}
-  if [[ -n $remote_name ]]; then
-    filtered=$(_dvc_config_config_vars_for_completion \\
-        | grep "^remote.${remote_name}\\\\." \\
-        | sed "s/^remote\\\\.${remote_name}\\\\.//"
-    )
-    compadd -- $=filtered
-  fi
-}
-"""
-
-FISH_PREAMBLE = """
-function __fish_complete_dvc_files
-    __fish_complete_path | string match -re '\\*?.dvc|Dvcfile|dvc\\.yaml'
-end
-
-function __fish_complete_dvc_stages
-    for line in (dvc stage list -q)
-        set -l parts (string split -m1 ' ' -- $line)
-        set -l name $parts[1]
-        set -l desc (string trim $parts[2])
-        echo -e "$name	$desc"
-    end
-end
-
-function __fish_complete_dvc_stages_and_files
-    __fish_complete_dvc_stages
-    __fish_complete_dvc_files
-end
-
-function __fish_complete_dvc_experiments
-    dvc exp list -q --all-commits --names-only
-end
-
-function __fish_complete_dvc_remotes
-    dvc remote list | cut -d' ' -f1
-end
-
-function __fish_complete_dvc_remote_config_vars
-    # last non-option word before cursor
-    set -l remote_name (commandline -opc | string match -rv '^-' | tail -n 1)
-
-    if test -n "$remote_name"
-        dvc config --available-options -q 2>/dev/null |
-            string match -re "^remote\\.$remote_name\\." |
-            string replace -r "^remote\\.$remote_name\\." ''
-    end
-end
-"""
-
-PREAMBLE = {
-    "bash": BASH_PREAMBLE,
-    "zsh": ZSH_PREAMBLE,
-    "fish": FISH_PREAMBLE,
-}
-
-FILE = shtab.FILE
-DIR = shtab.DIRECTORY
-DVC_FILE = {
-    "bash": "_dvc_compgen_DVCFiles",
-    "zsh": "_dvc_compadd_DVCFiles",
-    "fish": '-f -a "(__fish_complete_dvc_files)"',
-}
-STAGE = {
-    "bash": "_dvc_compgen_stages",
-    "zsh": "_dvc_compadd_stages",
-    "fish": '-f -a "(__fish_complete_dvc_stages)"',
-}
-DVCFILES_AND_STAGE = {
-    "bash": "_dvc_compgen_stages_and_files",
-    "zsh": "_dvc_compadd_stages_and_files",
-    "fish": '-f -a "(__fish_complete_dvc_stages_and_files)"',
-}
-EXPERIMENT = {
-    "bash": "_dvc_compgen_exps",
-    "zsh": "_dvc_compadd_exps",
-    "fish": '-f -a "(__fish_complete_dvc_experiments)"',
-}
-REMOTE = {
-    "bash": "_dvc_compgen_remotes",
-    "zsh": "_dvc_compadd_remotes",
-    "fish": '-f -a "(__fish_complete_dvc_remotes)"',
-}
-CONFIG_VARS = {
-    "bash": "_dvc_compgen_config_vars",
-    "zsh": "_dvc_compadd_config_vars",
-    "fish": '-f -a "(__fish_complete_dvc_config_vars)"',
-}
-REMOTE_CONFIG_VARS = {
-    "bash": "_dvc_compgen_remote_config_vars",
-    "zsh": "_dvc_compadd_remote_config_vars",
-    "fish": '-f -a "(__fish_complete_dvc_remote_config_vars)"',
-}
-
-
-def get_preamble() -> dict[str, str]:
-    from dvc.config_schema import config_vars_for_completion
-
-    ret: dict[str, str] = {}
-    config_vars = list(config_vars_for_completion())
-
-    nl = "\n\t".expandtabs(4)
-    config_vars_arr = f"""
-_dvc_config_vars=(
-    {nl.join(config_vars)}
-)
-"""
-    indent = "\t\t".expandtabs(4)  # 8 spaces
-    lines = (
-        "\n".join(
-            f"{indent}{c} \\"
-            for c in config_vars[:-1]  # all but last
-        )
-        + "\n"
-        + f"{indent}{config_vars[-1]}"
-    )  # last line without backslash
-    config_vars_arr_fish = f"""
-function __fish_complete_dvc_config_vars
-    set -l _dvc_config_vars \\
-{lines}
-    printf %s\\n $_dvc_config_vars
-end
-"""
-    for shell, preamble in PREAMBLE.items():
-        if shell != "fish":
-            ret[shell] = config_vars_arr + preamble
-        else:
-            ret[shell] = config_vars_arr_fish + preamble
-    return ret

dvc/cli/formatter.py

@@ -1,14 +0,0 @@
-import argparse
-
-
-class HelpFormatter(argparse.HelpFormatter):
-    def _get_default_metavar_for_optional(self, action: argparse.Action) -> str:
-        return action.dest
-
-
-class RawTextHelpFormatter(HelpFormatter, argparse.RawTextHelpFormatter):
-    pass
-
-
-class RawDescriptionHelpFormatter(HelpFormatter, argparse.RawDescriptionHelpFormatter):
-    pass

dvc/cli/utils.py

@@ -1,46 +0,0 @@
-import argparse
-
-
-class DictAction(argparse.Action):
-    def __init__(self, *args, **kwargs):
-        kwargs.setdefault("metavar", "<name>=<value>")
-        super().__init__(*args, **kwargs)
-
-    def __call__(self, parser, args, values, option_string=None):  # noqa: ARG002
-        d = getattr(args, self.dest) or {}
-
-        if isinstance(values, list):
-            kvs = values
-        else:
-            kvs = [values]
-
-        for kv in kvs:
-            key, value = kv.split("=", 1)
-            if not value:
-                raise argparse.ArgumentError(
-                    self,
-                    f'Could not parse argument "{values}" as k1=v1 k2=v2 ... format',
-                )
-            d[key] = value
-
-        setattr(args, self.dest, d)
-
-
-def append_doc_link(help_message, path):
-    from dvc.utils import format_link
-
-    if not path:
-        return help_message
-    doc_base = "https://man.dvc.org/"
-    return f"{help_message}\nDocumentation: {format_link(doc_base + path)}"
-
-
-def hide_subparsers_from_help(subparsers):
-    # metavar needs to be explicitly set in order to hide subcommands
-    # from the 'positional arguments' choices list
-    # see: https://bugs.python.org/issue22848
-    # Need to set `add_help=False`, but avoid setting `help`
-    # (not even to `argparse.SUPPRESS`).
-    # NOTE: The argument is the parent subparser, not the subcommand parser.
-    cmds = [cmd for cmd, parser in subparsers.choices.items() if parser.add_help]
-    subparsers.metavar = "{{{}}}".format(",".join(cmds))

dvc/command/add.py

@@ -0,0 +1,85 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
+
+logger = logging.getLogger(__name__)
+
+
+class CmdAdd(CmdBase):
+    def run(self):
+        try:
+            if len(self.args.targets) > 1 and self.args.file:
+                raise RecursiveAddingWhileUsingFilename()
+
+            self.repo.add(
+                self.args.targets,
+                recursive=self.args.recursive,
+                no_commit=self.args.no_commit,
+                fname=self.args.file,
+                external=self.args.external,
+                glob=self.args.glob,
+                desc=self.args.desc,
+            )
+
+        except DvcException:
+            logger.exception("")
+            return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    ADD_HELP = "Track data files or directories with DVC."
+
+    parser = subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(ADD_HELP, "add"),
+        help=ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Recursively add files under directory targets.",
+    )
+    parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow targets that are outside of the DVC repository.",
+    )
+    parser.add_argument(
+        "--glob",
+        action="store_true",
+        default=False,
+        help="Allows targets containing shell-style wildcards.",
+    )
+    parser.add_argument(
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
+    )
+    parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
+    parser.add_argument(
+        "targets", nargs="+", help="Input files/directories to add.",
+    ).complete = completion.FILE
+    parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -0,0 +1,65 @@
+import logging
+import os
+from abc import ABC, abstractmethod
+
+logger = logging.getLogger(__name__)
+
+
+def fix_subparsers(subparsers):
+    """Workaround for bug in Python 3. See more info at:
+        https://bugs.python.org/issue16308
+        https://github.com/iterative/dvc/issues/769
+
+        Args:
+            subparsers: subparsers to fix.
+    """
+    subparsers.required = True
+    subparsers.dest = "cmd"
+
+
+def append_doc_link(help_message, path):
+    from dvc.utils import format_link
+
+    if not path:
+        return help_message
+    doc_base = "https://man.dvc.org/"
+    return "{message}\nDocumentation: {link}".format(
+        message=help_message, link=format_link(doc_base + path)
+    )
+
+
+class CmdBase(ABC):
+    UNINITIALIZED = False
+
+    def __init__(self, args):
+        from dvc.repo import Repo
+        from dvc.updater import Updater
+
+        os.chdir(args.cd)
+
+        self.repo = Repo(uninitialized=self.UNINITIALIZED)
+        self.config = self.repo.config
+        self.args = args
+        if self.repo.tmp_dir:
+            hardlink_lock = self.config["core"].get("hardlink_lock", False)
+            updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
+            updater.check()
+
+    @property
+    def default_targets(self):
+        """Default targets for `dvc repro`."""
+        from dvc.dvcfile import PIPELINE_FILE
+
+        logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
+        return [PIPELINE_FILE]
+
+    @abstractmethod
+    def run(self):
+        pass
+
+
+class CmdBaseNoRepo(CmdBase):
+    def __init__(self, args):  # pylint: disable=super-init-not-called
+        self.args = args
+
+        os.chdir(args.cd)

dvc/command/cache.py

@@ -0,0 +1,68 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import append_doc_link, fix_subparsers
+from dvc.command.config import CmdConfig
+
+logger = logging.getLogger(__name__)
+
+
+class CmdCacheDir(CmdConfig):
+    def run(self):
+        if self.args.value is None and not self.args.unset:
+            logger.info(self.config["cache"]["dir"])
+            return 0
+        with self.config.edit(level=self.args.level) as edit:
+            edit["cache"]["dir"] = self.args.value
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    CACHE_HELP = "Manage cache settings."
+
+    cache_parser = subparsers.add_parser(
+        "cache",
+        parents=[parent_parser],
+        description=append_doc_link(CACHE_HELP, "cache"),
+        help=CACHE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    cache_subparsers = cache_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc cache CMD --help` for command-specific " "help.",
+    )
+
+    fix_subparsers(cache_subparsers)
+
+    parent_cache_config_parser = argparse.ArgumentParser(
+        add_help=False, parents=[parent_config_parser]
+    )
+    CACHE_DIR_HELP = "Configure cache directory location."
+
+    cache_dir_parser = cache_subparsers.add_parser(
+        "dir",
+        parents=[parent_parser, parent_cache_config_parser],
+        description=append_doc_link(CACHE_HELP, "cache/dir"),
+        help=CACHE_DIR_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    cache_dir_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    cache_dir_parser.add_argument(
+        "value",
+        help="Path to cache directory. Relative paths are resolved relative "
+        "to the current directory and saved to config relative to the "
+        "config file location. If no path is provided, it returns the "
+        "current cache directory.",
+        nargs="?",
+    ).complete = completion.DIR
+    cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/check_ignore.py

@@ -1,40 +1,38 @@
-from typing import TYPE_CHECKING
+import argparse
+import logging
 
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
-if TYPE_CHECKING:
-    from dvc.ignore import CheckIgnoreResult
+logger = logging.getLogger(__name__)
 
 
 class CmdCheckIgnore(CmdBase):
     def __init__(self, args):
         super().__init__(args)
-        self.ignore_filter = self.repo.dvcignore
+        self.ignore_filter = self.repo.tree.dvcignore
 
-    def _show_results(self, result: "CheckIgnoreResult"):
-        if not result.match and not self.args.details:
-            return None
+    def _show_results(self, result):
+        if not result.match and not self.args.non_matching:
+            return
 
         if self.args.details:
-            pattern_infos = result.pattern_infos
-            patterns = [str(pi) for pi in pattern_infos]
-            if not patterns and self.args.non_matching:
-                patterns = ["::"]
+            patterns = result.patterns
             if not self.args.all:
                 patterns = patterns[-1:]
 
             for pattern in patterns:
-                ui.write(pattern, result.file, sep="\t")
+                logger.info("{}\t{}".format(pattern, result.file))
         else:
-            ui.write(result.file)
-        return bool(result.pattern_infos)
+            logger.info(result.file)
 
     def _check_one_file(self, target):
         result = self.ignore_filter.check_ignore(target)
-        return bool(self._show_results(result))
+        self._show_results(result)
+        if result.match:
+            return 0
+        return 1
 
     def _interactive_mode(self):
         ret = 1
@@ -43,22 +41,20 @@ class CmdCheckIgnore(CmdBase):
                 target = input()
             except (KeyboardInterrupt, EOFError):
                 break
-            if not target:
+            if target == "":
                 break
-            if self._check_one_file(target):
+            if not self._check_one_file(target):
                 ret = 0
         return ret
 
     def _normal_mode(self):
         ret = 1
         for target in self.args.targets:
-            if self._check_one_file(target):
+            if not self._check_one_file(target):
                 ret = 0
         return ret
 
     def _check_args(self):
-        from dvc.exceptions import DvcException
-
         if not self.args.stdin and not self.args.targets:
             raise DvcException("`targets` or `--stdin` needed")
 
@@ -66,7 +62,9 @@ class CmdCheckIgnore(CmdBase):
             raise DvcException("cannot have both `targets` and `--stdin`")
 
         if self.args.non_matching and not self.args.details:
-            raise DvcException("`--non-matching` is only valid with `--details`")
+            raise DvcException(
+                "`--non-matching` is only valid with `--details`"
+            )
 
         if self.args.all and not self.args.details:
             raise DvcException("`--all` is only valid with `--details`")
@@ -82,16 +80,16 @@ class CmdCheckIgnore(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    import argparse
-
-    ADD_HELP = "Check whether files or directories are excluded due to `.dvcignore`."
+    ADD_HELP = (
+        "Check whether files or directories are excluded due to `.dvcignore`."
+    )
 
     parser = subparsers.add_parser(
         "check-ignore",
         parents=[parent_parser],
         description=append_doc_link(ADD_HELP, "check-ignore"),
         help=ADD_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     parser.add_argument(
         "-d",
@@ -101,17 +99,20 @@ def add_parser(subparsers, parent_parser):
         help="Show the exclude patterns along with each target path.",
     )
     parser.add_argument(
-        "-a", "--all", action="store_true", default=False, help=argparse.SUPPRESS
+        "-a",
+        "--all",
+        action="store_true",
+        default=False,
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
     )
     parser.add_argument(
         "-n",
         "--non-matching",
         action="store_true",
         default=False,
-        help=(
-            "Include the target paths which don't match any pattern "
-            "in the `--details` list."
-        ),
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
     )
     parser.add_argument(
         "--stdin",
@@ -120,6 +121,6 @@ def add_parser(subparsers, parent_parser):
         help="Read paths from standard input instead of providing `targets`.",
     )
     parser.add_argument(
-        "targets", nargs="*", help="File or directory paths to check"
+        "targets", nargs="*", help="File or directory paths to check",
     ).complete = completion.FILE
     parser.set_defaults(func=CmdCheckIgnore)

dvc/command/checkout.py

@@ -1,61 +1,68 @@
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+import argparse
+import logging
+import operator
+
+import colorama
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
-from dvc.ui import ui
+from dvc.utils.humanize import get_summary
+
+logger = logging.getLogger(__name__)
 
 
 def log_changes(stats):
-    colors = {
-        "modified": "yellow",
-        "added": "green",
-        "deleted": "red",
-    }
+    colors = [
+        ("modified", colorama.Fore.YELLOW,),
+        ("added", colorama.Fore.GREEN),
+        ("deleted", colorama.Fore.RED,),
+    ]
 
-    for state, color in colors.items():
+    for state, color in colors:
         entries = stats.get(state)
 
         if not entries:
             continue
 
         for entry in entries:
-            ui.write(f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t")
+            logger.info(
+                "{color}{state}{nc}{spacing}{entry}".format(
+                    color=color,
+                    state=state[0].upper(),
+                    nc=colorama.Fore.RESET,
+                    spacing="\t",
+                    entry=entry,
+                )
+            )
 
 
 class CmdCheckout(CmdBase):
     def run(self):
-        from dvc.utils.humanize import get_summary
-
         stats, exc = None, None
         try:
-            result = self.repo.checkout(
+            stats = self.repo.checkout(
                 targets=self.args.targets,
                 with_deps=self.args.with_deps,
                 force=self.args.force,
                 relink=self.args.relink,
                 recursive=self.args.recursive,
-                allow_missing=self.args.allow_missing,
             )
         except CheckoutError as _exc:
             exc = _exc
-            result = exc.result
+            stats = exc.stats
 
         if self.args.summary:
             default_message = "No changes."
-            stats = result["stats"]
-            assert isinstance(stats, dict)
-            msg = get_summary(stats.items())
-            ui.write(msg or default_message)
+            msg = get_summary(
+                sorted(stats.items(), key=operator.itemgetter(0))
+            )
+            logger.info(msg or default_message)
         else:
-            result.pop("stats", {})
-            log_changes(result)
+            log_changes(stats)
 
         if exc:
             raise exc
-
-        if self.args.relink:
-            msg = "Relinked successfully"
-            ui.write(msg)
         return 0
 
 
@@ -67,7 +74,7 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(CHECKOUT_HELP, "checkout"),
         help=CHECKOUT_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     checkout_parser.add_argument(
         "--summary",
@@ -102,18 +109,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Recreate links or copies from cache to workspace.",
     )
-    checkout_parser.add_argument(
-        "--allow-missing",
-        action="store_true",
-        default=False,
-        help="Ignore errors if some of the files or directories are missing.",
-    )
     checkout_parser.add_argument(
         "targets",
         nargs="*",
         help=(
             "Limit command scope to these tracked files/directories, "
-            ".dvc files and stage names."
+            ".dvc files, or stage names."
         ),
     ).complete = completion.DVC_FILE
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -1,15 +1,15 @@
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.log import logger
+import argparse
+import logging
 
-logger = logger.getChild(__name__)
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 class CmdCommit(CmdBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         if not self.args.targets:
             self.args.targets = [None]
 
@@ -20,36 +20,33 @@ class CmdCommit(CmdBase):
                     with_deps=self.args.with_deps,
                     recursive=self.args.recursive,
                     force=self.args.force,
-                    relink=self.args.relink,
                 )
             except DvcException:
-                logger.exception("failed to commit%s", (" " + target) if target else "")
+                logger.exception(
+                    "failed to commit{}".format(
+                        (" " + target) if target else ""
+                    )
+                )
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    COMMIT_HELP = (
-        "Record changes to files or directories tracked by DVC"
-        " by storing the current versions in the cache."
-    )
+    COMMIT_HELP = "Save changed data to cache and update DVC-files."
 
     commit_parser = subparsers.add_parser(
         "commit",
         parents=[parent_parser],
         description=append_doc_link(COMMIT_HELP, "commit"),
         help=COMMIT_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     commit_parser.add_argument(
         "-f",
         "--force",
         action="store_true",
         default=False,
-        help=(
-            "Commit data even if hash values for dependencies or "
-            "outputs did not change."
-        ),
+        help="Commit even if hash value for dependencies/outputs changed.",
     )
     commit_parser.add_argument(
         "-d",
@@ -65,19 +62,10 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Commit cache for subdirectories of the specified directory.",
     )
-    commit_parser.add_argument(
-        "--no-relink",
-        dest="relink",
-        action="store_false",
-        help="Don't recreate links from cache to workspace.",
-    )
-    commit_parser.set_defaults(relink=True)
     commit_parser.add_argument(
         "targets",
         nargs="*",
-        help=(
-            "Limit command scope to these tracked files/directories, "
-            ".dvc files and stage names."
-        ),
-    ).complete = completion.DVCFILES_AND_STAGE
+        help="DVC-files to commit. Optional. "
+        "(Finds all DVC-files in the workspace by default.)",
+    ).complete = completion.DVC_FILE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -0,0 +1,57 @@
+import argparse
+import logging
+
+import shtab
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+
+logger = logging.getLogger(__name__)
+FILE = shtab.FILE
+DIR = shtab.DIRECTORY
+DVC_FILE = {
+    "bash": "_dvc_compgen_DVCFiles",
+    "zsh": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'",
+}
+PREAMBLE = {
+    "bash": """
+# $1=COMP_WORDS[1]
+_dvc_compgen_DVCFiles() {
+  compgen -d -S '/' -- $1  # recurse into subdirs
+  compgen -f -X '!*?.dvc' -- $1
+  compgen -f -X '!*Dvcfile' -- $1
+  compgen -f -X '!*dvc.yaml' -- $1
+}
+""",
+    "zsh": "",
+}
+
+
+class CmdCompletion(CmdBaseNoRepo):
+    def run(self):
+        from dvc.cli import get_main_parser
+
+        parser = get_main_parser()
+        shell = self.args.shell
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
+        print(script)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMPLETION_HELP = "Generate shell tab completion."
+    COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
+    completion_parser = subparsers.add_parser(
+        "completion",
+        parents=[parent_parser],
+        description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
+        help=COMPLETION_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    completion_parser.add_argument(
+        "-s",
+        "--shell",
+        help="Shell syntax for completions.",
+        default="bash",
+        choices=["bash", "zsh"],
+    )
+    completion_parser.set_defaults(func=CmdCompletion)

dvc/command/config.py

@@ -0,0 +1,125 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.config import Config, ConfigError
+
+logger = logging.getLogger(__name__)
+
+NAME_REGEX = r"^(?P<remote>remote\.)?(?P<section>[^\.]*)\.(?P<option>[^\.]*)$"
+
+
+def _name_type(value):
+    import re
+
+    match = re.match(NAME_REGEX, value)
+    if not match:
+        raise argparse.ArgumentTypeError(
+            "name argument should look like "
+            "remote.name.option or "
+            "section.option"
+        )
+    return (
+        bool(match.group("remote")),
+        match.group("section").lower(),
+        match.group("option").lower(),
+    )
+
+
+class CmdConfig(CmdBaseNoRepo):
+    def __init__(self, args):
+        super().__init__(args)
+
+        self.config = Config(validate=False)
+
+    def run(self):
+        remote, section, opt = self.args.name
+
+        if self.args.value is None and not self.args.unset:
+            conf = self.config.load_one(self.args.level)
+            if remote:
+                conf = conf["remote"]
+            self._check(conf, remote, section, opt)
+            logger.info(conf[section][opt])
+            return 0
+
+        with self.config.edit(self.args.level) as conf:
+            if remote:
+                conf = conf["remote"]
+            if self.args.unset:
+                self._check(conf, remote, section, opt)
+                del conf[section][opt]
+            else:
+                self._check(conf, remote, section)
+                conf[section][opt] = self.args.value
+
+        if self.args.name == "cache.type":
+            logger.warning(
+                "You have changed the 'cache.type' option. This doesn't update"
+                " any existing workspace file links, but it can be done with:"
+                "\n             dvc checkout --relink"
+            )
+
+        return 0
+
+    def _check(self, conf, remote, section, opt=None):
+        name = "remote" if remote else "section"
+        if section not in conf:
+            raise ConfigError(f"{name} '{section}' doesn't exist")
+
+        if opt and opt not in conf[section]:
+            raise ConfigError(
+                f"option '{opt}' doesn't exist in {name} '{section}'"
+            )
+
+
+parent_config_parser = argparse.ArgumentParser(add_help=False)
+level_group = parent_config_parser.add_mutually_exclusive_group()
+level_group.add_argument(
+    "--global",
+    dest="level",
+    action="store_const",
+    const="global",
+    help="Use global config.",
+)
+level_group.add_argument(
+    "--system",
+    dest="level",
+    action="store_const",
+    const="system",
+    help="Use system config.",
+)
+level_group.add_argument(
+    "--local",
+    dest="level",
+    action="store_const",
+    const="local",
+    help="Use local config.",
+)
+parent_config_parser.set_defaults(level="repo")
+
+
+def add_parser(subparsers, parent_parser):
+    CONFIG_HELP = "Get or set config options."
+
+    config_parser = subparsers.add_parser(
+        "config",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(CONFIG_HELP, "config"),
+        help=CONFIG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    config_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    config_parser.add_argument(
+        "name",
+        type=_name_type,
+        help="Option name (remote.name.option or section.option).",
+    )
+    config_parser.add_argument("value", nargs="?", help="Option value.")
+    config_parser.set_defaults(func=CmdConfig)

dvc/command/daemon.py

@@ -1,8 +1,5 @@
-from dvc.cli import completion
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.log import logger
-
-logger = logger.getChild(__name__)
+from dvc.command import completion
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):
@@ -23,8 +20,6 @@ class CmdDaemonUpdater(CmdDaemonBase):
         config = Config(dvc_dir, validate=False)
         hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
         updater = Updater(tmp_dir, hardlink_lock=hardlink_lock)
-
-        logger.info("Starting updater to fetch the latest version")
         updater.fetch(detach=False)
 
         return 0
@@ -34,7 +29,6 @@ class CmdDaemonAnalytics(CmdDaemonBase):
     def run(self):
         from dvc import analytics
 
-        logger.info("Sending analytics")
         analytics.send(self.args.target)
 
         return 0
@@ -51,10 +45,11 @@ def add_parser(subparsers, parent_parser):
 
     daemon_subparsers = daemon_parser.add_subparsers(
         dest="cmd",
-        help="Use `dvc daemon CMD --help` for command-specific help.",
-        required=True,
+        help="Use `dvc daemon CMD --help` for command-specific " "help.",
     )
 
+    fix_subparsers(daemon_subparsers)
+
     DAEMON_UPDATER_HELP = "Fetch latest available version."
     daemon_updater_parser = daemon_subparsers.add_parser(
         "updater",
@@ -72,6 +67,6 @@ def add_parser(subparsers, parent_parser):
         help=DAEMON_ANALYTICS_HELP,
     )
     daemon_analytics_parser.add_argument(
-        "target", help="Analytics file."
+        "target", help="Analytics file.",
     ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/dag.py

@@ -0,0 +1,149 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.stage import Stage
+
+logger = logging.getLogger(__name__)
+
+
+def _show_ascii(G):
+    from dvc.dagascii import draw
+    from dvc.repo.graph import get_pipelines
+
+    pipelines = get_pipelines(G)
+
+    ret = []
+    for pipeline in pipelines:
+        ret.append(draw(pipeline.nodes, pipeline.edges))
+
+    return "\n".join(ret)
+
+
+def _show_dot(G):
+    import io
+
+    from networkx.drawing.nx_pydot import write_dot
+
+    dot_file = io.StringIO()
+    write_dot(G, dot_file)
+    return dot_file.getvalue()
+
+
+def _collect_targets(repo, target, outs):
+    if not target:
+        return []
+
+    pairs = repo.stage.collect_granular(target)
+    if not outs:
+        return [stage.addressing for stage, _ in pairs]
+
+    targets = []
+    for stage, info in pairs:
+        if not info:
+            targets.extend([str(out) for out in stage.outs])
+            continue
+
+        for out in repo.outs_trie.itervalues(prefix=info.parts):  # noqa: B301
+            targets.extend(str(out))
+
+    return targets
+
+
+def _transform(repo, outs):
+    import networkx as nx
+
+    def _relabel(node) -> str:
+        return node.addressing if isinstance(node, Stage) else str(node)
+
+    G = repo.outs_graph if outs else repo.graph
+    return nx.relabel_nodes(G, _relabel, copy=True)
+
+
+def _filter(G, targets, full):
+    import networkx as nx
+
+    if not targets:
+        return G
+
+    H = G.copy()
+    if not full:
+        descendants = set()
+        for target in targets:
+            descendants.update(nx.descendants(G, target))
+            descendants.add(target)
+        H.remove_nodes_from(set(G.nodes()) - descendants)
+
+    undirected = H.to_undirected()
+    connected = set()
+    for target in targets:
+        connected.update(nx.node_connected_component(undirected, target))
+
+    H.remove_nodes_from(set(H.nodes()) - connected)
+
+    return H
+
+
+def _build(repo, target=None, full=False, outs=False):
+    targets = _collect_targets(repo, target, outs)
+    G = _transform(repo, outs)
+    return _filter(G, targets, full)
+
+
+class CmdDAG(CmdBase):
+    def run(self):
+        G = _build(
+            self.repo,
+            target=self.args.target,
+            full=self.args.full,
+            outs=self.args.outs,
+        )
+
+        if self.args.dot:
+            logger.info(_show_dot(G))
+        else:
+            from dvc.utils.pager import pager
+
+            pager(_show_ascii(G))
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    DAG_HELP = "Visualize DVC project DAG."
+    dag_parser = subparsers.add_parser(
+        "dag",
+        parents=[parent_parser],
+        description=append_doc_link(DAG_HELP, "dag"),
+        help=DAG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    dag_parser.add_argument(
+        "--dot",
+        action="store_true",
+        default=False,
+        help="Print DAG with .dot format.",
+    )
+    dag_parser.add_argument(
+        "--full",
+        action="store_true",
+        default=False,
+        help=(
+            "Show full DAG that the target belongs too, instead of "
+            "showing DAG consisting only of ancestors."
+        ),
+    )
+    dag_parser.add_argument(
+        "-o",
+        "--outs",
+        action="store_true",
+        default=False,
+        help="Print output files instead of stages.",
+    )
+    dag_parser.add_argument(
+        "target",
+        nargs="?",
+        help="Stage or output to show pipeline for (optional). "
+        "Finds all stages in the workspace by default.",
+    )
+    dag_parser.set_defaults(func=CmdDAG)

dvc/command/data_sync.py

@@ -1,39 +1,29 @@
 import argparse
+import logging
 
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.log import logger
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.checkout import log_changes
+from dvc.exceptions import CheckoutError, DvcException
+from dvc.utils.humanize import get_summary
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
     def log_summary(self, stats):
-        from dvc.ui import ui
-        from dvc.utils.humanize import get_summary
-
         default_msg = "Everything is up to date."
-
-        if not self.args.remote and not self.repo.config["core"].get("remote"):
-            ui.warn("No remote provided and no default remote set.")
-
-        ui.write(get_summary(stats.items()) or default_msg)
+        logger.info(get_summary(stats.items()) or default_msg)
 
 
 class CmdDataPull(CmdDataBase):
-    def log_summary(self, result):
-        from dvc.commands.checkout import log_changes
-
-        stats = result.pop("stats", {})
-        log_changes(result)
+    def log_summary(self, stats):
+        log_changes(stats)
         super().log_summary(stats)
 
     def run(self):
-        from dvc.exceptions import CheckoutError, DvcException
-
         try:
-            result = self.repo.pull(
+            stats = self.repo.pull(
                 targets=self.args.targets,
                 jobs=self.args.jobs,
                 remote=self.args.remote,
@@ -44,13 +34,10 @@ class CmdDataPull(CmdDataBase):
                 force=self.args.force,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
-                glob=self.args.glob,
-                allow_missing=self.args.allow_missing,
             )
-            self.log_summary(result)
+            self.log_summary(stats)
         except (CheckoutError, DvcException) as exc:
-            if result := getattr(exc, "result", {}):
-                self.log_summary(result)
+            self.log_summary(getattr(exc, "stats", {}))
             logger.exception("failed to pull data from the cloud")
             return 1
 
@@ -59,8 +46,6 @@ class CmdDataPull(CmdDataBase):
 
 class CmdDataPush(CmdDataBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
             processed_files_count = self.repo.push(
                 targets=self.args.targets,
@@ -72,7 +57,6 @@ class CmdDataPush(CmdDataBase):
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
-                glob=self.args.glob,
             )
             self.log_summary({"pushed": processed_files_count})
         except DvcException:
@@ -83,8 +67,6 @@ class CmdDataPush(CmdDataBase):
 
 class CmdDataFetch(CmdDataBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
             processed_files_count = self.repo.fetch(
                 targets=self.args.targets,
@@ -96,8 +78,6 @@ class CmdDataFetch(CmdDataBase):
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
-                max_size=self.args.max_size,
-                types=self.args.types,
             )
             self.log_summary({"fetched": processed_files_count})
         except DvcException:
@@ -107,7 +87,7 @@ class CmdDataFetch(CmdDataBase):
 
 
 def shared_parent_parser():
-    from dvc.cli.parser import get_parent_parser
+    from dvc.cli import get_parent_parser
 
     # Parent parser used in pull/push/status
     parent_parser = argparse.ArgumentParser(
@@ -120,6 +100,7 @@ def shared_parent_parser():
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )
@@ -128,15 +109,15 @@ def shared_parent_parser():
         nargs="*",
         help=(
             "Limit command scope to these tracked files/directories, "
-            ".dvc files and stage names."
+            ".dvc files, or stage names."
         ),
-    ).complete = completion.DVC_FILE  # type: ignore[attr-defined]
+    ).complete = completion.DVC_FILE
 
     return parent_parser
 
 
 def add_parser(subparsers, _parent_parser):
-    from dvc.commands.status import CmdDataStatus
+    from dvc.command.status import CmdDataStatus
 
     # Pull
     PULL_HELP = "Download tracked files or directories from remote storage."
@@ -146,11 +127,11 @@ def add_parser(subparsers, _parent_parser):
         parents=[shared_parent_parser()],
         description=append_doc_link(PULL_HELP, "pull"),
         help=PULL_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
-        "-r", "--remote", help="Remote storage to pull from", metavar="<name>"
-    ).complete = completion.REMOTE
+        "-r", "--remote", help="Remote storage to pull from", metavar="<name>",
+    )
     pull_parser.add_argument(
         "-a",
         "--all-branches",
@@ -166,7 +147,6 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     pull_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -195,22 +175,10 @@ def add_parser(subparsers, _parent_parser):
     )
     pull_parser.add_argument(
         "--run-cache",
-        action=argparse.BooleanOptionalAction,
+        action="store_true",
         default=False,
         help="Fetch run history for all stages.",
     )
-    pull_parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
-    )
-    pull_parser.add_argument(
-        "--allow-missing",
-        action="store_true",
-        default=False,
-        help="Ignore errors if some of the files or directories are missing.",
-    )
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
@@ -221,11 +189,11 @@ def add_parser(subparsers, _parent_parser):
         parents=[shared_parent_parser()],
         description=append_doc_link(PUSH_HELP, "push"),
         help=PUSH_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
-        "-r", "--remote", help="Remote storage to push to", metavar="<name>"
-    ).complete = completion.REMOTE
+        "-r", "--remote", help="Remote storage to push to", metavar="<name>",
+    )
     push_parser.add_argument(
         "-a",
         "--all-branches",
@@ -241,7 +209,6 @@ def add_parser(subparsers, _parent_parser):
         help="Push cache for all tags.",
     )
     push_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -263,31 +230,28 @@ def add_parser(subparsers, _parent_parser):
     )
     push_parser.add_argument(
         "--run-cache",
-        action=argparse.BooleanOptionalAction,
-        default=False,
-        help="Push run history for all stages.",
-    )
-    push_parser.add_argument(
-        "--glob",
         action="store_true",
         default=False,
-        help="Allows targets containing shell-style wildcards.",
+        help="Push run history for all stages.",
     )
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = "Download files or directories from remote storage to the cache."
+    FETCH_HELP = "Download cached files or directories from remote storage."
 
     fetch_parser = subparsers.add_parser(
         "fetch",
         parents=[shared_parent_parser()],
         description=append_doc_link(FETCH_HELP, "fetch"),
         help=FETCH_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
-        "-r", "--remote", help="Remote storage to fetch from", metavar="<name>"
-    ).complete = completion.REMOTE
+        "-r",
+        "--remote",
+        help="Remote storage to fetch from",
+        metavar="<name>",
+    )
     fetch_parser.add_argument(
         "-a",
         "--all-branches",
@@ -303,7 +267,6 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     fetch_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -314,7 +277,7 @@ def add_parser(subparsers, _parent_parser):
         "--with-deps",
         action="store_true",
         default=False,
-        help="Fetch cache for all dependencies of the specified target.",
+        help="Fetch cache for all dependencies of the " "specified target.",
     )
     fetch_parser.add_argument(
         "-R",
@@ -325,30 +288,16 @@ def add_parser(subparsers, _parent_parser):
     )
     fetch_parser.add_argument(
         "--run-cache",
-        action=argparse.BooleanOptionalAction,
+        action="store_true",
         default=False,
         help="Fetch run history for all stages.",
     )
-    fetch_parser.add_argument(
-        "--max-size",
-        type=int,
-        help="Fetch data files/directories that are each below specified size (bytes).",
-    )
-    fetch_parser.add_argument(
-        "--type",
-        dest="types",
-        action="append",
-        default=[],
-        help=(
-            "Only fetch data files/directories that are of a particular "
-            "type (metrics, plots)."
-        ),
-        choices=["metrics", "plots"],
-    )
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
-    STATUS_HELP = "Show changed stages, compare local cache and a remote storage."
+    STATUS_HELP = (
+        "Show changed stages, compare local cache and a remote storage."
+    )
 
     status_parser = subparsers.add_parser(
         "status",
@@ -356,7 +305,7 @@ def add_parser(subparsers, _parent_parser):
         description=append_doc_link(STATUS_HELP, "status"),
         help=STATUS_HELP,
         conflict_handler="resolve",
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     status_parser.add_argument(
         "-q",
@@ -380,35 +329,29 @@ def add_parser(subparsers, _parent_parser):
         "--remote",
         help="Remote storage to compare local cache to",
         metavar="<name>",
-    ).complete = completion.REMOTE
+    )
     status_parser.add_argument(
         "-a",
         "--all-branches",
         action="store_true",
         default=False,
-        help=(
-            "Show status of a local cache compared to a remote repository "
-            "for all branches."
-        ),
+        help="Show status of a local cache compared to a remote repository "
+        "for all branches.",
     )
     status_parser.add_argument(
         "-T",
         "--all-tags",
         action="store_true",
         default=False,
-        help=(
-            "Show status of a local cache compared to a remote repository for all tags."
-        ),
+        help="Show status of a local cache compared to a remote repository "
+        "for all tags.",
     )
     status_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
-        help=(
-            "Show status of a local cache compared to a remote repository "
-            "for all commits."
-        ),
+        help="Show status of a local cache compared to a remote repository "
+        "for all commits.",
     )
     status_parser.add_argument(
         "-d",
@@ -425,16 +368,10 @@ def add_parser(subparsers, _parent_parser):
         help="Show status of all stages in the specified directory.",
     )
     status_parser.add_argument(
-        "--json",
+        "--show-json",
         action="store_true",
         default=False,
         help="Show status in JSON format.",
     )
-    status_parser.add_argument(
-        "--no-updates",
-        dest="check_updates",
-        action="store_false",
-        help="Ignore updates to imported data.",
-    )
 
     status_parser.set_defaults(func=CmdDataStatus)

dvc/command/destroy.py

@@ -1,16 +1,15 @@
-from dvc.cli import formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.log import logger
+import argparse
+import logging
 
-logger = logger.getChild(__name__)
+import dvc.prompt as prompt
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 class CmdDestroy(CmdBase):
     def run(self):
-        from dvc.exceptions import DvcException
-        from dvc.ui import ui
-
         try:
             statement = (
                 "This will destroy all information about your pipelines,"
@@ -19,8 +18,8 @@ class CmdDestroy(CmdBase):
                 "Are you sure you want to continue?"
             )
 
-            if not self.args.force and not ui.confirm(statement):
-                raise DvcException(  # noqa: TRY301
+            if not self.args.force and not prompt.confirm(statement):
+                raise DvcException(
                     "cannot destroy without a confirmation from the user."
                     " Use `-f` to force."
                 )
@@ -33,14 +32,14 @@ class CmdDestroy(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
+    DESTROY_HELP = "Remove DVC-files, local DVC config and data cache."
 
     destroy_parser = subparsers.add_parser(
         "destroy",
         parents=[parent_parser],
         description=append_doc_link(DESTROY_HELP, "destroy"),
         help=DESTROY_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     destroy_parser.add_argument(
         "-f",

dvc/command/diff.py

@@ -1,12 +1,14 @@
+import argparse
+import json
+import logging
 import os
 
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.log import logger
-from dvc.ui import ui
+import colorama
 
-logger = logger.getChild(__name__)
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 def _digest(checksum):
@@ -15,33 +17,32 @@ def _digest(checksum):
     return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
 
 
-def _show_markdown(diff, show_hash=False, hide_missing=False):
-    headers = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
+def _show_md(diff, show_hash=False, hide_missing=False):
+    from dvc.utils.diff import table
+
+    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
-    statuses = ["added", "deleted", "renamed", "modified"]
+    statuses = ["added", "deleted", "modified"]
     if not hide_missing:
         statuses.append("not in cache")
-
     for status in statuses:
         entries = diff.get(status, [])
         if not entries:
             continue
         for entry in entries:
             path = entry["path"]
-            if isinstance(path, dict):
-                path = f"{path['old']} -> {path['new']}"
             if show_hash:
                 check_sum = _digest(entry.get("hash", ""))
                 rows.append([status, check_sum, path])
             else:
                 rows.append([status, path])
 
-    ui.table(rows, headers=headers, markdown=True)
+    return table(header, rows, True)
 
 
 class CmdDiff(CmdBase):
     @staticmethod
-    def _show_diff(diff, hide_missing=False):
+    def _format(diff, hide_missing=False):
         """
         Given a diff structure, generate a string of paths separated
         by new lines and grouped together by their state.
@@ -68,16 +69,16 @@ class CmdDiff(CmdBase):
         """
 
         colors = {
-            "added": "green",
-            "modified": "yellow",
-            "deleted": "red",
-            "renamed": "green",
-            "not in cache": "yellow",
+            "added": colorama.Fore.GREEN,
+            "modified": colorama.Fore.YELLOW,
+            "deleted": colorama.Fore.RED,
+            "not in cache": colorama.Fore.YELLOW,
         }
 
         summary = {}
+        groups = []
 
-        states = ["added", "deleted", "renamed", "modified"]
+        states = ["added", "deleted", "modified"]
         if not hide_missing:
             states.append("not in cache")
         for state in states:
@@ -87,64 +88,66 @@ class CmdDiff(CmdBase):
             if not entries:
                 continue
 
-            header = state.capitalize()
-            ui.write(f"[{colors[state]}]{header}[/]:", styled=True)
+            content = []
 
             for entry in entries:
                 path = entry["path"]
-                if isinstance(path, dict):
-                    path = f"{path['old']} -> {path['new']}"
                 checksum = entry.get("hash")
                 summary[state] += 1 if not path.endswith(os.sep) else 0
-                ui.write(
+                content.append(
                     "{space}{checksum}{separator}{path}".format(
                         space="    ",
                         checksum=_digest(checksum) if checksum else "",
                         separator="  " if checksum else "",
-                        path=path,
+                        path=entry["path"],
                     )
                 )
 
-            ui.write()
+            groups.append(
+                "{color}{header}{nc}:\n{content}".format(
+                    color=colors[state],
+                    header=state.capitalize(),
+                    nc=colorama.Fore.RESET,
+                    content="\n".join(content),
+                )
+            )
 
         if not sum(summary.values()):
-            return
+            return None
 
-        states_summary = ", ".join(
-            f"{summary[state]} {state}" for state in states if summary[state] > 0
+        fmt = (
+            "files summary: {added} added, {deleted} deleted,"
+            " {modified} modified"
         )
-        ui.write("files summary:", states_summary)
+        if not hide_missing:
+            fmt += ", {not in cache} not in cache"
+        groups.append(fmt.format_map(summary))
+
+        return "\n\n".join(groups)
 
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
-            diff = self.repo.diff(self.args.a_rev, self.args.b_rev, self.args.targets)
+            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
             show_hash = self.args.show_hash
             hide_missing = self.args.b_rev or self.args.hide_missing
             if hide_missing:
-                diff.pop("not in cache", None)
+                del diff["not in cache"]
 
             for key, entries in diff.items():
-                entries = sorted(
-                    entries,
-                    key=lambda entry: (
-                        entry["path"]["old"]
-                        if isinstance(entry["path"], dict)
-                        else entry["path"]
-                    ),
-                )
+                entries = sorted(entries, key=lambda entry: entry["path"])
                 if not show_hash:
                     for entry in entries:
                         del entry["hash"]
                 diff[key] = entries
 
-            if self.args.json:
-                ui.write_json(diff)
-            elif self.args.markdown:
-                _show_markdown(diff, show_hash, hide_missing)
+            if self.args.show_json:
+                logger.info(json.dumps(diff))
+            elif self.args.show_md:
+                logger.info(_show_md(diff, show_hash, hide_missing))
             elif diff:
-                self._show_diff(diff, hide_missing)
+                output = self._format(diff, hide_missing)
+                if output:
+                    logger.info(output)
 
         except DvcException:
             logger.exception("failed to get diff")
@@ -162,14 +165,8 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(DIFF_DESCRIPTION, "diff"),
         help=DIFF_DESCRIPTION,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    diff_parser.add_argument(
-        "--targets",
-        nargs="*",
-        help="Specific DVC-tracked files to compare. Accepts one or more file paths.",
-        metavar="<paths>",
-    ).complete = completion.FILE
     diff_parser.add_argument(
         "a_rev",
         help="Old Git commit to compare (defaults to HEAD)",
@@ -178,11 +175,11 @@ def add_parser(subparsers, parent_parser):
     )
     diff_parser.add_argument(
         "b_rev",
-        help="New Git commit to compare (defaults to the current workspace)",
+        help=("New Git commit to compare (defaults to the current workspace)"),
         nargs="?",
     )
     diff_parser.add_argument(
-        "--json",
+        "--show-json",
         help="Format the output into a JSON",
         action="store_true",
         default=False,
@@ -194,10 +191,9 @@ def add_parser(subparsers, parent_parser):
         default=False,
     )
     diff_parser.add_argument(
-        "--md",
+        "--show-md",
         help="Show tabulated output in the Markdown format (GFM).",
         action="store_true",
-        dest="markdown",
         default=False,
     )
     diff_parser.add_argument(

dvc/command/experiments.py

@@ -0,0 +1,903 @@
+import argparse
+import io
+import logging
+import os
+from collections import OrderedDict
+from collections.abc import Mapping
+from datetime import date, datetime
+from itertools import groupby
+from typing import Iterable, Optional
+
+import dvc.prompt as prompt
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.command.metrics import DEFAULT_PRECISION
+from dvc.command.repro import CmdRepro
+from dvc.command.repro import add_arguments as add_repro_arguments
+from dvc.exceptions import DvcException, InvalidArgumentError
+from dvc.repo.experiments import Experiments
+from dvc.scm.git import Git
+from dvc.utils.flatten import flatten
+
+logger = logging.getLogger(__name__)
+
+
+def _filter_names(
+    names: Iterable,
+    label: str,
+    include: Optional[Iterable],
+    exclude: Optional[Iterable],
+):
+    if include and exclude:
+        intersection = set(include) & set(exclude)
+        if intersection:
+            values = ", ".join(intersection)
+            raise InvalidArgumentError(
+                f"'{values}' specified in both --include-{label} and"
+                f" --exclude-{label}"
+            )
+
+    names = [tuple(name.split(".")) for name in names]
+
+    def _filter(filters, update_func):
+        filters = [tuple(name.split(".")) for name in filters]
+        for length, groups in groupby(filters, len):
+            for group in groups:
+                matches = [name for name in names if name[:length] == group]
+                if not matches:
+                    name = ".".join(group)
+                    raise InvalidArgumentError(
+                        f"'{name}' does not match any known {label}"
+                    )
+                update_func({match: None for match in matches})
+
+    if include:
+        ret: OrderedDict = OrderedDict()
+        _filter(include, ret.update)
+    else:
+        ret = OrderedDict({name: None for name in names})
+
+    if exclude:
+        _filter(exclude, ret.difference_update)  # type: ignore[attr-defined]
+
+    return [".".join(name) for name in ret]
+
+
+def _update_names(names, items):
+    for name, item in items:
+        if isinstance(item, dict):
+            item = flatten(item)
+            names.update(item.keys())
+        else:
+            names[name] = None
+
+
+def _collect_names(all_experiments, **kwargs):
+    metric_names = set()
+    param_names = set()
+
+    for _, experiments in all_experiments.items():
+        for exp in experiments.values():
+            _update_names(metric_names, exp.get("metrics", {}).items())
+            _update_names(param_names, exp.get("params", {}).items())
+
+    metric_names = _filter_names(
+        sorted(metric_names),
+        "metrics",
+        kwargs.get("include_metrics"),
+        kwargs.get("exclude_metrics"),
+    )
+    param_names = _filter_names(
+        sorted(param_names),
+        "params",
+        kwargs.get("include_params"),
+        kwargs.get("exclude_params"),
+    )
+
+    return metric_names, param_names
+
+
+def _collect_rows(
+    base_rev,
+    experiments,
+    metric_names,
+    param_names,
+    precision=DEFAULT_PRECISION,
+    no_timestamp=False,
+    sort_by=None,
+    sort_order=None,
+):
+    if sort_by:
+        if sort_by in metric_names:
+            sort_type = "metrics"
+        elif sort_by in param_names:
+            sort_type = "params"
+        else:
+            raise InvalidArgumentError(f"Unknown sort column '{sort_by}'")
+        reverse = sort_order == "desc"
+        experiments = _sort_exp(experiments, sort_by, sort_type, reverse)
+
+    new_checkpoint = True
+    for i, (rev, exp) in enumerate(experiments.items()):
+        row = []
+        style = None
+        queued = "*" if exp.get("queued", False) else ""
+
+        tip = exp.get("checkpoint_tip")
+        parent = ""
+        if rev == "baseline":
+            if Git.is_sha(base_rev):
+                name_rev = base_rev[:7]
+            else:
+                name_rev = base_rev
+            name = exp.get("name", name_rev)
+            row.append(f"{name}")
+            style = "bold"
+        else:
+            if tip:
+                parent_rev = exp.get("checkpoint_parent", "")
+                parent_exp = experiments.get(parent_rev, {})
+                parent_tip = parent_exp.get("checkpoint_tip")
+                if tip == parent_tip:
+                    if new_checkpoint:
+                        tree = "│ ╓"
+                    else:
+                        tree = "│ ╟"
+                    new_checkpoint = False
+                else:
+                    if parent_rev == base_rev:
+                        tree = "├─╨"
+                    else:
+                        tree = "│ ╟"
+                        parent = f" ({parent_rev[:7]})"
+                    new_checkpoint = True
+            else:
+                if i < len(experiments) - 1:
+                    tree = "├──"
+                else:
+                    tree = "└──"
+                new_checkpoint = True
+            name = exp.get("name", rev[:7])
+            row.append(f"{tree} {queued}{name}{parent}")
+
+        if not no_timestamp:
+            row.append(_format_time(exp.get("timestamp")))
+
+        _extend_row(
+            row, metric_names, exp.get("metrics", {}).items(), precision
+        )
+        _extend_row(row, param_names, exp.get("params", {}).items(), precision)
+
+        yield row, style
+
+
+def _sort_exp(experiments, sort_by, typ, reverse):
+    def _sort(item):
+        rev, exp = item
+        tip = exp.get("checkpoint_tip")
+        if tip and tip != rev:
+            # Sort checkpoint experiments by tip commit
+            return _sort((tip, experiments[tip]))
+        for fname, item in exp.get(typ, {}).items():
+            if isinstance(item, dict):
+                item = flatten(item)
+            else:
+                item = {fname: item}
+            if sort_by in item:
+                val = item[sort_by]
+                return (val is None, val)
+        return (True, None)
+
+    ret = OrderedDict()
+    if "baseline" in experiments:
+        ret["baseline"] = experiments.pop("baseline")
+
+    ret.update(sorted(experiments.items(), key=_sort, reverse=reverse))
+    return ret
+
+
+def _format_time(timestamp):
+    if timestamp is None:
+        return "-"
+    if timestamp.date() == date.today():
+        fmt = "%I:%M %p"
+    else:
+        fmt = "%b %d, %Y"
+    return timestamp.strftime(fmt)
+
+
+def _format_field(val, precision=DEFAULT_PRECISION):
+    if isinstance(val, float):
+        fmt = f"{{:.{precision}g}}"
+        return fmt.format(val)
+    elif isinstance(val, Mapping):
+        return {k: _format_field(v) for k, v in val.items()}
+    elif isinstance(val, list):
+        return [_format_field(x) for x in val]
+    return str(val)
+
+
+def _extend_row(row, names, items, precision):
+    from rich.text import Text
+
+    if not items:
+        row.extend(["-"] * len(names))
+        return
+
+    for fname, item in items:
+        if isinstance(item, dict):
+            item = flatten(item)
+        else:
+            item = {fname: item}
+        for name in names:
+            if name in item:
+                value = item[name]
+                if value is None:
+                    text = "-"
+                else:
+                    # wrap field data in rich.Text, otherwise rich may
+                    # interpret unescaped braces from list/dict types as rich
+                    # markup tags
+                    text = Text(str(_format_field(value, precision)))
+                row.append(text)
+            else:
+                row.append("-")
+
+
+def _parse_list(param_list):
+    ret = []
+    for param_str in param_list:
+        # we don't care about filename prefixes for show, silently
+        # ignore it if provided to keep usage consistent with other
+        # metric/param list command options
+        _, _, param_str = param_str.rpartition(":")
+        ret.extend(param_str.split(","))
+    return ret
+
+
+def _show_experiments(all_experiments, console, **kwargs):
+    from rich.table import Table
+
+    include_metrics = _parse_list(kwargs.pop("include_metrics", []))
+    exclude_metrics = _parse_list(kwargs.pop("exclude_metrics", []))
+    include_params = _parse_list(kwargs.pop("include_params", []))
+    exclude_params = _parse_list(kwargs.pop("exclude_params", []))
+
+    metric_names, param_names = _collect_names(
+        all_experiments,
+        include_metrics=include_metrics,
+        exclude_metrics=exclude_metrics,
+        include_params=include_params,
+        exclude_params=exclude_params,
+    )
+
+    table = Table()
+    table.add_column("Experiment", no_wrap=True)
+    if not kwargs.get("no_timestamp", False):
+        table.add_column("Created")
+    for name in metric_names:
+        table.add_column(name, justify="right", no_wrap=True)
+    for name in param_names:
+        table.add_column(name, justify="left")
+
+    for base_rev, experiments in all_experiments.items():
+        for row, _, in _collect_rows(
+            base_rev, experiments, metric_names, param_names, **kwargs,
+        ):
+            table.add_row(*row)
+
+    console.print(table)
+
+
+def _format_json(item):
+    if isinstance(item, (date, datetime)):
+        return item.isoformat()
+    raise TypeError
+
+
+class CmdExperimentsShow(CmdBase):
+    def run(self):
+        from rich.console import Console
+
+        from dvc.utils.pager import pager
+
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            all_experiments = self.repo.experiments.show(
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                sha_only=self.args.sha,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(all_experiments, default=_format_json))
+                return 0
+
+            if self.args.no_pager:
+                console = Console()
+            else:
+                # Note: rich does not currently include a native way to force
+                # infinite width for use with a pager
+                console = Console(
+                    file=io.StringIO(), force_terminal=True, width=9999
+                )
+
+            if self.args.precision is None:
+                precision = DEFAULT_PRECISION
+            else:
+                precision = self.args.precision
+
+            _show_experiments(
+                all_experiments,
+                console,
+                include_metrics=self.args.include_metrics,
+                exclude_metrics=self.args.exclude_metrics,
+                include_params=self.args.include_params,
+                exclude_params=self.args.exclude_params,
+                no_timestamp=self.args.no_timestamp,
+                sort_by=self.args.sort_by,
+                sort_order=self.args.sort_order,
+                precision=precision,
+            )
+
+            if not self.args.no_pager:
+                pager(console.file.getvalue())
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
+
+        return 0
+
+
+class CmdExperimentsApply(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        self.repo.experiments.apply(self.args.experiment)
+
+        return 0
+
+
+def _show_diff(
+    diff,
+    title="",
+    markdown=False,
+    no_path=False,
+    old=False,
+    precision=DEFAULT_PRECISION,
+):
+    from dvc.utils.diff import table
+
+    rows = []
+    for fname, diff_ in diff.items():
+        sorted_diff = OrderedDict(sorted(diff_.items()))
+        for item, change in sorted_diff.items():
+            row = [] if no_path else [fname]
+            row.append(item)
+            if old:
+                row.append(_format_field(change.get("old"), precision))
+            row.append(_format_field(change["new"], precision))
+            row.append(
+                _format_field(
+                    change.get("diff", "diff not supported"), precision
+                )
+            )
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append(title)
+    if old:
+        header.extend(["Old", "New"])
+    else:
+        header.append("Value")
+    header.append("Change")
+
+    return table(header, rows, markdown)
+
+
+class CmdExperimentsDiff(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            diff = self.repo.experiments.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                if self.args.precision is None:
+                    precision = DEFAULT_PRECISION
+                else:
+                    precision = self.args.precision
+
+                diffs = [("metrics", "Metric"), ("params", "Param")]
+                for key, title in diffs:
+                    table = _show_diff(
+                        diff[key],
+                        title=title,
+                        markdown=self.args.show_md,
+                        no_path=self.args.no_path,
+                        old=self.args.old,
+                        precision=precision,
+                    )
+                    if table:
+                        logger.info(table)
+                        logger.info("")
+
+        except DvcException:
+            logger.exception("failed to show experiments diff")
+            return 1
+
+        return 0
+
+
+class CmdExperimentsRun(CmdRepro):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        saved_dir = os.path.realpath(os.curdir)
+        os.chdir(self.args.cwd)
+
+        # Dirty hack so the for loop below can at least enter once
+        if self.args.all_pipelines:
+            self.args.targets = [None]
+        elif not self.args.targets:
+            self.args.targets = self.default_targets
+
+        ret = 0
+        for target in self.args.targets:
+            try:
+                self.repo.experiments.run(
+                    target,
+                    name=self.args.name,
+                    queue=self.args.queue,
+                    run_all=self.args.run_all,
+                    jobs=self.args.jobs,
+                    params=self.args.params,
+                    checkpoint_resume=self.args.checkpoint_resume,
+                    **self._repro_kwargs,
+                )
+            except DvcException:
+                logger.exception("")
+                ret = 1
+                break
+
+        os.chdir(saved_dir)
+        return ret
+
+
+class CmdExperimentsGC(CmdRepro):
+    def run(self):
+        from dvc.repo.gc import _raise_error_if_all_disabled
+
+        if not self.repo.experiments:
+            return 0
+
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+        )
+
+        msg = "This will remove all experiments except those derived from "
+
+        msg += "the workspace"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+        msg += " of the current repo."
+        if self.args.queued:
+            msg += " Run queued experiments will be preserved."
+        if self.args.queued:
+            msg += " Run queued experiments will be removed."
+
+        logger.warning(msg)
+
+        msg = "Are you sure you want to proceed?"
+        if not self.args.force and not prompt.confirm(msg):
+            return 1
+
+        removed = self.repo.experiments.gc(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+            queued=self.args.queued,
+        )
+
+        if removed:
+            logger.info(
+                f"Removed {removed} experiments. To remove unused cache files "
+                "use 'dvc gc'."
+            )
+        else:
+            logger.info("No experiments to remove.")
+        return 0
+
+
+class CmdExperimentsBranch(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        self.repo.experiments.branch(self.args.experiment, self.args.branch)
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    EXPERIMENTS_HELP = "Commands to display and compare experiments."
+
+    experiments_parser = subparsers.add_parser(
+        "experiments",
+        parents=[parent_parser],
+        aliases=["exp"],
+        description=append_doc_link(EXPERIMENTS_HELP, "experiments"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    experiments_subparsers = experiments_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc experiments CMD --help` to display "
+        "command-specific help.",
+    )
+
+    fix_subparsers(experiments_subparsers)
+
+    EXPERIMENTS_SHOW_HELP = "Print experiments."
+    experiments_show_parser = experiments_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_SHOW_HELP, "experiments/show"),
+        help=EXPERIMENTS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_show_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show metrics for all branches.",
+    )
+    experiments_show_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show metrics for all tags.",
+    )
+    experiments_show_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show metrics for all commits.",
+    )
+    experiments_show_parser.add_argument(
+        "--no-pager",
+        action="store_true",
+        default=False,
+        help="Do not pipe output into a pager.",
+    )
+    experiments_show_parser.add_argument(
+        "--include-metrics",
+        action="append",
+        default=[],
+        help="Include the specified metrics in output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-metrics",
+        action="append",
+        default=[],
+        help="Exclude the specified metrics from output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--include-params",
+        action="append",
+        default=[],
+        help="Include the specified params in output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-params",
+        action="append",
+        default=[],
+        help="Exclude the specified params from output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-by",
+        help="Sort related experiments by the specified metric or param.",
+        metavar="<metric/param>",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-order",
+        help="Sort order to use with --sort-by.",
+        choices=("asc", "desc"),
+        default="asc",
+    )
+    experiments_show_parser.add_argument(
+        "--no-timestamp",
+        action="store_true",
+        default=False,
+        help="Do not show experiment timestamps.",
+    )
+    experiments_show_parser.add_argument(
+        "--sha",
+        action="store_true",
+        default=False,
+        help="Always show git commit SHAs instead of branch/tag names.",
+    )
+    experiments_show_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Print output in JSON format instead of a human-readable table.",
+    )
+    experiments_show_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_show_parser.set_defaults(func=CmdExperimentsShow)
+
+    EXPERIMENTS_APPLY_HELP = (
+        "Apply the changes from an experiment to your workspace."
+    )
+    experiments_apply_parser = experiments_subparsers.add_parser(
+        "apply",
+        parents=[parent_parser],
+        description=append_doc_link(
+            EXPERIMENTS_APPLY_HELP, "experiments/apply"
+        ),
+        help=EXPERIMENTS_APPLY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_apply_parser.add_argument(
+        "experiment", help="Experiment to be applied.",
+    )
+    experiments_apply_parser.set_defaults(func=CmdExperimentsApply)
+
+    EXPERIMENTS_DIFF_HELP = (
+        "Show changes between experiments in the DVC repository."
+    )
+    experiments_diff_parser = experiments_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_DIFF_HELP, "experiments/diff"),
+        help=EXPERIMENTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old experiment to compare (defaults to HEAD)"
+    )
+    experiments_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help="New experiment to compare (defaults to the current workspace)",
+    )
+    experiments_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics/params as well.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    experiments_diff_parser.add_argument(
+        "--old",
+        action="store_true",
+        default=False,
+        help="Show old metric/param value.",
+    )
+    experiments_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric/param path.",
+    )
+    experiments_diff_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)
+
+    EXPERIMENTS_RUN_HELP = (
+        "Reproduce complete or partial experiment pipelines."
+    )
+    experiments_run_parser = experiments_subparsers.add_parser(
+        "run",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_RUN_HELP, "experiments/run"),
+        help=EXPERIMENTS_RUN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    _add_run_common(experiments_run_parser)
+    experiments_run_parser.add_argument(
+        "--checkpoint-resume", type=str, default=None, help=argparse.SUPPRESS,
+    )
+    experiments_run_parser.set_defaults(func=CmdExperimentsRun)
+
+    EXPERIMENTS_RESUME_HELP = "Resume checkpoint experiments."
+    experiments_resume_parser = experiments_subparsers.add_parser(
+        "resume",
+        parents=[parent_parser],
+        aliases=["res"],
+        description=append_doc_link(
+            EXPERIMENTS_RESUME_HELP, "experiments/resume"
+        ),
+        help=EXPERIMENTS_RESUME_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    _add_run_common(experiments_resume_parser)
+    experiments_resume_parser.add_argument(
+        "-r",
+        "--rev",
+        type=str,
+        default=Experiments.LAST_CHECKPOINT,
+        dest="checkpoint_resume",
+        help=(
+            "Continue the specified checkpoint experiment. "
+            "If no experiment revision is provided, "
+            "the most recently run checkpoint experiment will be used."
+        ),
+        metavar="<experiment_rev>",
+    )
+    experiments_resume_parser.set_defaults(func=CmdExperimentsRun)
+
+    EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
+    EXPERIMENTS_GC_DESCRIPTION = (
+        "Removes all experiments which are not derived from the specified"
+        "Git revisions."
+    )
+    experiments_gc_parser = experiments_subparsers.add_parser(
+        "gc",
+        parents=[parent_parser],
+        description=append_doc_link(
+            EXPERIMENTS_GC_DESCRIPTION, "experiments/gc"
+        ),
+        help=EXPERIMENTS_GC_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the current workspace.",
+    )
+    experiments_gc_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the tips of all Git branches.",
+    )
+    experiments_gc_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git tags.",
+    )
+    experiments_gc_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git commits.",
+    )
+    experiments_gc_parser.add_argument(
+        "--queued",
+        action="store_true",
+        default=False,
+        help=(
+            "Keep queued experiments (experiments run queue will be cleared "
+            "by default)."
+        ),
+    )
+    experiments_gc_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force garbage collection - automatically agree to all prompts.",
+    )
+    experiments_gc_parser.set_defaults(func=CmdExperimentsGC)
+
+    EXPERIMENTS_BRANCH_HELP = "Promote an experiment to a Git branch."
+    experiments_branch_parser = experiments_subparsers.add_parser(
+        "branch",
+        parents=[parent_parser],
+        description=append_doc_link(
+            EXPERIMENTS_BRANCH_HELP, "experiments/branch"
+        ),
+        help=EXPERIMENTS_BRANCH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_branch_parser.add_argument(
+        "experiment", help="Experiment to be promoted.",
+    )
+    experiments_branch_parser.add_argument(
+        "branch", help="Git branch name to use.",
+    )
+    experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)
+
+
+def _add_run_common(parser):
+    """Add common args for 'exp run' and 'exp resume'."""
+    # inherit arguments from `dvc repro`
+    add_repro_arguments(parser)
+    parser.add_argument(
+        "-n",
+        "--name",
+        default=None,
+        help=(
+            "Human-readable experiment name. If not specified, a name will "
+            "be auto-generated."
+        ),
+    )
+    parser.add_argument(
+        "--params",
+        action="append",
+        default=[],
+        help="Use the specified param values when reproducing pipelines.",
+        metavar="[<filename>:]<params_list>",
+    )
+    parser.add_argument(
+        "--queue",
+        action="store_true",
+        default=False,
+        help="Stage this experiment in the run queue for future execution.",
+    )
+    parser.add_argument(
+        "--run-all",
+        action="store_true",
+        default=False,
+        help="Execute all experiments in the run queue.",
+    )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help="Run the specified number of experiments at a time in parallel.",
+        metavar="<number>",
+    )

dvc/command/freeze.py

@@ -1,10 +1,11 @@
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.exceptions import DvcException
-from dvc.log import logger
+import argparse
+import logging
 
-logger = logger.getChild(__name__)
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 class CmdFreezeBase(CmdBase):
@@ -14,7 +15,7 @@ class CmdFreezeBase(CmdBase):
             try:
                 func(target)
             except DvcException:
-                logger.exception("failed to %s '%s'", name, target)
+                logger.exception(f"failed to {name} '{target}'")
                 ret = 1
         return ret
 
@@ -36,10 +37,10 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(FREEZE_HELP, "freeze"),
         help=FREEZE_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to freeze"
+        "targets", nargs="+", help="Stages or .dvc files to freeze",
     ).complete = completion.DVC_FILE
     freeze_parser.set_defaults(func=CmdFreeze)
 
@@ -49,9 +50,9 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(UNFREEZE_HELP, "unfreeze"),
         help=UNFREEZE_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to unfreeze"
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
     ).complete = completion.DVC_FILE
     unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -0,0 +1,144 @@
+import argparse
+import logging
+import os
+
+import dvc.prompt as prompt
+from dvc.command.base import CmdBase, append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdGC(CmdBase):
+    def run(self):
+        from dvc.repo.gc import _raise_error_if_all_disabled
+
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+        )
+
+        msg = "This will remove all cache except items used in "
+
+        msg += "the workspace"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+
+        if self.args.repos:
+            msg += " of the current and the following repos:"
+
+            for repo_path in self.args.repos:
+                msg += "\n  - %s" % os.path.abspath(repo_path)
+        else:
+            msg += " of the current repo."
+
+        logger.warning(msg)
+
+        msg = "Are you sure you want to proceed?"
+        if not self.args.force and not prompt.confirm(msg):
+            return 1
+
+        self.repo.gc(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            cloud=self.args.cloud,
+            remote=self.args.remote,
+            force=self.args.force,
+            jobs=self.args.jobs,
+            repos=self.args.repos,
+            workspace=self.args.workspace,
+        )
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    GC_HELP = "Garbage collect unused objects from cache or remote storage."
+    GC_DESCRIPTION = (
+        "Removes all files in the cache or a remote which are not in\n"
+        "use by the specified Git revisions (defaults to just HEAD)."
+    )
+    gc_parser = subparsers.add_parser(
+        "gc",
+        parents=[parent_parser],
+        description=append_doc_link(GC_DESCRIPTION, "gc"),
+        help=GC_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep data files used in the current workspace.",
+    )
+    gc_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Keep data files for the tips of all Git branches.",
+    )
+    gc_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Keep data files for all Git tags.",
+    )
+    gc_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Keep data files for all Git commits.",
+    )
+    gc_parser.add_argument(
+        "-c",
+        "--cloud",
+        action="store_true",
+        default=False,
+        help="Collect garbage in remote repository.",
+    )
+    gc_parser.add_argument(
+        "-r",
+        "--remote",
+        help="Remote storage to collect garbage in",
+        metavar="<name>",
+    )
+    gc_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force garbage collection - automatically agree to all prompts.",
+    )
+    gc_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
+    gc_parser.add_argument(
+        "-p",
+        "--projects",
+        dest="repos",
+        type=str,
+        nargs="*",
+        help="Keep data files required by these projects "
+        "in addition to the current one. "
+        "Useful if you share a single cache across repos.",
+        metavar="<paths>",
+    )
+    gc_parser.set_defaults(func=CmdGC)

dvc/command/get.py

@@ -0,0 +1,87 @@
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+
+from . import completion
+from .base import CmdBaseNoRepo, append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdGet(CmdBaseNoRepo):
+    def _show_url(self):
+        from dvc.api import get_url
+
+        try:
+            url = get_url(
+                self.args.path, repo=self.args.url, rev=self.args.rev
+            )
+            logger.info(url)
+        except DvcException:
+            logger.exception("failed to show URL")
+            return 1
+
+        return 0
+
+    def run(self):
+        if self.args.show_url:
+            return self._show_url()
+
+        return self._get_file_from_repo()
+
+    def _get_file_from_repo(self):
+        from dvc.repo import Repo
+
+        try:
+            Repo.get(
+                self.args.url,
+                path=self.args.path,
+                out=self.args.out,
+                rev=self.args.rev,
+            )
+            return 0
+        except DvcException:
+            logger.exception(
+                "failed to get '{}' from '{}'".format(
+                    self.args.path, self.args.url
+                )
+            )
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    GET_HELP = "Download file or directory tracked by DVC or by Git."
+    get_parser = subparsers.add_parser(
+        "get",
+        parents=[parent_parser],
+        description=append_doc_link(GET_HELP, "get"),
+        help=GET_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    get_parser.add_argument(
+        "url", help="Location of DVC or Git repository to download from"
+    )
+    get_parser.add_argument(
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
+    get_parser.add_argument(
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to download files to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    get_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    get_parser.add_argument(
+        "--show-url",
+        action="store_true",
+        help="Print the storage location (URL) the target data would be "
+        "downloaded from, and exit.",
+    )
+    get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -0,0 +1,39 @@
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+
+from . import completion
+from .base import CmdBaseNoRepo, append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdGetUrl(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            Repo.get_url(self.args.url, out=self.args.out)
+            return 0
+        except DvcException:
+            logger.exception(f"failed to get '{self.args.url}'")
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    GET_HELP = "Download or copy files from URL."
+    get_parser = subparsers.add_parser(
+        "get-url",
+        parents=[parent_parser],
+        description=append_doc_link(GET_HELP, "get-url"),
+        help=GET_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    get_parser.add_argument(
+        "url", help="See `dvc import-url -h` for full list of supported URLs."
+    )
+    get_parser.add_argument(
+        "out", nargs="?", help="Destination path to put data to.",
+    ).complete = completion.DIR
+    get_parser.set_defaults(func=CmdGetUrl)

dvc/command/git_hook.py

@@ -1,11 +1,10 @@
+import logging
 import os
 
-from dvc.cli import formatter
-from dvc.cli.command import CmdBaseNoRepo
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 from dvc.exceptions import NotDvcRepoError
-from dvc.log import logger
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 class CmdHookBase(CmdBaseNoRepo):
@@ -26,7 +25,7 @@ class CmdHookBase(CmdBaseNoRepo):
 
 class CmdPreCommit(CmdHookBase):
     def _run(self):
-        from dvc.cli import main
+        from dvc.main import main
 
         return main(["status"])
 
@@ -50,33 +49,34 @@ class CmdPostCheckout(CmdHookBase):
         if os.path.isdir(os.path.join(".git", "rebase-merge")):
             return 0
 
-        from dvc.cli import main
+        from dvc.main import main
 
         return main(["checkout"])
 
 
 class CmdPrePush(CmdHookBase):
     def _run(self):
-        from dvc.cli import main
+        from dvc.main import main
 
         return main(["push"])
 
 
 class CmdMergeDriver(CmdHookBase):
     def _run(self):
-        from dvc.dvcfile import load_file
+        from dvc.dvcfile import Dvcfile
         from dvc.repo import Repo
 
         dvc = Repo()
 
         try:
-            ancestor = load_file(dvc, self.args.ancestor, verify=False)
-            our = load_file(dvc, self.args.our, verify=False)
-            their = load_file(dvc, self.args.their, verify=False)
+            with dvc.state:
+                ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
+                our = Dvcfile(dvc, self.args.our, verify=False)
+                their = Dvcfile(dvc, self.args.their, verify=False)
 
-            our.merge(ancestor, their, allowed=["add", "remove", "change"])
+                our.merge(ancestor, their)
 
-            return 0
+                return 0
         finally:
             dvc.close()
 
@@ -94,9 +94,10 @@ def add_parser(subparsers, parent_parser):
     git_hook_subparsers = git_hook_parser.add_subparsers(
         dest="cmd",
         help="Use `dvc daemon CMD --help` for command-specific help.",
-        required=True,
     )
 
+    fix_subparsers(git_hook_subparsers)
+
     PRE_COMMIT_HELP = "Run pre-commit GIT hook."
     pre_commit_parser = git_hook_subparsers.add_parser(
         "pre-commit",
@@ -105,7 +106,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_COMMIT_HELP,
     )
     pre_commit_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
     )
     pre_commit_parser.set_defaults(func=CmdPreCommit)
 
@@ -117,7 +118,7 @@ def add_parser(subparsers, parent_parser):
         help=POST_CHECKOUT_HELP,
     )
     post_checkout_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
     )
     post_checkout_parser.set_defaults(func=CmdPostCheckout)
 
@@ -129,7 +130,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_PUSH_HELP,
     )
     pre_push_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
     )
     pre_push_parser.set_defaults(func=CmdPrePush)
 
@@ -139,7 +140,6 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=MERGE_DRIVER_HELP,
         help=MERGE_DRIVER_HELP,
-        formatter_class=formatter.HelpFormatter,
     )
     merge_driver_parser.add_argument(
         "--ancestor",
@@ -147,7 +147,9 @@ def add_parser(subparsers, parent_parser):
         help="Ancestor's version of the conflicting file.",
     )
     merge_driver_parser.add_argument(
-        "--our", required=True, help="Current version of the conflicting file."
+        "--our",
+        required=True,
+        help="Current version of the conflicting file.",
     )
     merge_driver_parser.add_argument(
         "--their",

dvc/command/imp.py

@@ -0,0 +1,85 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdImport(CmdBase):
+    def run(self):
+        try:
+            self.repo.imp(
+                self.args.url,
+                self.args.path,
+                out=self.args.out,
+                fname=self.args.file,
+                rev=self.args.rev,
+                no_exec=self.args.no_exec,
+                desc=self.args.desc,
+            )
+        except DvcException:
+            logger.exception(
+                "failed to import '{}' from '{}'.".format(
+                    self.args.path, self.args.url
+                )
+            )
+            return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    IMPORT_HELP = (
+        "Download file or directory tracked by DVC or by Git "
+        "into the workspace, and track it."
+    )
+
+    import_parser = subparsers.add_parser(
+        "import",
+        parents=[parent_parser],
+        description=append_doc_link(IMPORT_HELP, "import"),
+        help=IMPORT_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    import_parser.add_argument(
+        "url", help="Location of DVC or Git repository to download from"
+    )
+    import_parser.add_argument(
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
+    import_parser.add_argument(
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to download files to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    import_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    import_parser.add_argument(
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
+    )
+    import_parser.add_argument(
+        "--no-exec",
+        action="store_true",
+        default=False,
+        help="Only create DVC-file without actually downloading it.",
+    )
+    import_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
+    import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -0,0 +1,80 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdImportUrl(CmdBase):
+    def run(self):
+        try:
+            self.repo.imp_url(
+                self.args.url,
+                out=self.args.out,
+                fname=self.args.file,
+                no_exec=self.args.no_exec,
+                desc=self.args.desc,
+            )
+        except DvcException:
+            logger.exception(
+                "failed to import {}. You could also try downloading "
+                "it manually, and adding it with `dvc add`.".format(
+                    self.args.url
+                )
+            )
+            return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    IMPORT_HELP = (
+        "Download or copy file from URL and take it under DVC control."
+    )
+
+    import_parser = subparsers.add_parser(
+        "import-url",
+        parents=[parent_parser],
+        description=append_doc_link(IMPORT_HELP, "import-url"),
+        help=IMPORT_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    import_parser.add_argument(
+        "url",
+        help="Location of the data to download. Supported URLs:\n"
+        "/absolute/path/to/file/or/dir\n"
+        "relative/path/to/file/or/dir\n"
+        "C:\\\\path\\to\\file\\or\\dir\n"
+        "https://example.com/path/to/file\n"
+        "s3://bucket/key/path\n"
+        "gs://bucket/path/to/file/or/dir\n"
+        "hdfs://example.com/path/to/file\n"
+        "ssh://example.com/absolute/path/to/file/or/dir\n"
+        "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
+    )
+    import_parser.add_argument(
+        "out", nargs="?", help="Destination path to put files to.",
+    ).complete = completion.DIR
+    import_parser.add_argument(
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
+    ).complete = completion.DIR
+    import_parser.add_argument(
+        "--no-exec",
+        action="store_true",
+        default=False,
+        help="Only create DVC-file without actually downloading it.",
+    )
+    import_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
+    import_parser.set_defaults(func=CmdImportUrl)

dvc/command/init.py

@@ -1,21 +1,19 @@
+import argparse
+import logging
+
 import colorama
 
 from dvc import analytics
-from dvc.cli import formatter
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
-from dvc.log import logger
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.utils import boxify
 from dvc.utils import format_link as fmt_link
 
-logger = logger.getChild(__name__)
+logger = logging.getLogger(__name__)
 
 
 def _welcome_message():
-    from dvc.ui import ui
-
     if analytics.is_enabled():
-        ui.write(
+        logger.info(
             boxify(
                 "DVC has enabled anonymous aggregate usage analytics.\n"
                 "Read the analytics documentation (and how to opt-out) here:\n"
@@ -32,7 +30,7 @@ def _welcome_message():
         f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
     ).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
 
-    ui.write(msg)
+    logger.info(msg)
 
 
 class CmdInit(CmdBaseNoRepo):
@@ -41,14 +39,14 @@ class CmdInit(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            with Repo.init(
-                self.args.directory,
+            self.repo = Repo.init(
+                ".",
                 no_scm=self.args.no_scm,
                 force=self.args.force,
                 subdir=self.args.subdir,
-            ) as repo:
-                self.config = repo.config
-                _welcome_message()
+            )
+            self.config = self.repo.config
+            _welcome_message()
         except InitError:
             logger.exception("failed to initiate DVC")
             return 1
@@ -57,11 +55,10 @@ class CmdInit(CmdBaseNoRepo):
 
 def add_parser(subparsers, parent_parser):
     """Setup parser for `dvc init`."""
-    INIT_HELP = "Initialize DVC repository."
+    INIT_HELP = "Initialize DVC in the current directory."
     INIT_DESCRIPTION = (
-        "Initialize DVC repository in the given directory (defaults to the current "
-        "working directory).\n"
-        "Expects directory to be a Git repository unless --no-scm option is specified."
+        "Initialize DVC in the current directory. Expects directory\n"
+        "to be a Git repository unless --no-scm option is specified."
     )
 
     init_parser = subparsers.add_parser(
@@ -69,23 +66,14 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(INIT_DESCRIPTION, "init"),
         help=INIT_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
-    )
-    init_parser.add_argument(
-        "directory",
-        nargs="?",
-        default=".",
-        help=(
-            "Directory to initialize DVC in. Defaults to the current working directory."
-        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     init_parser.add_argument(
         "--no-scm",
         action="store_true",
         default=False,
-        help=(
-            "Initiate DVC in directory that is not tracked by any SCM tool (e.g. Git)."
-        ),
+        help="Initiate DVC in directory that is "
+        "not tracked by any SCM tool (e.g. Git).",
     )
     init_parser.add_argument(
         "-f",
@@ -93,7 +81,8 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
         help=(
-            "Overwrite existing '.dvc/' directory. This operation removes local cache."
+            "Overwrite existing '.dvc/' directory. "
+            "This operation removes local cache."
         ),
     )
     init_parser.add_argument(

dvc/command/install.py

@@ -1,10 +1,10 @@
-from dvc.cli import formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.exceptions import DvcException
-from dvc.log import logger
+import argparse
+import logging
 
-logger = logger.getChild(__name__)
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 class CmdInstall(CmdBase):
@@ -24,15 +24,13 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(INSTALL_HELP, "install"),
         help=INSTALL_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     install_parser.add_argument(
         "--use-pre-commit-tool",
         action="store_true",
         default=False,
-        help=(
-            "Install DVC hooks using pre-commit "
-            "(https://pre-commit.com) if it is installed."
-        ),
+        help="Install DVC hooks using pre-commit "
+        "(https://pre-commit.com) if it is installed.",
     )
     install_parser.set_defaults(func=CmdInstall)

dvc/command/ls/__init__.py

@@ -0,0 +1,79 @@
+import argparse
+import logging
+import sys
+
+from dvc.command import completion
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.ls.ls_colors import LsColors
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _prettify(entries, with_color=False):
+    if with_color:
+        ls_colors = LsColors()
+        fmt = ls_colors.format
+    else:
+
+        def fmt(entry):
+            return entry["path"]
+
+    return [fmt(entry) for entry in entries]
+
+
+class CmdList(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            entries = Repo.ls(
+                self.args.url,
+                self.args.path,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+                dvc_only=self.args.dvc_only,
+            )
+            if entries:
+                entries = _prettify(entries, sys.stdout.isatty())
+                logger.info("\n".join(entries))
+            return 0
+        except DvcException:
+            logger.exception(f"failed to list '{self.args.url}'")
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    LIST_HELP = (
+        "List repository contents, including files"
+        " and directories tracked by DVC and by Git."
+    )
+    list_parser = subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(LIST_HELP, "list"),
+        help=LIST_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    list_parser.add_argument("url", help="Location of DVC repository to list")
+    list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        help="Recursively list files.",
+    )
+    list_parser.add_argument(
+        "--dvc-only", action="store_true", help="Show only DVC outputs."
+    )
+    list_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    list_parser.add_argument(
+        "path",
+        nargs="?",
+        help="Path to directory within the repository to list outputs for",
+    ).complete = completion.DIR
+    list_parser.set_defaults(func=CmdList)

dvc/command/ls/ls_colors.py

@@ -32,9 +32,7 @@ class LsColors:
         if entry.get("isexec", False):
             return self._format(text, code="ex")
 
-        stem, ext = os.path.splitext(text)
-        if not ext and stem.startswith("."):
-            ext = stem
+        _, ext = os.path.splitext(text)
         return self._format(text, ext=ext)
 
     def _format(self, text, code=None, ext=None):

dvc/command/metrics.py

@@ -1,99 +1,143 @@
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.log import logger
-from dvc.ui import ui
-from dvc.utils.serialize import encode_exception
+import argparse
+import logging
 
-logger = logger.getChild(__name__)
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import BadMetricError, DvcException
+
+logger = logging.getLogger(__name__)
 
 
 DEFAULT_PRECISION = 5
 
 
+def _show_metrics(
+    metrics, all_branches=False, all_tags=False, all_commits=False
+):
+    from dvc.utils.diff import format_dict
+    from dvc.utils.flatten import flatten
+
+    # When `metrics` contains a `None` key, it means that some files
+    # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
+    missing = metrics.pop(None, None)
+
+    lines = []
+    for branch, val in metrics.items():
+        if all_branches or all_tags or all_commits:
+            lines.append(f"{branch}:")
+
+        for fname, metric in val.items():
+            if not isinstance(metric, dict):
+                lines.append("\t{}: {}".format(fname, str(metric)))
+                continue
+
+            lines.append(f"\t{fname}:")
+            for key, value in flatten(format_dict(metric)).items():
+                lines.append(f"\t\t{key}: {value}")
+
+    if missing:
+        raise BadMetricError(missing)
+
+    return "\n".join(lines)
+
+
 class CmdMetricsBase(CmdBase):
     UNINITIALIZED = True
 
 
 class CmdMetricsShow(CmdMetricsBase):
     def run(self):
-        from dvc.repo.metrics.show import to_relpath
-        from dvc.utils import errored_revisions
-
-        metrics = self.repo.metrics.show(
-            self.args.targets,
-            all_branches=self.args.all_branches,
-            all_tags=self.args.all_tags,
-            all_commits=self.args.all_commits,
-        )
-        metrics = {
-            k: to_relpath(self.repo.fs, self.repo.root_dir, v)
-            for k, v in metrics.items()
-        }
-
-        if errored := errored_revisions(metrics):
-            ui.error_write(
-                "DVC failed to load some metrics for following revisions:"
-                f" '{', '.join(errored)}'."
-            )
-
-        if self.args.json:
-            ui.write_json(metrics, default=encode_exception)
-        else:
-            from dvc.compare import show_metrics
-
-            show_metrics(
-                metrics,
-                markdown=self.args.markdown,
+        try:
+            metrics = self.repo.metrics.show(
+                self.args.targets,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
                 all_commits=self.args.all_commits,
-                precision=self.args.precision or DEFAULT_PRECISION,
-                round_digits=True,
+                recursive=self.args.recursive,
             )
 
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(metrics))
+            else:
+                table = _show_metrics(
+                    metrics,
+                    self.args.all_branches,
+                    self.args.all_tags,
+                    self.args.all_commits,
+                )
+                if table:
+                    logger.info(table)
+        except DvcException:
+            logger.exception("")
+            return 1
+
         return 0
 
 
+def _show_diff(diff, markdown=False, no_path=False, precision=None):
+    from collections import OrderedDict
+
+    from dvc.utils.diff import table
+
+    if precision is None:
+        precision = DEFAULT_PRECISION
+
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+
+        return val
+
+    rows = []
+    for fname, mdiff in diff.items():
+        sorted_mdiff = OrderedDict(sorted(mdiff.items()))
+        for metric, change in sorted_mdiff.items():
+            row = [] if no_path else [fname]
+            row.append(metric)
+            row.append(_round(change.get("old")))
+            row.append(_round(change["new"]))
+            row.append(_round(change.get("diff")))
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append("Metric")
+    header.extend(["Old", "New"])
+    header.append("Change")
+
+    return table(header, rows, markdown)
+
+
 class CmdMetricsDiff(CmdMetricsBase):
     def run(self):
-        import os
-        from os.path import relpath
-
-        diff_result = self.repo.metrics.diff(
-            a_rev=self.args.a_rev,
-            b_rev=self.args.b_rev,
-            targets=self.args.targets,
-            all=self.args.all,
-        )
-
-        errored = [rev for rev, err in diff_result.get("errors", {}).items() if err]
-        if errored:
-            ui.error_write(
-                "DVC failed to load some metrics for following revisions:"
-                f" '{', '.join(errored)}'."
-            )
-
-        start = relpath(os.getcwd(), self.repo.root_dir)
-        diff = diff_result.get("diff", {})
-        diff = {relpath(path, start): result for path, result in diff.items()}
-
-        if self.args.json:
-            ui.write_json(diff)
-        else:
-            from dvc.compare import show_diff
-
-            show_diff(
-                diff,
-                title="Metric",
-                markdown=self.args.markdown,
-                no_path=self.args.no_path,
-                precision=self.args.precision or DEFAULT_PRECISION,
-                round_digits=True,
+        try:
+            diff = self.repo.metrics.diff(
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
+                targets=self.args.targets,
+                recursive=self.args.recursive,
+                all=self.args.all,
             )
 
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                table = _show_diff(
+                    diff,
+                    self.args.show_md,
+                    self.args.no_path,
+                    precision=self.args.precision,
+                )
+                if table:
+                    logger.info(table)
+
+        except DvcException:
+            logger.exception("failed to show metrics diff")
+            return 1
+
         return 0
 
 
@@ -105,22 +149,23 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(METRICS_HELP, "metrics"),
         help=METRICS_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     metrics_subparsers = metrics_parser.add_subparsers(
         dest="cmd",
         help="Use `dvc metrics CMD --help` to display command-specific help.",
-        required=True,
     )
 
+    fix_subparsers(metrics_subparsers)
+
     METRICS_SHOW_HELP = "Print metrics, with optional formatting."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
         parents=[parent_parser],
         description=append_doc_link(METRICS_SHOW_HELP, "metrics/show"),
         help=METRICS_SHOW_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_show_parser.add_argument(
         "targets",
@@ -145,25 +190,17 @@ def add_parser(subparsers, parent_parser):
         help="Show metrics for all tags.",
     )
     metrics_show_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
         help="Show metrics for all commits.",
     )
     metrics_show_parser.add_argument(
-        "--json",
+        "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
-    metrics_show_parser.add_argument(
-        "--md",
-        action="store_true",
-        default=False,
-        dest="markdown",
-        help="Show tabulated output in the Markdown format (GFM).",
-    )
     metrics_show_parser.add_argument(
         "-R",
         "--recursive",
@@ -174,15 +211,6 @@ def add_parser(subparsers, parent_parser):
             "metrics files."
         ),
     )
-    metrics_show_parser.add_argument(
-        "--precision",
-        type=int,
-        help=(
-            "Round metrics to `n` digits precision after the decimal point. "
-            f"Rounds to {DEFAULT_PRECISION} digits by default."
-        ),
-        metavar="<n>",
-    )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
     METRICS_DIFF_HELP = (
@@ -194,17 +222,13 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(METRICS_DIFF_HELP, "metrics/diff"),
         help=METRICS_DIFF_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_diff_parser.add_argument(
-        "a_rev",
-        nargs="?",
-        help="Old Git commit to compare (defaults to HEAD)",
-        default="HEAD",
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
     )
     metrics_diff_parser.add_argument(
         "b_rev",
-        default="workspace",
         nargs="?",
         help="New Git commit to compare (defaults to the current workspace)",
     )
@@ -212,11 +236,8 @@ def add_parser(subparsers, parent_parser):
         "--targets",
         nargs="*",
         help=(
-            "Specific metrics file(s) to compare "
-            "(even if not found as `metrics` in `dvc.yaml`). "
-            "Using -R, directories to search metrics files in "
-            "can also be given."
-            "Shows all tracked metrics by default."
+            "Limit command scope to these metrics files. Using -R, "
+            "directories to search metrics files in can also be given."
         ),
         metavar="<paths>",
     ).complete = completion.FILE
@@ -237,16 +258,15 @@ def add_parser(subparsers, parent_parser):
         help="Show unchanged metrics as well.",
     )
     metrics_diff_parser.add_argument(
-        "--json",
+        "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
     metrics_diff_parser.add_argument(
-        "--md",
+        "--show-md",
         action="store_true",
         default=False,
-        dest="markdown",
         help="Show tabulated output in the Markdown format (GFM).",
     )
     metrics_diff_parser.add_argument(

dvc/command/move.py

@@ -1,10 +1,11 @@
-from dvc.cli import completion, formatter
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.exceptions import DvcException
-from dvc.log import logger
+import argparse
+import logging
 
-logger = logger.getChild(__name__)
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 class CmdMove(CmdBase):
@@ -12,7 +13,9 @@ class CmdMove(CmdBase):
         try:
             self.repo.move(self.args.src, self.args.dst)
         except DvcException:
-            msg = f"failed to move '{self.args.src}' -> '{self.args.dst}'"
+            msg = "failed to move '{}' -> '{}'".format(
+                self.args.src, self.args.dst
+            )
             logger.exception(msg)
             return 1
         return 0
@@ -22,20 +25,21 @@ def add_parser(subparsers, parent_parser):
     MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
     MOVE_DESCRIPTION = (
         "Rename or move a DVC controlled data file or a directory.\n"
-        "It renames and modifies the corresponding .dvc file to reflect the"
+        "It renames and modifies the corresponding DVC-file to reflect the"
         " changes."
     )
 
     move_parser = subparsers.add_parser(
         "move",
-        aliases=["mv"],
         parents=[parent_parser],
         description=append_doc_link(MOVE_DESCRIPTION, "move"),
         help=MOVE_HELP,
-        formatter_class=formatter.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src", help="Source path to a data file or directory."
+        "src", help="Source path to a data file or directory.",
+    ).complete = completion.FILE
+    move_parser.add_argument(
+        "dst", help="Destination path.",
     ).complete = completion.FILE
-    move_parser.add_argument("dst", help="Destination path.").complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)